diff --git a/recipes/deploy/deploy_cps_test.yaml b/recipes/deploy/deploy_cps_test.yaml new file mode 100644 index 0000000..78f8dce --- /dev/null +++ b/recipes/deploy/deploy_cps_test.yaml @@ -0,0 +1,36 @@ +concurrency_groups: + copy_raw_serial: + limit: 1 + +flows: + copy_raw: + deployment_name: copy-raw-cps-test + interval: 3 + inject_time_offset: true + concurrency_group: copy_raw_serial + + raw2Sv: + deployment_name: raw2Sv-cps-test + triggers: + - expect: "echodataflow.raw.updated" + resource_name: "raw-monitor" + resource_scope: primary + + simulate_transects: + deployment_name: simulate-transects-cps-test + interval: 3 + + process_CPS: + deployment_name: process-CPS-cps-test + triggers: + - expect: "echodataflow.transect.updated" + resource_name: "transect-monitor" + resource_scope: related + + - expect: "echodataflow.sv.updated" + resource_name: "sv-monitor" + resource_scope: primary + + update_cache_CPS: + deployment_name: update-cache-CPS-cps-test + interval: 1 \ No newline at end of file diff --git a/recipes/params/params_cps_test.yaml b/recipes/params/params_cps_test.yaml new file mode 100644 index 0000000..149ae46 --- /dev/null +++ b/recipes/params/params_cps_test.yaml @@ -0,0 +1,63 @@ +flows: + copy_raw: + path_raw_list: "/path/to/SH2407_raw_list_organized.csv" + path_copy: "/path/to/cps_workflow/raw" + s3_bucket: noaa-wcsd-pds + exclude_before: "2024-08-05T13:10:00+00:00" + exclude_after: "2024-08-06T12:30:00+00:00" + endpoint_url: "https://s3.amazonaws.com" + + simulate_transects: + path_transect_csv: "/path/to/cps_workflow/plotSurvey_Survey_Data_Visualizer.csv" + survey_start: "2024-08-05T13:16:00+00:00" + transect_duration_minutes: 20 + start_transect_num: 1 + max_transects: 20 + + raw2Sv: + exclude_before: "2024-08-05T13:10:00+00:00" + exclude_raw_file: [] + parallel: false + new_file_num_limit: 10 + + encode_mode: power + waveform_mode: CW + depth_offset: 7 + sonar_model: EK80 + datagram_type: + nmea_sentence: GGA + + add_depth: true + add_location: true + add_splitbeam_angle: true + + path_main: "/path/to/cps_workflow" + processing_db: "postgresql+psycopg://USER:PASSWORD@HOST:5432/DATABASE" + + process_CPS: + path_transect_csv: "/path/to/cps_workflow/plotSurvey_Survey_Data_Visualizer.csv" + path_snapshot_csv: "/path/to/cps_workflow/plotSurvey_Survey_Data_Visualizer_snapshot.csv" + path_main: "/path/to/cps_workflow" + processing_db: "postgresql+psycopg://USER:PASSWORD@HOST:5432/DATABASE" + + target_frequency: 70000 + min_depth: 10 + + seafloor_threshold: [-40, 2.4, 1.0] + seafloor_offset: 0.5 + seafloor_r0: 10 + seafloor_r1: 1000 + seafloor_wtheta: 28 + seafloor_wphi: 52 + + mask_mode: cps + fallback_sv_threshold: -70 + + range_bin: 10m + dist_bin: 0.5nmi + + update_cache_CPS: + path_CPS: "/path/to/cps_workflow/CPS_Masks_Zarr" + path_cache: "/path/to/cps_workflow/viz_cache_CPS" + path_transect_csv: "/path/to/cps_workflow/plotSurvey_Survey_Data_Visualizer.csv" + file_CPS_zarr: latest_CPS.zarr \ No newline at end of file diff --git a/scripts/watch_raw_updates.py b/scripts/watch_raw_updates.py index 255045f..b6b122d 100644 --- a/scripts/watch_raw_updates.py +++ b/scripts/watch_raw_updates.py @@ -2,30 +2,41 @@ import time from pathlib import Path -from echodataflow.utils.raw_monitor import watch_raw_directory +from echodataflow.operations.operations_watchdog import watch_raw_directory def main(): parser = argparse.ArgumentParser( - description="Watch a RAW directory and emit Prefect update events." + description="Watch a RAW directory and register RAW updates." ) parser.add_argument( "path", type=Path, help="Path to the RAW directory to watch.", ) + parser.add_argument( + "--db-path", + type=str, + required=True, + help="Processing database path or SQLAlchemy database URL.", + ) args = parser.parse_args() target_dir = args.path.resolve() + db_path = args.db_path if not target_dir.exists(): raise FileNotFoundError( f"RAW directory does not exist: {target_dir}" ) - observer = watch_raw_directory(target_dir) + observer = watch_raw_directory( + target_dir, + db_path=db_path, + ) print(f"Watching {target_dir}") + print(f"Processing ledger: {db_path}") try: while True: diff --git a/tests/monitoring/deploy.yaml b/templates/monitoring/deploy.yaml similarity index 100% rename from tests/monitoring/deploy.yaml rename to templates/monitoring/deploy.yaml diff --git a/tests/monitoring/params.yaml b/templates/monitoring/params.yaml similarity index 100% rename from tests/monitoring/params.yaml rename to templates/monitoring/params.yaml