Skip to content
36 changes: 36 additions & 0 deletions recipes/deploy/deploy_cps_test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
concurrency_groups:
copy_raw_serial:
limit: 1

flows:
copy_raw:
deployment_name: copy-raw-cps-test
interval: 3
inject_time_offset: true
concurrency_group: copy_raw_serial

raw2Sv:
deployment_name: raw2Sv-cps-test
triggers:
- expect: "echodataflow.raw.updated"
resource_name: "raw-monitor"
resource_scope: primary

simulate_transects:
deployment_name: simulate-transects-cps-test
interval: 3

process_CPS:
deployment_name: process-CPS-cps-test
triggers:
- expect: "echodataflow.transect.updated"
resource_name: "transect-monitor"
resource_scope: related

- expect: "echodataflow.sv.updated"
resource_name: "sv-monitor"
resource_scope: primary

update_cache_CPS:
deployment_name: update-cache-CPS-cps-test
interval: 1
63 changes: 63 additions & 0 deletions recipes/params/params_cps_test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
flows:
copy_raw:
path_raw_list: "/path/to/SH2407_raw_list_organized.csv"
path_copy: "/path/to/cps_workflow/raw"
s3_bucket: noaa-wcsd-pds
exclude_before: "2024-08-05T13:10:00+00:00"
exclude_after: "2024-08-06T12:30:00+00:00"
endpoint_url: "https://s3.amazonaws.com"

simulate_transects:
path_transect_csv: "/path/to/cps_workflow/plotSurvey_Survey_Data_Visualizer.csv"
survey_start: "2024-08-05T13:16:00+00:00"
transect_duration_minutes: 20
start_transect_num: 1
max_transects: 20

raw2Sv:
exclude_before: "2024-08-05T13:10:00+00:00"
exclude_raw_file: []
parallel: false
new_file_num_limit: 10

encode_mode: power
waveform_mode: CW
depth_offset: 7
sonar_model: EK80
datagram_type:
nmea_sentence: GGA

add_depth: true
add_location: true
add_splitbeam_angle: true

path_main: "/path/to/cps_workflow"
processing_db: "postgresql+psycopg://USER:PASSWORD@HOST:5432/DATABASE"

process_CPS:
path_transect_csv: "/path/to/cps_workflow/plotSurvey_Survey_Data_Visualizer.csv"
path_snapshot_csv: "/path/to/cps_workflow/plotSurvey_Survey_Data_Visualizer_snapshot.csv"
path_main: "/path/to/cps_workflow"
processing_db: "postgresql+psycopg://USER:PASSWORD@HOST:5432/DATABASE"

target_frequency: 70000
min_depth: 10

seafloor_threshold: [-40, 2.4, 1.0]
seafloor_offset: 0.5
seafloor_r0: 10
seafloor_r1: 1000
seafloor_wtheta: 28
seafloor_wphi: 52

mask_mode: cps
fallback_sv_threshold: -70

range_bin: 10m
dist_bin: 0.5nmi

update_cache_CPS:
path_CPS: "/path/to/cps_workflow/CPS_Masks_Zarr"
path_cache: "/path/to/cps_workflow/viz_cache_CPS"
path_transect_csv: "/path/to/cps_workflow/plotSurvey_Survey_Data_Visualizer.csv"
file_CPS_zarr: latest_CPS.zarr
17 changes: 14 additions & 3 deletions scripts/watch_raw_updates.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,30 +2,41 @@
import time
from pathlib import Path

from echodataflow.utils.raw_monitor import watch_raw_directory
from echodataflow.operations.operations_watchdog import watch_raw_directory


def main():
parser = argparse.ArgumentParser(
description="Watch a RAW directory and emit Prefect update events."
description="Watch a RAW directory and register RAW updates."
)
parser.add_argument(
"path",
type=Path,
help="Path to the RAW directory to watch.",
)
parser.add_argument(
"--db-path",
type=str,
required=True,
help="Processing database path or SQLAlchemy database URL.",
)
args = parser.parse_args()

target_dir = args.path.resolve()
db_path = args.db_path

if not target_dir.exists():
raise FileNotFoundError(
f"RAW directory does not exist: {target_dir}"
)

observer = watch_raw_directory(target_dir)
observer = watch_raw_directory(
target_dir,
db_path=db_path,
)

print(f"Watching {target_dir}")
print(f"Processing ledger: {db_path}")

try:
while True:
Expand Down
File renamed without changes.
File renamed without changes.
Loading