From 9aca43a4c5f141dc3f4365512ff922317951a319 Mon Sep 17 00:00:00 2001 From: Joshua McCurry Date: Thu, 3 Sep 2026 00:31:37 -0400 Subject: [PATCH 1/4] updated handling for conventional data obs builder scripts --- src/swell/tasks/bufr_to_ioda.py | 129 +++++++++++++++++++++++++++++--- 1 file changed, 120 insertions(+), 9 deletions(-) diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 39294f718..b72d747b5 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -52,11 +52,6 @@ 'ncep_gpsro_bufr': 'gnssro.py', 'gpsro': 'gnssro.py', - # prepbufr - 'ncep_acftpfl_bufr': 'prepbufr_aircraft.py', - 'acftpfl': 'prepbufr_aircraft.py', - 'acft_profiles': 'prepbufr_aircraft.py', - # Rest of obs_classes from GetBufr # 'gmao_amsr2_bufr': 'spoc_radiance_amsr2.py', # 'gmao_gmi_bufr': 'spoc_radiance_gmi.py', @@ -105,20 +100,51 @@ def find_obstype_match(self, bufr_path_file: Path) -> str: # -------------------------------------------------------------------------------------------------- + def get_conventional_obs_builders(self, bufr_path_file: Path): + """ + Detects the "conventional" obs bufr sources (aircraft, prepbufr) that + don't map one-to-one with a single ObsBuilder script/output the way + every other obs type in obs_builder_dict does, and returns the list + of builder scripts that need to be run for it. + """ + + filename = bufr_path_file.name + + # Aircraft profiles: a single bufr source that maps to a single + # builder script. + if 'acft_profiles' in filename or 'acftpfl' in filename: + return [('acft_profile',('aircraft_wind',), 'prepbufr_aircraft_wind.py'), + ('acft_profile',('aircraft_temperature',),'prepbufr_aircraft_temperature.py'), + ] + + + # Conventional prepbufr (everything except aircraft): a single bufr + # source that has to be run through two separate builder scripts to + # produce all of its ioda output types. + if 'prepbufr' in filename and 'acft' not in filename: + return [ + ('prepbufr',('sonde','pibal'), 'prepbufr_adpupa.py'), # produces sonde and pibal ioda files + ('prepbufr',('sfc','sfcship'), 'prepbufr_sfc.py'), # produces sfc and sfcship ioda files + ] + + return None + + # -------------------------------------------------------------------------------------------------- + def get_obs_builder_file(self, spoc_script_path: Path, - obs_type: str) -> Path: + obs_builder_file: str) -> Path: """ Returns the path to the ObsBuilder python file Parameters: spoc_script_path: Path to the spoc scripts + obs_builder_file: Filename of the ObsBuilder script (e.g. "radiance_atms.py") Returns: Path to the specific ObsBuilder python file """ - obs_builder_file = obs_builder_dict[obs_type] obs_builder_glob = list(spoc_script_path.glob(obs_builder_file)) if len(obs_builder_glob) > 0: return obs_builder_glob[0] @@ -128,6 +154,80 @@ def get_obs_builder_file(self, return None + # -------------------------------------------------------------------------------------------------- + + def process_bufr_file_conventional(self, + bufr_path_file: Path, + obs_type: str, + obs_spaces: tuple, + obs_builder_filename: str, + spoc_script_path: Path, + ioda_dir: Path) -> None: + """ + Runs a single ObsBuilder script against a bufr file, writing the + resulting ioda file(s) into ioda_dir/obs_type. This is shared by both + the generic (one obs_type -> one script) path and the conventional + (one bufr source -> multiple scripts) path. + + Parameters: + bufr_path_file: Path to input bufr file + obs_type: obs type name used to name the output directory + obs_builder_filename: Filename of the ObsBuilder script to run + spoc_script_path: Path to the spoc scripts + ioda_dir: Path to the top-level ioda output directory + """ + + obs_builder_file = self.get_obs_builder_file(spoc_script_path, obs_builder_filename) + + if obs_builder_file is None: + self.logger.info(f'SKIPPING: ObsBuilder file `{obs_builder_filename}` ' + f'not found for {bufr_path_file}') + return + + self.logger.info(f' MATCH FOUND: [ {obs_builder_file} ]') + + # Get the name of the output directory + obs_type_dir = ioda_dir / obs_type + obs_type_dir.mkdir(mode=0o755, exist_ok=True) + + self.logger.info(f'obs_type_dir: {obs_type_dir}') + + if bufr_path_file.suffix == '.bufr_d': + bufr_file_parts = bufr_path_file.name.rsplit('.', 2) + base_name = bufr_file_parts[0] + else: + bufr_file_parts = [bufr_path_file.name] + base_name = bufr_path_file.name + + # Output IODA filepath + if 'aircraft' in obs_builder_filename: + if 'wind' in obs_builder_filename: + ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{}'.format(obs_spaces[0]) +'.tm00.nc4') + elif 'temp' in obs_builder_filename: + ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{}'.format(obs_spaces[0]) +'.tm00.nc4') + else: + ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{splits/obsType}.tm00.nc4') + + existing_files = list(obs_type_dir.glob(f'{base_name}*')) + if len(existing_files) > 0: + self.logger.info(f'SKIPPING: Output files already exist for {bufr_path_file}: ' + f'{existing_files}') + return + + subprocess.run(['python', obs_builder_file, '--input', bufr_path_file, + '--output', ioda_file_target],cwd=spoc_script_path, check=True) + + for obs_space in obs_spaces: + obs_space_dir=ioda_dir / obs_space + obs_space_dir.mkdir(mode=0o755,exist_ok=True) + output_file=list(obs_type_dir.glob(f"*{base_name}*{obs_space}.tm00.nc4"))[0] + output_file.rename(obs_space_dir / output_file.name) + + obs_type_dir.rmdir() + + + + # -------------------------------------------------------------------------------------------------- def execute(self): @@ -156,12 +256,23 @@ def execute(self): ioda_dir.mkdir(mode=0o755, parents=True, exist_ok=True) + spoc_script_path = Path(self.experiment_path()) / 'spoc' / 'dump' / 'scripts' / 'atmosphere' + # Get the list of bufr files to convert bufr_path_files = list(bufr_dir.glob('*bufr*')) for bufr_path_file in bufr_path_files: + + conventional_builders = self.get_conventional_obs_builders(bufr_path_file) + + if conventional_builders is not None: + for obs_type,obs_spaces,obs_builder_filename in conventional_builders: + self.process_bufr_file_conventional(bufr_path_file, obs_type, obs_spaces, obs_builder_filename, + spoc_script_path, ioda_dir) + continue + obs_type = self.find_obstype_match(bufr_path_file) - obs_builder_file = self.get_obs_builder_file(spoc_script_path, obs_type) + obs_builder_file = self.get_obs_builder_file(spoc_script_path, obs_builder_dict[obs_type]) if obs_builder_file is None: self.logger.info(f'SKIPPING: No valid observation type ' @@ -192,6 +303,6 @@ def execute(self): continue subprocess.run(['python', obs_builder_file, '--input', bufr_path_file, - '--output', ioda_file_target], check=True) + '--output', ioda_file_target],cwd=spoc_script_path, check=True) # -------------------------------------------------------------------------------------------------- From 73b56ab69b81f974c6ad98a97c0c921dc7b44aa2 Mon Sep 17 00:00:00 2001 From: Joshua McCurry Date: Thu, 3 Sep 2026 14:54:35 -0400 Subject: [PATCH 2/4] updated comments and corrected typos --- src/swell/tasks/bufr_to_ioda.py | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index b72d747b5..547986578 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -102,10 +102,8 @@ def find_obstype_match(self, bufr_path_file: Path) -> str: def get_conventional_obs_builders(self, bufr_path_file: Path): """ - Detects the "conventional" obs bufr sources (aircraft, prepbufr) that - don't map one-to-one with a single ObsBuilder script/output the way - every other obs type in obs_builder_dict does, and returns the list - of builder scripts that need to be run for it. + For conventional data in prepbufr / prepbufr profile input files - returns input file type, + obs spaces produced, and name of obs builder conversion script """ filename = bufr_path_file.name @@ -113,8 +111,8 @@ def get_conventional_obs_builders(self, bufr_path_file: Path): # Aircraft profiles: a single bufr source that maps to a single # builder script. if 'acft_profiles' in filename or 'acftpfl' in filename: - return [('acft_profile',('aircraft_wind',), 'prepbufr_aircraft_wind.py'), - ('acft_profile',('aircraft_temperature',),'prepbufr_aircraft_temperature.py'), + return [('acft_profiles',('aircraft_wind',), 'prepbufr_aircraft_wind.py'), + ('acft_profiles',('aircraft_temperature',),'prepbufr_aircraft_temperature.py'), ] @@ -164,17 +162,9 @@ def process_bufr_file_conventional(self, spoc_script_path: Path, ioda_dir: Path) -> None: """ - Runs a single ObsBuilder script against a bufr file, writing the - resulting ioda file(s) into ioda_dir/obs_type. This is shared by both - the generic (one obs_type -> one script) path and the conventional - (one bufr source -> multiple scripts) path. - - Parameters: - bufr_path_file: Path to input bufr file - obs_type: obs type name used to name the output directory - obs_builder_filename: Filename of the ObsBuilder script to run - spoc_script_path: Path to the spoc scripts - ioda_dir: Path to the top-level ioda output directory + Handles subprocess call for conventional obs spaces - to handle scripts producing + multiple obs spaces, conv ioda files are sent to temp directories based on obs_type + and then sent to respective obs space directories after conversion """ obs_builder_file = self.get_obs_builder_file(spoc_script_path, obs_builder_filename) @@ -201,10 +191,7 @@ def process_bufr_file_conventional(self, # Output IODA filepath if 'aircraft' in obs_builder_filename: - if 'wind' in obs_builder_filename: - ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{}'.format(obs_spaces[0]) +'.tm00.nc4') - elif 'temp' in obs_builder_filename: - ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{}'.format(obs_spaces[0]) +'.tm00.nc4') + ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{}'.format(obs_spaces[0]) +'.tm00.nc4') else: ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{splits/obsType}.tm00.nc4') @@ -217,8 +204,10 @@ def process_bufr_file_conventional(self, subprocess.run(['python', obs_builder_file, '--input', bufr_path_file, '--output', ioda_file_target],cwd=spoc_script_path, check=True) + # Remove temporary obs_type directories and move ioda files to new directories for each obs space for obs_space in obs_spaces: obs_space_dir=ioda_dir / obs_space + self.logger.info(f'obs_space_dir: {obs_space_dir}') obs_space_dir.mkdir(mode=0o755,exist_ok=True) output_file=list(obs_type_dir.glob(f"*{base_name}*{obs_space}.tm00.nc4"))[0] output_file.rename(obs_space_dir / output_file.name) From 608f1ce57b6097f11256a7c4086703f874b5c499 Mon Sep 17 00:00:00 2001 From: Joshua McCurry Date: Thu, 3 Sep 2026 15:46:56 -0400 Subject: [PATCH 3/4] removed definition of spoc_scripts_path from old version of code --- src/swell/tasks/bufr_to_ioda.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 547986578..6524c27f8 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -245,8 +245,6 @@ def execute(self): ioda_dir.mkdir(mode=0o755, parents=True, exist_ok=True) - spoc_script_path = Path(self.experiment_path()) / 'spoc' / 'dump' / 'scripts' / 'atmosphere' - # Get the list of bufr files to convert bufr_path_files = list(bufr_dir.glob('*bufr*')) From ba2519343be254dcd2cd89633e99fc223b7e7fd9 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 11 Sep 2026 15:58:10 -0400 Subject: [PATCH 4/4] coding norms --- src/swell/tasks/bufr_to_ioda.py | 57 +++++++++++++++++---------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 6524c27f8..eacc28aff 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -111,18 +111,19 @@ def get_conventional_obs_builders(self, bufr_path_file: Path): # Aircraft profiles: a single bufr source that maps to a single # builder script. if 'acft_profiles' in filename or 'acftpfl' in filename: - return [('acft_profiles',('aircraft_wind',), 'prepbufr_aircraft_wind.py'), - ('acft_profiles',('aircraft_temperature',),'prepbufr_aircraft_temperature.py'), - ] - + return [('acft_profiles', ('aircraft_wind',), 'prepbufr_aircraft_wind.py'), + ('acft_profiles', ('aircraft_temperature',), + 'prepbufr_aircraft_temperature.py'),] # Conventional prepbufr (everything except aircraft): a single bufr # source that has to be run through two separate builder scripts to # produce all of its ioda output types. if 'prepbufr' in filename and 'acft' not in filename: return [ - ('prepbufr',('sonde','pibal'), 'prepbufr_adpupa.py'), # produces sonde and pibal ioda files - ('prepbufr',('sfc','sfcship'), 'prepbufr_sfc.py'), # produces sfc and sfcship ioda files + # produces sonde and pibal ioda files + ('prepbufr', ('sonde', 'pibal'), 'prepbufr_adpupa.py'), + # produces sfc and sfcship ioda files + ('prepbufr', ('sfc', 'sfcship'), 'prepbufr_sfc.py'), ] return None @@ -155,12 +156,12 @@ def get_obs_builder_file(self, # -------------------------------------------------------------------------------------------------- def process_bufr_file_conventional(self, - bufr_path_file: Path, - obs_type: str, - obs_spaces: tuple, - obs_builder_filename: str, - spoc_script_path: Path, - ioda_dir: Path) -> None: + bufr_path_file: Path, + obs_type: str, + obs_spaces: tuple, + obs_builder_filename: str, + spoc_script_path: Path, + ioda_dir: Path) -> None: """ Handles subprocess call for conventional obs spaces - to handle scripts producing multiple obs spaces, conv ioda files are sent to temp directories based on obs_type @@ -191,9 +192,11 @@ def process_bufr_file_conventional(self, # Output IODA filepath if 'aircraft' in obs_builder_filename: - ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{}'.format(obs_spaces[0]) +'.tm00.nc4') + ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{}'.format( + obs_spaces[0]) + '.tm00.nc4') else: - ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{splits/obsType}.tm00.nc4') + ioda_file_target = obs_type_dir / (bufr_file_parts[0] + + '.{splits/obsType}.tm00.nc4') existing_files = list(obs_type_dir.glob(f'{base_name}*')) if len(existing_files) > 0: @@ -202,21 +205,19 @@ def process_bufr_file_conventional(self, return subprocess.run(['python', obs_builder_file, '--input', bufr_path_file, - '--output', ioda_file_target],cwd=spoc_script_path, check=True) + '--output', ioda_file_target], cwd=spoc_script_path, check=True) - # Remove temporary obs_type directories and move ioda files to new directories for each obs space + # Remove temporary obs_type directories and move ioda files to new directories + # for each obs space for obs_space in obs_spaces: - obs_space_dir=ioda_dir / obs_space + obs_space_dir = ioda_dir / obs_space self.logger.info(f'obs_space_dir: {obs_space_dir}') - obs_space_dir.mkdir(mode=0o755,exist_ok=True) - output_file=list(obs_type_dir.glob(f"*{base_name}*{obs_space}.tm00.nc4"))[0] + obs_space_dir.mkdir(mode=0o755, exist_ok=True) + output_file = list(obs_type_dir.glob(f"*{base_name}*{obs_space}.tm00.nc4"))[0] output_file.rename(obs_space_dir / output_file.name) obs_type_dir.rmdir() - - - # -------------------------------------------------------------------------------------------------- def execute(self): @@ -253,13 +254,15 @@ def execute(self): conventional_builders = self.get_conventional_obs_builders(bufr_path_file) if conventional_builders is not None: - for obs_type,obs_spaces,obs_builder_filename in conventional_builders: - self.process_bufr_file_conventional(bufr_path_file, obs_type, obs_spaces, obs_builder_filename, - spoc_script_path, ioda_dir) + for obs_type, obs_spaces, obs_builder_filename in conventional_builders: + self.process_bufr_file_conventional(bufr_path_file, obs_type, obs_spaces, + obs_builder_filename, spoc_script_path, + ioda_dir) continue obs_type = self.find_obstype_match(bufr_path_file) - obs_builder_file = self.get_obs_builder_file(spoc_script_path, obs_builder_dict[obs_type]) + obs_builder_file = self.get_obs_builder_file(spoc_script_path, + obs_builder_dict[obs_type]) if obs_builder_file is None: self.logger.info(f'SKIPPING: No valid observation type ' @@ -290,6 +293,6 @@ def execute(self): continue subprocess.run(['python', obs_builder_file, '--input', bufr_path_file, - '--output', ioda_file_target],cwd=spoc_script_path, check=True) + '--output', ioda_file_target], cwd=spoc_script_path, check=True) # --------------------------------------------------------------------------------------------------