diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 39294f718..eacc28aff 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -52,11 +52,6 @@ 'ncep_gpsro_bufr': 'gnssro.py', 'gpsro': 'gnssro.py', - # prepbufr - 'ncep_acftpfl_bufr': 'prepbufr_aircraft.py', - 'acftpfl': 'prepbufr_aircraft.py', - 'acft_profiles': 'prepbufr_aircraft.py', - # Rest of obs_classes from GetBufr # 'gmao_amsr2_bufr': 'spoc_radiance_amsr2.py', # 'gmao_gmi_bufr': 'spoc_radiance_gmi.py', @@ -105,20 +100,50 @@ def find_obstype_match(self, bufr_path_file: Path) -> str: # -------------------------------------------------------------------------------------------------- + def get_conventional_obs_builders(self, bufr_path_file: Path): + """ + For conventional data in prepbufr / prepbufr profile input files - returns input file type, + obs spaces produced, and name of obs builder conversion script + """ + + filename = bufr_path_file.name + + # Aircraft profiles: a single bufr source that maps to a single + # builder script. + if 'acft_profiles' in filename or 'acftpfl' in filename: + return [('acft_profiles', ('aircraft_wind',), 'prepbufr_aircraft_wind.py'), + ('acft_profiles', ('aircraft_temperature',), + 'prepbufr_aircraft_temperature.py'),] + + # Conventional prepbufr (everything except aircraft): a single bufr + # source that has to be run through two separate builder scripts to + # produce all of its ioda output types. + if 'prepbufr' in filename and 'acft' not in filename: + return [ + # produces sonde and pibal ioda files + ('prepbufr', ('sonde', 'pibal'), 'prepbufr_adpupa.py'), + # produces sfc and sfcship ioda files + ('prepbufr', ('sfc', 'sfcship'), 'prepbufr_sfc.py'), + ] + + return None + + # -------------------------------------------------------------------------------------------------- + def get_obs_builder_file(self, spoc_script_path: Path, - obs_type: str) -> Path: + obs_builder_file: str) -> Path: """ Returns the path to the ObsBuilder python file Parameters: spoc_script_path: Path to the spoc scripts + obs_builder_file: Filename of the ObsBuilder script (e.g. "radiance_atms.py") Returns: Path to the specific ObsBuilder python file """ - obs_builder_file = obs_builder_dict[obs_type] obs_builder_glob = list(spoc_script_path.glob(obs_builder_file)) if len(obs_builder_glob) > 0: return obs_builder_glob[0] @@ -130,6 +155,71 @@ def get_obs_builder_file(self, # -------------------------------------------------------------------------------------------------- + def process_bufr_file_conventional(self, + bufr_path_file: Path, + obs_type: str, + obs_spaces: tuple, + obs_builder_filename: str, + spoc_script_path: Path, + ioda_dir: Path) -> None: + """ + Handles subprocess call for conventional obs spaces - to handle scripts producing + multiple obs spaces, conv ioda files are sent to temp directories based on obs_type + and then sent to respective obs space directories after conversion + """ + + obs_builder_file = self.get_obs_builder_file(spoc_script_path, obs_builder_filename) + + if obs_builder_file is None: + self.logger.info(f'SKIPPING: ObsBuilder file `{obs_builder_filename}` ' + f'not found for {bufr_path_file}') + return + + self.logger.info(f' MATCH FOUND: [ {obs_builder_file} ]') + + # Get the name of the output directory + obs_type_dir = ioda_dir / obs_type + obs_type_dir.mkdir(mode=0o755, exist_ok=True) + + self.logger.info(f'obs_type_dir: {obs_type_dir}') + + if bufr_path_file.suffix == '.bufr_d': + bufr_file_parts = bufr_path_file.name.rsplit('.', 2) + base_name = bufr_file_parts[0] + else: + bufr_file_parts = [bufr_path_file.name] + base_name = bufr_path_file.name + + # Output IODA filepath + if 'aircraft' in obs_builder_filename: + ioda_file_target = obs_type_dir / (bufr_file_parts[0] + '.{}'.format( + obs_spaces[0]) + '.tm00.nc4') + else: + ioda_file_target = obs_type_dir / (bufr_file_parts[0] + + '.{splits/obsType}.tm00.nc4') + + existing_files = list(obs_type_dir.glob(f'{base_name}*')) + if len(existing_files) > 0: + self.logger.info(f'SKIPPING: Output files already exist for {bufr_path_file}: ' + f'{existing_files}') + return + + subprocess.run(['python', obs_builder_file, '--input', bufr_path_file, + '--output', ioda_file_target], cwd=spoc_script_path, check=True) + + # Remove temporary obs_type directories and move ioda files to new directories + # for each obs space + for obs_space in obs_spaces: + obs_space_dir = ioda_dir / obs_space + self.logger.info(f'obs_space_dir: {obs_space_dir}') + obs_space_dir.mkdir(mode=0o755, exist_ok=True) + output_file = list(obs_type_dir.glob(f"*{base_name}*{obs_space}.tm00.nc4"))[0] + output_file.rename(obs_space_dir / output_file.name) + + obs_type_dir.rmdir() + + # -------------------------------------------------------------------------------------------------- + def execute(self): """ Converts collected bufr files to ioda using ObsBuilder python files @@ -160,8 +250,19 @@ def execute(self): bufr_path_files = list(bufr_dir.glob('*bufr*')) for bufr_path_file in bufr_path_files: + + conventional_builders = self.get_conventional_obs_builders(bufr_path_file) + + if conventional_builders is not None: + for obs_type, obs_spaces, obs_builder_filename in conventional_builders: + self.process_bufr_file_conventional(bufr_path_file, obs_type, obs_spaces, + obs_builder_filename, spoc_script_path, + ioda_dir) + continue + obs_type = self.find_obstype_match(bufr_path_file) - obs_builder_file = self.get_obs_builder_file(spoc_script_path, obs_type) + obs_builder_file = self.get_obs_builder_file(spoc_script_path, + obs_builder_dict[obs_type]) if obs_builder_file is None: self.logger.info(f'SKIPPING: No valid observation type ' @@ -192,6 +293,6 @@ def execute(self): continue subprocess.run(['python', obs_builder_file, '--input', bufr_path_file, - '--output', ioda_file_target], check=True) + '--output', ioda_file_target], cwd=spoc_script_path, check=True) # --------------------------------------------------------------------------------------------------