diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml index 763690911..9a1473bdb 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/geos_atmosphere.yaml @@ -13,4 +13,5 @@ executables: obsfilters: test_ObsFilters.x eda3D: fv3jedi_var.x eda4D: fv3jedi_var.x + edaControlPert: fv3jedi_controlpert.x diffstates: fv3jedi_diffstates.x diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_eda.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_eda.py index 71dd9d893..a28df4192 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_eda.py +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_eda.py @@ -24,7 +24,7 @@ 'rain_water', 'snow_water', 'mole_fraction_of_ozone_in_air', - 'geopotential_height_times_gravity_at_surface', + 'geopotential_at_surface', 'initial_mass_fraction_of_large_scale_cloud_condensate', 'initial_mass_fraction_of_convective_cloud_condensate', 'convective_cloud_area_fraction', diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_eda_control_pert.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_eda_control_pert.py new file mode 100644 index 000000000..6a4e00d6c --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_eda_control_pert.py @@ -0,0 +1,83 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +from collections.abc import Mapping +from swell.configuration.jedi.interfaces.geos_atmosphere.model.shared import \ + field_io_names + +# -------------------------------------------------------------------------------------------------- + +state_variables = [ + 'eastward_wind', + 'northward_wind', + 'air_temperature', + 'air_pressure_at_surface', + 'air_pressure_levels', + 'water_vapor_mixing_ratio_wrt_moist_air', + 'cloud_liquid_ice', + 'cloud_liquid_water', + 'rain_water', + 'snow_water', + 'mole_fraction_of_ozone_in_air', + 'geopotential_at_surface', + 'initial_mass_fraction_of_large_scale_cloud_condensate', + 'initial_mass_fraction_of_convective_cloud_condensate', + 'convective_cloud_area_fraction', + 'fraction_of_ocean', + 'fraction_of_land', + 'isotropic_variance_of_filtered_topography', + 'surface_velocity_scale', + 'surface_buoyancy_scale', + 'planetary_boundary_layer_height', + 'surface_exchange_coefficient_for_momentum', + 'surface_exchange_coefficient_for_heat', + 'surface_exchange_coefficient_for_moisture', + 'KCBL_before_moist', + 'surface_temp_before_moist', + 'lower_index_where_Kh_greater_than_2', + 'upper_index_where_Kh_greater_than_2', + 'fraction_of_lake', + 'fraction_of_ice', + 'vtype', + 'stype', + 'vfrac', + 'sheleg', + 'skin_temperature_at_surface', + 'soilt', + 'soilm', + 'eastward_wind_at_surface', + 'northward_wind_at_surface', + # 'sea_surface_temperature', + # 'mole_fraction_of_carbon_dioxide_in_air', +] + +# -------------------------------------------------------------------------------------------------- + + +def background_eda_control_pert(template_dict: Mapping) -> Mapping: + horizontal_resolution = template_dict['horizontal_resolution'] + ichunk = template_dict['ensemble_ichunk'] + cycle_dir = template_dict['cycle_dir'] + background = { + 'datetime': template_dict['local_background_time_iso'], + 'filetype': 'cube sphere history', + 'provider': 'geos', + 'compute edge pressure from surface pressure': True, + 'max allowable geometry difference': 1e-3, + 'datapath': f'{cycle_dir}/ebkg_chunk/chunk{ichunk:03d}/geos.mem%mem_pad%/', + 'filenames': [ + f'%yyyy%mm%dd_%hh%MM%ssz.nc4', + f'../../../fv3-jedi/bkg/geos.crtmsrf.{horizontal_resolution}.nc4' + ], + 'state variables': state_variables, + 'field io names': field_io_names, + } + + return background + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error_eda_gsiB.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error_eda_gsiB.py index 6350dbedd..b3eb657ec 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error_eda_gsiB.py +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error_eda_gsiB.py @@ -25,7 +25,7 @@ 'fraction_of_ocean', 'fraction_of_lake', 'fraction_of_ice', - 'geopotential_height_times_gravity_at_surface', + 'geopotential_at_surface', 'skin_temperature_at_surface', ] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error_eda_hybridB.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error_eda_hybridB.py index e1c2acc68..780ecbb6e 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error_eda_hybridB.py +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/background_error_eda_hybridB.py @@ -28,7 +28,7 @@ 'fraction_of_ocean', 'fraction_of_lake', 'fraction_of_ice', - 'geopotential_height_times_gravity_at_surface', + 'geopotential_at_surface', 'skin_temperature_at_surface' ] diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/eda_analysis_control_pert.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/eda_analysis_control_pert.py new file mode 100644 index 000000000..c37718f20 --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/eda_analysis_control_pert.py @@ -0,0 +1,30 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +from collections.abc import Mapping +from swell.configuration.jedi.interfaces.geos_atmosphere.model.shared import field_io_names + +# -------------------------------------------------------------------------------------------------- + + +def eda_analysis_control_pert(template_dict: Mapping) -> Mapping: + + ichunk = template_dict.get('ensemble_ichunk', None) + analysis = { + 'filetype': 'cube sphere history', + 'provider': 'geos', + 'datapath': f'./analysis_chunk/chunk{ichunk:03d}/mem%mem_pad%', + 'filename': 'eda.ana.mem%mem_pad%.%yyyy%mm%dd_%hh%MM%ssz.nc4', + 'first': 'PT0H', + 'frequency': 'PT1H', + 'field io names': field_io_names, + } + + return analysis + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/eda_varincrement1.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/eda_varincrement1.py index d6f02a837..e76823f9f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/eda_varincrement1.py +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/eda_varincrement1.py @@ -34,7 +34,7 @@ def eda_varincrement1(template_dict: Mapping) -> Mapping: 'rain_water': 'qr', 'snow_water': 'qs', 'mole_fraction_of_ozone_in_air': 'o3ppmv', - 'geopotential_height_times_gravity_at_surface': 'phis', + 'geopotential_at_surface': 'phis', 'skin_temperature_at_surface': 'ts', 'air_pressure_at_surface': 'ps', } diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/ensemble_block.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/ensemble_block.py index b332cb065..fd0447a70 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/ensemble_block.py +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/ensemble_block.py @@ -62,7 +62,7 @@ def ensemble_block(template_dict: Mapping) -> Mapping: 'rain_water': 'qr', 'snow_water': 'qs', 'mole_fraction_of_ozone_in_air': 'o3ppmv', - 'geopotential_height_at_surface': 'phis', + 'geopotential_at_surface': 'phis', 'fraction_of_ocean': 'frocean', 'fraction_of_lake': 'frlake', 'fraction_of_ice': 'frseaice', diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/shared.py b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/shared.py index 6989e2f51..6c4b56a09 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/shared.py +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/model/shared.py @@ -104,7 +104,7 @@ 'rain_water': 'qr', 'snow_water': 'qs', 'mole_fraction_of_ozone_in_air': 'o3ppmv', - 'geopotential_height_times_gravity_at_surface': 'phis', + 'geopotential_at_surface': 'phis', 'fraction_of_ocean': 'frocean', 'fraction_of_lake': 'frlake', 'fraction_of_ice': 'frseaice', @@ -129,7 +129,7 @@ 'rain_water', 'snow_water', 'mole_fraction_of_ozone_in_air', - 'geopotential_height_times_gravity_at_surface', + 'geopotential_at_surface', 'fraction_of_ocean', 'fraction_of_lake', 'fraction_of_ice', diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml index 9521c6673..bbb189859 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml @@ -129,6 +129,10 @@ ensemble_num_members: default_value: 16 options: None +ensemble_num_chunks: + default_value: 8 + options: None + geovals_experiment: default_value: x0050-geovals options: @@ -199,6 +203,7 @@ minimizer: default_value: DRPCG options: - DRPCG + - DRPLanczos ncdiag_experiments: default_value: x0050_fgat diff --git a/src/swell/configuration/jedi/oops/eda_control_pert.py b/src/swell/configuration/jedi/oops/eda_control_pert.py new file mode 100644 index 000000000..5834aaa0d --- /dev/null +++ b/src/swell/configuration/jedi/oops/eda_control_pert.py @@ -0,0 +1,81 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +from swell.utilities.oops_config import OopsConfig + +# -------------------------------------------------------------------------------------------------- + + +class eda_control_pert(OopsConfig): + + def render_oops(self): + nmember = self.template_dict['ensemble_num_members'] + nchunk = self.template_dict['ensemble_num_chunks'] + ichunk = self.template_dict['ensemble_ichunk'] + nstate = int(nmember/nchunk) + if ichunk == 1: + num_pert_mem = nstate - 1 + else: + num_pert_mem = nstate + pert_start_index = 1 + + oops = { + 'assimilation': { + 'cost function': { + 'cost type': '3D-Var', + 'jb evaluation': False, + 'time window': { + 'begin': self.template_dict['window_begin_iso'], + 'length': self.template_dict['window_length'], + 'bound to include': 'begin' + }, + 'geometry': self.interface_model('geometry'), + 'analysis variables': self.template_dict['analysis_variables'], + 'background': self.interface_model('background_eda_control_pert'), + 'background error': self.interface_model('background_error_eda_gsiB'), + 'observations': { + 'get values': self.interface_model('getvalues'), + 'observers': self.special_observations(), + } + }, + 'variational': { + 'minimizer': { + 'algorithm': self.template_dict['minimizer'] + }, + 'iterations': [{ + 'geometry': self.interface_model('geometry_inner'), + 'gradient norm reduction': float( + self.template_dict['gradient_norm_reduction']), + 'ninner': self.template_dict['number_of_iterations'], + }], + }, + 'final': { + 'diagnostics': { + 'departures': 'oman' + }, + 'prints': { + 'frequency': 'PT3H' + } + }, + 'output': self.interface_model('eda_analysis_control_pert') + }, + 'template': { + 'pattern with zero padding': "%mem_pad%", + 'pattern without zero padding': "%mem_wo_pad%", + 'number of pert members': num_pert_mem, + 'first pert member index': pert_start_index, + 'run pert members only': False + } + } + + # TODO: Implement this more cleanly in the OOPS schema + if self.jedi_interface == 'geos_cf': + oops['final']['increment'] = {'geometry': self.interface_model('geometry'), + 'output': self.interface_model('increment_cs')} + + return oops diff --git a/src/swell/deployment/create_experiment.py b/src/swell/deployment/create_experiment.py index 101849248..d042cfc9e 100644 --- a/src/swell/deployment/create_experiment.py +++ b/src/swell/deployment/create_experiment.py @@ -580,6 +580,8 @@ def prepare_cylc_suite_jinja2( render_dictionary['scheduling']['RunGeos']['execution_time_limit'] = 'PT30M' render_dictionary['scheduling']['RunJediLocalEnsembleDaExecutable'][ 'execution_time_limit'] = 'PT1H' + render_dictionary['scheduling']['RunJediEdaControlPertExecutable'][ + 'execution_time_limit'] = 'PT30M' render_dictionary['scheduling']['EvaObservations'][ 'execution_time_limit'] = 'PT30M' diff --git a/src/swell/suites/eda_atmos/suite_config.py b/src/swell/suites/eda_atmos/suite_config.py index a632d8b86..f95b836df 100644 --- a/src/swell/suites/eda_atmos/suite_config.py +++ b/src/swell/suites/eda_atmos/suite_config.py @@ -64,7 +64,7 @@ class SuiteConfig(QuestionContainer, Enum): "rain_water", "snow_water", "mole_fraction_of_ozone_in_air", - "geopotential_height_times_gravity_at_surface", + "geopotential_at_surface", "fraction_of_ocean", "fraction_of_lake", "fraction_of_ice", @@ -143,7 +143,7 @@ class SuiteConfig(QuestionContainer, Enum): "rain_water", "snow_water", "mole_fraction_of_ozone_in_air", - "geopotential_height_times_gravity_at_surface", + "geopotential_at_surface", "fraction_of_ocean", "fraction_of_lake", "fraction_of_ice", diff --git a/src/swell/suites/eda_controlpert_atmos/eva/increment-geos_atmosphere.yaml b/src/swell/suites/eda_controlpert_atmos/eva/increment-geos_atmosphere.yaml new file mode 100644 index 000000000..126435876 --- /dev/null +++ b/src/swell/suites/eda_controlpert_atmos/eva/increment-geos_atmosphere.yaml @@ -0,0 +1,760 @@ +datasets: + +- group: increment + type: LatLon + filename: {{increment_file_path}} + name: experiment_increment + variables: [ps, ts, ua, va, t, q, lat, lon] + +graphics: + + plotting_backend: Emcpy + figure_list: + + #map plot for surface pressure increment + - batch figure: + variables: [ps] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Surface Pressure Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ps + label: PS increment + colorbar: true + cmap: 'bwr' + vmin: -100 + vmax: 100 + - batch figure: + variables: [ts] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Tskin Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ts + label: TS increment + colorbar: true + cmap: 'bwr' + vmin: -2 + vmax: 2 + #map plot for temperature increment (lowest level) + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[71,...]' + label: T increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[62,...]' + label: T increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[49,...]' + label: T increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[42,...]' + label: T increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[24,...]' + label: T increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for temperature increment + - batch figure: + variables: [t] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Temperature Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::t + slices: '[14,...]' + label: T increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment (lowest level) + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[71,...]' + label: U increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[62,...]' + label: U increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[49,...]' + label: U increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[42,...]' + label: U increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[24,...]' + label: U increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Zonal Wind increment + - batch figure: + variables: [ua] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Zonal Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::ua + slices: '[14,...]' + label: U increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment (lowest level) + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[71,...]' + label: V increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[62,...]' + label: V increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[49,...]' + label: V increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[42,...]' + label: V increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[24,...]' + label: V increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Meridional Wind increment + - batch figure: + variables: [va] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Meridional Wind Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::va + slices: '[14,...]' + label: V increment (1 hPa) + colorbar: true + cmap: 'bwr' + vmin: -1 + vmax: 1 + #map plot for Specific Humidity increment (lowest level) + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_1000.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::q + slices: '[71,...]' + label: Q increment (1000 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_850.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::q + slices: '[62,...]' + label: Q increment (850 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_500.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::q + slices: '[49,...]' + label: Q increment (500 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_200.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::q + slices: '[42,...]' + label: Q increment (200 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 + #map plot for Specific Humidity increment + - batch figure: + variables: [q] + figure: + figure size: [20,10] + layout: [1,1] + title: 'Increment from JEDI' + output name: '{{cycle_dir}}/eva/increment/map_plots/${variable}/inc_${variable}_10.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: Specific Humidity Increment + add_grid: + layers: + - type: MapGridded + longitude: + variable: experiment_increment::increment::lon + latitude: + variable: experiment_increment::increment::lat + data: + variable: experiment_increment::increment::q + slices: '[24,...]' + label: Q increment (10 hPa) + colorbar: true + cmap: 'bwr' + vmin: -0.001 + vmax: 0.001 diff --git a/src/swell/suites/eda_controlpert_atmos/eva/jedi_log-geos_atmosphere.yaml b/src/swell/suites/eda_controlpert_atmos/eva/jedi_log-geos_atmosphere.yaml new file mode 100644 index 000000000..2d848d549 --- /dev/null +++ b/src/swell/suites/eda_controlpert_atmos/eva/jedi_log-geos_atmosphere.yaml @@ -0,0 +1,89 @@ +datasets: + +- type: JediLog + collection_name: JediLogTest + jedi_log_to_parse: '{{cycle_dir}}/jedi_variational_log.log' + data_to_parse: + convergence: true + +transforms: +- transform: arithmetic + new name: JediLogTest::convergence::${variable}_log + equals: log(JediLogTest::convergence::${variable}) + for: + variable: [residual_norm, norm_reduction] + +graphics: + + plotting_backend: Emcpy + figure_list: + + - figure: + layout: [3,1] + figure size: [12,10] + title: 'Residual Norm and Norm Reduction Plots' + output name: '{{cycle_dir}}/eva/jedi_log/convergence/residual_norm_reduction.png' + plots: + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Residual norm' + layers: + - type: LinePlot + x: + variable: JediLogTest::convergence::total_iteration + y: + variable: JediLogTest::convergence::residual_norm + color: 'black' + + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Log(norm reduction)' + layers: + - type: LinePlot + x: + variable: JediLogTest::convergence::total_iteration + y: + variable: JediLogTest::convergence::norm_reduction + color: 'black' + + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Log(reduction)' + add_legend: + layers: + - type: LinePlot + x: + variable: JediLogTest::convergence::total_iteration + y: + variable: JediLogTest::convergence::residual_norm_log + color: 'red' + label: 'Log(residual norm)' + - type: LinePlot + x: + variable: JediLogTest::convergence::total_iteration + y: + variable: JediLogTest::convergence::norm_reduction_log + color: 'blue' + label: 'Log norm reduction' + + - figure: + title: 'Cost Function Plot' + output name: '{{cycle_dir}}/eva/jedi_log/cost_function/cost_function.png' + plots: + - add_xlabel: 'Total inner iteration number' + add_ylabel: 'Quadratic Cost Function' + add_legend: + layers: + - type: LinePlot + label: 'jojc' + x: + variable: JediLogTest::convergence::total_iteration + y: + variable: JediLogTest::convergence::jojc + color: 'blue' + markersize: 2 + - type: LinePlot + label: 'jb' + x: + variable: JediLogTest::convergence::total_iteration + y: + variable: JediLogTest::convergence::jb + color: 'red' + markersize: 2 diff --git a/src/swell/suites/eda_controlpert_atmos/eva/observations-geos_atmosphere.yaml b/src/swell/suites/eda_controlpert_atmos/eva/observations-geos_atmosphere.yaml new file mode 100644 index 000000000..edee73160 --- /dev/null +++ b/src/swell/suites/eda_controlpert_atmos/eva/observations-geos_atmosphere.yaml @@ -0,0 +1,564 @@ +datasets: + +- name: experiment + type: IodaObsSpace + filenames: + - {{obs_path_file}} + channels: &channels {{channels}} + groups: + - name: ObsValue + variables: &variables {{simulated_variables}} + - name: GsiHofXBc + #- name: GsiEffectiveQC + - name: hofx0 + - name: hofx1 + - name: ombg + - name: oman + - name: EffectiveQC0 + - name: EffectiveQC1 + - name: MetaData + +transforms: + +# Generate hofx0 for GSI +- transform: arithmetic + new name: experiment::ObsValueMinusGsiHofXBc::${variable} + equals: experiment::ObsValue::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables + +# Generate hofx0 for JEDI +- transform: arithmetic + new name: experiment::ObsValueMinusHofx0::${variable} + equals: experiment::ObsValue::${variable}-experiment::hofx0::${variable} + for: + variable: *variables + +# Generate hofx difference +- transform: arithmetic + new name: experiment::Hofx0MinusGsiHofXBc::${variable} + equals: experiment::hofx0::${variable}-experiment::GsiHofXBc::${variable} + for: + variable: *variables + +# Generate hofx that passed QC for JEDI +- transform: accept where + new name: experiment::hofx0PassedQc::${variable} + starting field: experiment::hofx0::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +# Generate GSI hofx that passed JEDI QC +- transform: accept where + new name: experiment::GsiHofXBcPassedQc::${variable} + starting field: experiment::GsiHofXBc::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +# Generate hofx0 that passed QC for JEDI +- transform: accept where + new name: experiment::ObsValueMinushofx0PassedQc::${variable} + starting field: experiment::ObsValueMinusHofx0::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +# Generate hofx0 that passed QC for GSI +- transform: accept where + new name: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + starting field: experiment::ObsValueMinusGsiHofXBc::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +# Generate ombg that passed QC for JEDI +- transform: accept where + new name: experiment::ombgPassedQc::${variable} + starting field: experiment::ombg::${variable} + where: + - experiment::EffectiveQC0::${variable} == 0 + for: + variable: *variables + +# Generate oman that passed QC for JEDI +- transform: accept where + new name: experiment::omanPassedQc::${variable} + starting field: experiment::oman::${variable} + where: + - experiment::EffectiveQC1::${variable} == 0 + for: + variable: *variables + +# Generate obs contribution to analysis (OmA*OmA)-(OmB*OmB) +- transform: arithmetic + new name: experiment::ResidualRMSdiff::${variable} + equals: (experiment::omanPassedQc::${variable})*(experiment::omanPassedQc::${variable})-(experiment::ombgPassedQc::${variable})*(experiment::ombgPassedQc::${variable}) + for: + variable: *variables + +graphics: + + plotting_backend: Emcpy + figure_list: + + # Correlation scatter plots + # ------------------------- + + # JEDI h(x) vs Observations + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'Observations vs. JEDI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/jedi_hofx0_vs_obs_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofx0::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::hofx0PassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus obs (passed QC in JEDI)' + + # GSI h(x) vs Observations + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'Observations vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx0_vs_obs_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'Observation Value' + add_ylabel: 'GSI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBc::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'GSI h(x) versus obs (all obs)' + - type: Scatter + x: + variable: experiment::ObsValue::${variable} + y: + variable: experiment::GsiHofXBcPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'GSI h(x) versus obs (passed QC in JEDI)' + + # JEDI h(x) vs GSI h(x) + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'JEDI h(x) vs. GSI h(x) | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx_vs_jedi0_hofx_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'GSI h(x)' + add_ylabel: 'JEDI h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::GsiHofXBc::${variable} + y: + variable: experiment::hofx0::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'JEDI h(x) versus GSI h(x)' + - type: Scatter + x: + variable: experiment::GsiHofXBcPassedQc::${variable} + y: + variable: experiment::hofx0PassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'JEDI h(x) versus GSI h(x) (passed QC in JEDI)' + + # JEDI hofx0 vs GSI hofx0 + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'JEDI hofx0 vs. GSI hofx0 | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/gsi_hofx0_vs_jedi_hofx0_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'GSI observation minus h(x)' + add_ylabel: 'JEDI observation minus h(x)' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + y: + variable: experiment::ObsValueMinusHofx0::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'GSI hofx0 vs JEDI hofx0 (all obs)' + - type: Scatter + x: + variable: experiment::ObsValueMinusGsiHofXBcPassedQc::${variable} + y: + variable: experiment::ObsValueMinushofx0PassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'GSI hofx0 vs JEDI hofx0 (passed QC in JEDI)' + + # JEDI oma vs omb + - batch figure: + variables: *variables + channels: *channels + figure: + layout: [1,1] + title: 'OmA vs. OmB | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/correlation_scatter/${variable}${channel}/jedi_oma_vs_omb_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'OmA' + add_ylabel: 'OmB' + add_grid: + add_legend: + loc: 'upper left' + layers: + - type: Scatter + x: + variable: experiment::oman::${variable} + y: + variable: experiment::ombg::${variable} + channel: ${channel} + markersize: 5 + color: 'black' + label: 'OmA versus OmB (all residuals)' + - type: Scatter + x: + variable: experiment::omanPassedQc::${variable} + y: + variable: experiment::ombgPassedQc::${variable} + channel: ${channel} + markersize: 5 + color: 'red' + label: 'OmA versus OmB (passed QC)' + +# Map plots# --------- + + # Observations + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::ObsValue::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'Observations | {{instrument_title}} | Obs Value' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/observations_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: ObsValue + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::ObsValue::${variable} + channel: ${channel} + markersize: 2 + label: ObsValue + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + # hofx0 jedi + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::ObsValueMinusHofx0::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'JEDI hofx0 | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/hofx0_jedi_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: '${variable}' + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::ObsValueMinusHofx0::${variable} + channel: ${channel} + markersize: 2 + label: '${variable}' + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + # hofx0 gsi + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::ObsValueMinusGsiHofXBc::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'GSI hofx0 | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/hofx0_gsi_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: '${variable}' + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + channel: ${channel} + markersize: 2 + label: '${variable}' + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + # hofx difference + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::Hofx0MinusGsiHofXBc::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'Hofx0 Difference | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/hofx0_difference_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: '${variable}' + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::Hofx0MinusGsiHofXBc::${variable} + channel: ${channel} + markersize: 2 + label: '${variable}' + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} + + + # RMS(oma)-RMS(omb) difference + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: vminvmaxcmap + channel: ${channel} + data variable: experiment::ResidualRMSdiff::${variable} + figure: + figure size: [20,10] + layout: [1,1] + title: 'RMS Residual Difference | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/map_plots/${variable}${channel}/rmsres_difference_{{instrument}}_${variable}${channel}.png' + plots: + - mapping: + projection: plcarr + domain: global + add_map_features: ['coastline'] + add_colorbar: + label: '${variable}' + add_grid: + layers: + - type: MapScatter + longitude: + variable: experiment::MetaData::longitude + latitude: + variable: experiment::MetaData::latitude + data: + variable: experiment::ResidualRMSdiff::${variable} + channel: ${channel} + markersize: 2 + label: '${variable}' + colorbar: true + cmap: ${dynamic_cmap} + vmin: ${dynamic_vmin} + vmax: ${dynamic_vmax} +# Histogram plots# --------------- + + # hofx0 vs hofx0 + - batch figure: + variables: *variables + channels: *channels + dynamic options: + - type: histogram_bins + channel: ${channel} + number of bins rule: sturges + data variable: experiment::ObsValueMinusHofx0::${variable} + figure: + layout: [1,1] + title: 'JEDI hofx0 vs. GSI hofx0 | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/histograms/${variable}${channel}/gsi_hofx0_vs_jedi_hofx0_{{instrument}}_${variable}${channel}.png' + plots: + - add_xlabel: 'Observation minus h(x)' + add_ylabel: 'Count' + add_legend: + loc: 'upper left' + layers: + - type: Histogram + data: + variable: experiment::ObsValueMinusGsiHofXBc::${variable} + channel: ${channel} + color: 'blue' + label: 'GSI hofx0 (all obs)' + bins: ${dynamic_bins} + alpha: 0.5 + - type: Histogram + data: + variable: experiment::ObsValueMinusHofx0::${variable} + channel: ${channel} + color: 'red' + label: 'JEDI hofx0 (all obs)' + bins: ${dynamic_bins} + alpha: 0.5 + + # JEDI omb vs oma + - batch figure: + variables: *variables + dynamic options: + - type: histogram_bins + data variable: experiment::omanPassedQc::${variable} + number of bins rule: 'rice' + figure: + layout: [1,1] + title: 'OmB vs. OmA | {{instrument_title}} | ${variable_title}' + output name: '{{cycle_dir}}/eva/{{instrument}}/histograms/${variable}/ombg_oman_{{instrument}}_${variable}.png' + plots: + - add_xlabel: 'Difference' + add_ylabel: 'Count' + set_xlim: [-3, 3] + add_legend: + loc: 'upper left' + statistics: + fields: + - field_name: experiment::ombgPassedQc::${variable} + xloc: 0.5 + yloc: -0.10 + kwargs: + color: 'black' + fontsize: 8 + fontfamily: monospace + - field_name: experiment::omanPassedQc::${variable} + xloc: 0.5 + yloc: -0.13 + kwargs: + color: 'red' + fontsize: 8 + fontfamily: monospace + statistics_variables: + - n + - min + - mean + - max + - std + layers: + - type: Histogram + data: + variable: experiment::ombgPassedQc::${variable} + color: 'red' + label: 'observations minus background ' + bins: ${dynamic_bins} + alpha: 0.5 + density: true + - type: Histogram + data: + variable: experiment::omanPassedQc::${variable} + color: 'blue' + label: 'observations minus analysis' + bins: ${dynamic_bins} + alpha: 0.5 + density: true + diff --git a/src/swell/suites/eda_controlpert_atmos/flow.cylc b/src/swell/suites/eda_controlpert_atmos/flow.cylc new file mode 100644 index 000000000..da246ddf1 --- /dev/null +++ b/src/swell/suites/eda_controlpert_atmos/flow.cylc @@ -0,0 +1,252 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +# -------------------------------------------------------------------------------------------------- + +# Cylc suite for executing JEDI-based non-cycling variational data assimilation + +# -------------------------------------------------------------------------------------------------- + +[scheduler] + UTC mode = True + allow implicit tasks = False + +# -------------------------------------------------------------------------------------------------- + +[scheduling] + + initial cycle point = {{start_cycle_point}} + final cycle point = {{final_cycle_point}} + runahead limit = {{runahead_limit}} + + [[graph]] + R1 = """ + # Triggers for non cycle time dependent tasks + # ------------------------------------------- + # Clone JEDI source code + CloneJedi + + # Build JEDI source code by linking + CloneJedi => BuildJediByLinking? + + # If not able to link to build create the build + BuildJediByLinking:fail? => BuildJedi + + {% for model_component in model_components %} + # Clone geos ana for generating observing system records + CloneGeosMksi-{{model_component}} + {% endfor %} + """ + + {% for cycle_time in cycle_times %} + {{cycle_time.cycle_time}} = """ + {% for model_component in model_components %} + {% if cycle_time[model_component] %} + + # logic tree: + # prep (clone, stage, build, get bkg, get obs) -> SP1 + # -> mksi/observation + filter -> SP2 + # -> [ submit/wait for N chunk runs ]-> SP3 + # -> mean/variance + diffstate -> SP4 -> eva + + # Task triggers for: {{model_component}} + # ------------------ + # Generate satellite channel records + CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}} + + # Get observations + {% if cycling_varbc %} + # Cycling VarBC is active, biases from the previous cycle will be used + + {% if models[model_component]['ensemble_num_chunks'] is defined %} + {% for i in range( 1, models[model_component]['ensemble_num_chunks'] + 1 ) %} + RunJediEdaControlPertExecutable_chunk{{i}}-{{model_component}}[-PT6H] => GetObservations-{{model_component}} + {% endfor %} + {% endif %} + + {% else %} + + # Cycling VarBC is inactive, static bias files will be used + GetObsNotInR2d2-{{model_component}}: fail? => GetObservations-{{model_component}} + {% endif %} + + # Perform staging that is cycle dependent + StageJediCycle-{{model_component}} + + # Run Jedi variational executable + BuildJediByLinking[^]? | BuildJedi[^] => SP1-{{model_component}} + CloneJedi[^] => StageJediCycle-{{model_component}} + StageJediCycle-{{model_component}} => SP1-{{model_component}} + GetEnsembleGeosExperiment-{{model_component}} => SP1-{{model_component}} + GetObsNotInR2d2-{{model_component}}? | GetObservations-{{model_component}} => SP1-{{model_component}} + SP1-{{model_component}} => GenerateObservingSystemRecords-{{model_component}} + GenerateObservingSystemRecords-{{model_component}} => RenderJediObservations-{{model_component}} + RenderJediObservations-{{model_component}} => RunJediObsfiltersExecutable-{{model_component}} => SP2-{{model_component}} + + {% if models[model_component]['ensemble_num_chunks'] is defined %} + {% for i in range( 1, models[model_component]['ensemble_num_chunks'] + 1 ) %} + SP2-{{model_component}} => RunJediEdaControlPertExecutable_chunk{{i}}-{{model_component}} => SP3-{{model_component}} + {% endfor %} + {% endif %} + + # bkg/ana : mean/variance + SP3-{{model_component}} => ShuffleAnaMembers-{{model_component}} + ShuffleAnaMembers-{{model_component}} => RunJediEnsembleMeanVariance-{{model_component}} + RunJediEnsembleMeanVariance-{{model_component}} ==> RunJediDiffstates-{{model_component}} => SP4-{{model_component}} + + # CleanupEda directory + SP4-{{model_component}} => CleanEdaFiles-{{model_component}} + + # EvaIncrement + SP4-{{model_component}} => EvaIncrement-{{model_component}} + + {% endif %} + {% endfor %} + """ + {% endfor %} + +# -------------------------------------------------------------------------------------------------- + +[runtime] + + # Task defaults + # ------------- + [[root]] + pre-script = "source $CYLC_SUITE_DEF_PATH/modules" + + [[[environment]]] + datetime = $CYLC_TASK_CYCLE_POINT + config = $CYLC_SUITE_DEF_PATH/experiment.yaml + + # Tasks + # ----- + [[CloneJedi]] + script = "swell task CloneJedi $config" + + [[BuildJediByLinking]] + script = "swell task BuildJediByLinking $config" + + [[BuildJedi]] + script = "swell task BuildJedi $config" + platform = {{platform}} + execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %} + --{{key}} = {{value}} + {%- endfor %} + + {% for model_component in model_components %} + + [[CloneGeosMksi-{{model_component}}]] + script = "swell task CloneGeosMksi $config -m {{model_component}}" + + [[GenerateObservingSystemRecords-{{model_component}}]] + script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}" + + [[GetObsNotInR2d2-{{model_component}}]] + script = "swell task GetObsNotInR2d2 $config -d $datetime -m {{model_component}}" + + [[StageJediCycle-{{model_component}}]] + script = "swell task StageJedi $config -d $datetime -m {{model_component}}" + + [[GetBackground-{{model_component}} ]] + script = "swell task GetBackground $config -d $datetime -m {{model_component}}" + + [[GetBackgroundGeosExperiment-{{model_component}} ]] + script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}" + + [[GetEnsembleGeosExperiment-{{model_component}}]] + script = "swell task GetEnsembleGeosExperiment $config -d $datetime -m {{model_component}}" + + [[GetObservations-{{model_component}}]] + script = "swell task GetObservations $config -d $datetime -m {{model_component}}" + + [[RenderJediObservations-{{model_component}}]] + script = "swell task RenderJediObservations $config -d $datetime -m {{model_component}}" + + {% if models[model_component]['ensemble_num_chunks'] is defined %} + {% for i in range( 1, models[model_component]['ensemble_num_chunks'] + 1 ) %} + [[RunJediEdaControlPertExecutable_chunk{{i}}-{{model_component}}]] + script = "swell task RunJediEdaControlPertExecutable $config -d $datetime -m {{model_component}} -ichunk {{i}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediEdaControlPertExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediEdaControlPertExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + {% endfor %} + {% endif %} + + [[ShuffleAnaMembers-{{model_component}}]] + script = "swell task RunJediEdaControlPertExecutable $config -d $datetime -m {{model_component}} -ichunk -1" + + [[RunJediObsfiltersExecutable-{{model_component}}]] + script = "swell task RunJediObsfiltersExecutable $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediObsfiltersExecutable"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediObsfiltersExecutable"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[RunJediEnsembleMeanVariance-{{model_component}}]] + script = "swell task RunJediEnsembleMeanVariance $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediEnsembleMeanVariance"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediEnsembleMeanVariance"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[RunJediDiffstates-{{model_component}}]] + script = "swell task RunJediDiffstates $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["RunJediDiffstates"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["RunJediDiffstates"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[CleanEdaFiles-{{model_component}}]] + script = "swell task CleanEda $config -d $datetime -m {{model_component}}" + + [[EvaJediLog-{{model_component}}]] + script = "swell task EvaJediLog $config -d $datetime -m {{model_component}}" + + [[EvaIncrement-{{model_component}}]] + script = "swell task EvaIncrement $config -d $datetime -m {{model_component}}" + + [[EvaObservations-{{model_component}}]] + script = "swell task EvaObservations $config -d $datetime -m {{model_component}}" + platform = {{platform}} + execution time limit = {{scheduling["EvaObservations"]["execution_time_limit"]}} + [[[directives]]] + {%- for key, value in scheduling["EvaObservations"]["directives"][model_component].items() %} + --{{key}} = {{value}} + {%- endfor %} + + [[SaveObsDiags-{{model_component}}]] + script = "swell task SaveObsDiags $config -d $datetime -m {{model_component}}" + + [[CleanCycle-{{model_component}}]] + script = "swell task CleanCycle $config -d $datetime -m {{model_component}}" + + # SP: Sync Point + [[SP1-{{model_component}}]] + script = true + + [[SP2-{{model_component}}]] + script = true + + [[SP3-{{model_component}}]] + script = true + + [[SP4-{{model_component}}]] + script = true + + {% endfor %} + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/eda_controlpert_atmos/suite_config.py b/src/swell/suites/eda_controlpert_atmos/suite_config.py new file mode 100644 index 000000000..959c0cbaa --- /dev/null +++ b/src/swell/suites/eda_controlpert_atmos/suite_config.py @@ -0,0 +1,209 @@ +# -------------------------------------------------------------------------------------------------- +# @package configuration +# +# Class containing the configuration. This is a dictionary that is converted from +# an input yaml configuration file. Various function are included for interacting with the +# dictionary. +# +# -------------------------------------------------------------------------------------------------- + + +from swell.utilities.swell_questions import QuestionContainer, QuestionList +from swell.utilities.question_defaults import QuestionDefaults as qd +from swell.suites.suite_questions import SuiteQuestions as sq + +from enum import Enum + + +# -------------------------------------------------------------------------------------------------- + +class SuiteConfig(QuestionContainer, Enum): + + # -------------------------------------------------------------------------------------------------- + + eda_controlpert_atmos_tier1_fast = QuestionList( + list_name="eda_controlpert_atmos_tier1", + questions=[ + sq.common, + qd.start_cycle_point("2023-10-10T00:00:00Z"), + qd.final_cycle_point("2023-10-10T06:00:00Z"), + qd.runahead_limit("P2"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.cycle_times([ + "T00", + ]), + qd.background_experiment('x0050'), + qd.geos_x_background_directory("/discover/nobackup/projects/gmao/dadev/" + "rtodling/archive/Restarts/JEDI/541x"), + qd.geos_x_ensemble_directory("/discover/nobackup/projects/gmao/dadev/" + 'rtodling/archive/541/Milan'), + qd.npx_proc(4), + qd.npy_proc(5), + qd.perhost(120), + qd.window_length("PT6H"), + qd.window_type("3D"), + qd.horizontal_resolution("91"), + qd.gsibec_nlats("91"), + qd.gsibec_nlons("144"), + qd.vertical_resolution("72"), + qd.ensemble_num_members(4), + qd.ensemble_num_chunks(2), + qd.obs_pert_amplitude(0.5), + qd.number_of_iterations([10]), + qd.gradient_norm_reduction(1.e-3), + qd.minimizer("DRPLanczos"), + qd.analysis_variables([ + "eastward_wind", + "northward_wind", + "air_temperature", + "water_vapor_mixing_ratio_wrt_moist_air", + "air_pressure_at_surface", + "air_pressure_levels", + "cloud_liquid_ice", + "cloud_liquid_water", + "rain_water", + "snow_water", + "mole_fraction_of_ozone_in_air", + "geopotential_at_surface", + "fraction_of_ocean", + "fraction_of_lake", + "fraction_of_ice", + "skin_temperature_at_surface" + ]), + qd.observations([ + "sondes", + ]), + qd.obs_thinning_rej_fraction(0.8), + qd.ensmeanvariance_spec([ + {"state": "bkg", + "fn_input": "ebkg/mem%mem%/geos.mem%mem%.%yyyy%mm%dd_%hh%MM%ssz.nc4", + "fn_output_mean": "geos.prior.mean", + "fn_output_variance": "geos.prior.variance", + "grid_type": ['cs', 'latlon']}, + {"state": "analysis", + "fn_input": "analysis/mem%mem%/eda.ana.mem%mem%.%yyyy%mm%dd_%hh%MM%ssz.nc4", + "fn_output_mean": "eda.ana.mean", + "fn_output_variance": "eda.ana.variance", + "grid_type": ['cs', 'latlon']}, + ]), + qd.diffstates_spec({ + "state1": + {"fn_input": "geos.prior.mean.%yyyy%mm%dd_%hh%MM%ssz.nc4"}, + "state2": + {"fn_input": "eda.ana.mean.%yyyy%mm%dd_%hh%MM%ssz.nc4"}, + "state_diff": + {"fn_output": "eda.mean-inc", "grid_type": ['cs', 'latlon']}, + "state_type": "ensemble" + }), + qd.clean_patterns(['*.txt', '*.csv']), + ] + ) + + eda_controlpert_atmos_tier1 = QuestionList( + list_name="eda_controlpert_atmos_tier1", + questions=[ + sq.common, + qd.start_cycle_point("2023-10-10T00:00:00Z"), + qd.final_cycle_point("2023-10-10T06:00:00Z"), + qd.runahead_limit("P2"), + qd.jedi_build_method("use_existing"), + qd.model_components(['geos_atmosphere']), + ], + geos_atmosphere=[ + qd.cycle_times([ + "T00", + ]), + qd.background_experiment('x0050'), + qd.geos_x_background_directory("/discover/nobackup/projects/gmao/dadev/" + "rtodling/archive/Restarts/JEDI/541x"), + qd.geos_x_ensemble_directory("/discover/nobackup/projects/gmao/dadev/" + 'rtodling/archive/541/Milan'), + qd.npx_proc(4), + qd.npy_proc(5), + qd.perhost(120), + qd.window_length("PT6H"), + qd.window_type("3D"), + qd.horizontal_resolution("91"), + qd.gsibec_nlats("91"), + qd.gsibec_nlons("144"), + qd.vertical_resolution("72"), + qd.ensemble_num_members(32), + qd.ensemble_num_chunks(16), + qd.obs_pert_amplitude(0.5), + qd.number_of_iterations([50]), + qd.gradient_norm_reduction(1.e-3), + qd.minimizer("DRPLanczos"), + qd.analysis_variables([ + "eastward_wind", + "northward_wind", + "air_temperature", + "water_vapor_mixing_ratio_wrt_moist_air", + "air_pressure_at_surface", + "air_pressure_levels", + "cloud_liquid_ice", + "cloud_liquid_water", + "rain_water", + "snow_water", + "mole_fraction_of_ozone_in_air", + "geopotential_at_surface", + "fraction_of_ocean", + "fraction_of_lake", + "fraction_of_ice", + "skin_temperature_at_surface" + ]), + # + # Report first crash point: Bound-1 + # + qd.observations([ + "aircraft_temperature", + "aircraft_wind", + "airs_aqua", + "amsr2_gcom-w1", + "amsua_aqua", + "amsua_metop-b", + "amsua_metop-c", + "amsua_n15", + "amsua_n18", + "amsua_n19", + "atms_n20", + "atms_npp", + ]), + qd.obs_thinning_rej_fraction(0.8), + qd.ensmeanvariance_spec([ + {"state": "bkg", + "fn_input": "ebkg/mem%mem%/geos.mem%mem%.%yyyy%mm%dd_%hh%MM%ssz.nc4", + "fn_output_mean": "geos.prior.mean", + "fn_output_variance": "geos.prior.variance", + "grid_type": ['cs', 'latlon']}, + {"state": "analysis", + "fn_input": "analysis/mem%mem%/eda.ana.mem%mem%.%yyyy%mm%dd_%hh%MM%ssz.nc4", + "fn_output_mean": "eda.ana.mean", + "fn_output_variance": "eda.ana.variance", + "grid_type": ['cs', 'latlon']}, + ]), + qd.diffstates_spec({ + "state1": + {"fn_input": "geos.prior.mean.%yyyy%mm%dd_%hh%MM%ssz.nc4"}, + "state2": + {"fn_input": "eda.ana.mean.%yyyy%mm%dd_%hh%MM%ssz.nc4"}, + "state_diff": + {"fn_output": "eda.mean-inc", "grid_type": ['cs', 'latlon']}, + "state_type": "ensemble" + }), + qd.clean_patterns(['*.txt', '*.csv']), + ] + ) + + # -------------------------------------------------------------------------------------------------- + + eda_controlpert_atmos = QuestionList( + list_name="eda_controlpert_atmos", + questions=[ + eda_controlpert_atmos_tier1_fast + ] + ) + + # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/swell.py b/src/swell/swell.py index bfdabf285..0ff852ecc 100644 --- a/src/swell/swell.py +++ b/src/swell/swell.py @@ -209,6 +209,7 @@ def launch( @click.option('-a', '--additional-parameter', 'additional_parameter', default=None, help=additional_parameter_help) @click.option('-p', '--ensemblePacket', 'ensemblePacket', default=None, help=ensemble_help) +@click.option('-ichunk', '--ensemble_ichunk', 'ichunk', type=int, default=None, help=ensemble_help) @click.option('-imem', '--ensemble_imember', 'imember', type=int, default=None, help=ensemble_help) def task( task: str, @@ -217,6 +218,7 @@ def task( model: Optional[str], additional_parameter: Optional[str], ensemblePacket: Optional[str], + ichunk: Optional[int], imember: Optional[int] ) -> None: """ @@ -230,7 +232,7 @@ def task( """ task_wrapper(task, config, datetime, model, additional_parameter, - ensemblePacket, imember) + ensemblePacket, ichunk, imember) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py index b276a759e..32454d5aa 100644 --- a/src/swell/tasks/base/task_base.py +++ b/src/swell/tasks/base/task_base.py @@ -42,6 +42,7 @@ def __init__( model: str, ensemblePacket: Optional[str], additional_parameter: Optional[str], + ichunk: int | None, imember: int | None, task_name: str ) -> None: @@ -71,6 +72,7 @@ def __init__( # Keep copy of ensemblePacket # --------------------------- self.__ensemble_packet__ = ensemblePacket + self.__ensemble_ichunk__ = ichunk self.__ensemble_imember__ = imember # Keep copy of model directive @@ -175,6 +177,11 @@ def get_ensemble_packet(self) -> Optional[str]: # ---------------------------------------------------------------------------------------------- + def get_ensemble_ichunk(self) -> int | None: + return self.__ensemble_ichunk__ + + # ---------------------------------------------------------------------------------------------- + def get_ensemble_imember(self) -> int | None: return self.__ensemble_imember__ @@ -293,6 +300,7 @@ def create_task( model: str, additional_parameter: str | None, ensemblePacket: Optional[str], + ichunk: int | None = None, imember: int | None = None, ) -> taskBase: @@ -323,7 +331,7 @@ def create_task( # Return task object return task_class(config, datetime, model, ensemblePacket, - additional_parameter, imember, task) + additional_parameter, ichunk, imember, task) # -------------------------------------------------------------------------------------------------- @@ -365,6 +373,7 @@ def task_wrapper( model: Optional[str], additional_parameter: str | None, ensemblePacket: Optional[str], + ichunk: int | None = None, imember: int | None = None, ) -> None: @@ -372,7 +381,7 @@ def task_wrapper( constrc_start = time.perf_counter() creator = taskFactory() task_object = creator.create_task(task, config, datetime, model, additional_parameter, - ensemblePacket, imember=imember) + ensemblePacket, ichunk=ichunk, imember=imember) constrc_final = time.perf_counter() constrc_time = f'Constructed in {constrc_final - constrc_start:0.4f} seconds' diff --git a/src/swell/tasks/clean_eda.py b/src/swell/tasks/clean_eda.py index 7b745ffda..b0558c021 100644 --- a/src/swell/tasks/clean_eda.py +++ b/src/swell/tasks/clean_eda.py @@ -10,6 +10,7 @@ import os import glob import shutil +from pathlib import Path from swell.tasks.base.task_base import taskBase @@ -19,6 +20,8 @@ class CleanEda(taskBase): # ---------------------------------------------------------------------------------------------- + # this file handles both eda and eda_controlpert cases + # def execute(self) -> None: @@ -56,8 +59,6 @@ def execute(self) -> None: window_begin = self.da_window_params.window_begin(window_length) window_begin_iso = self.da_window_params.window_begin_iso(window_length) window_end_iso = self.da_window_params.window_end_iso(window_length) - nmember = self.config.ensemble_num_members() - # imember = self.get_ensemble_imember() # Populate jedi interface templates dictionary # -------------------------------------------- @@ -119,10 +120,20 @@ def execute(self) -> None: window_type, jedi_forecast_model) - for imem in range(1, nmember+1): - mem_dir = f'analysis/mem{imem:003d}/' - d1 = os.path.join(self.cycle_dir(), mem_dir) - d2 = os.path.join(d1, 'fv3-jedi') + # This special design works with either eda or eda_control_pert + # handle eda case: analysis/mem00x + # eda controlpert case: analysis_chunk/chunk00x/mem00y + # This avoids blindly search for mem0xx dir, as some of them + # only contains linked analysis files and donot require clean up + # -------------------------------------------------------------- + d1 = os.path.join(self.cycle_dir(), 'analysis_chunk') # control pert case + if not os.path.exists(d1): + d1 = os.path.join(self.cycle_dir(), 'analysis') # eda case + target_dirs = [str(p) for p in Path(d1).rglob('mem*') if p.is_dir()] + self.logger.info(f'target_dirs = {target_dirs}') + + for mem_dir in target_dirs: + d2 = os.path.join(mem_dir, 'fv3-jedi') if os.path.exists(d2): if os.path.islink(d2): # Only remove the symlink itself, never follow it @@ -134,10 +145,10 @@ def execute(self) -> None: for observer in jedi_config_dict['cost function']['observations']['observers']: # Get observation name observation = observer['observation_name'] - # Delete input obsfile in analysis/mem00x (.nc4 .tlapse.txt acftbias acftbias_cov) - for file_path in glob.glob(os.path.join(d1, f'{observation}.*')): + # Delete input obsfile (.nc4 .tlapse.txt acftbias acftbias_cov) + for file_path in glob.glob(os.path.join(mem_dir, f'{observation}.*')): if os.path.islink(file_path): - os.unlink(file_path) # safe: removes only the link + os.unlink(file_path) # safe: removes only the link self.logger.info(f"Deleted symlink file: {file_path}") else: try: diff --git a/src/swell/tasks/run_jedi_eda_control_pert_executable.py b/src/swell/tasks/run_jedi_eda_control_pert_executable.py new file mode 100644 index 000000000..87ac96b4d --- /dev/null +++ b/src/swell/tasks/run_jedi_eda_control_pert_executable.py @@ -0,0 +1,343 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import os +import shutil +from glob import glob +from pathlib import Path +from ruamel.yaml import YAML + +from swell.tasks.base.task_base import taskBase +from swell.utilities.run_jedi_executables import run_executable +from swell.utilities.yaml_utils import replace_string_value + +# -------------------------------------------------------------------------------------------------- + + +class RunJediEdaControlPertExecutable(taskBase): + + # ---------------------------------------------------------------------------------------------- + # + # ichunk: + # 1:nchunk: normal chunking + # 0 : reorder chunk ana file to analysis/mem00x/ana.nc4 by symlink + # + def execute(self) -> None: + + # Jedi application name + # --------------------- + jedi_application = 'eda_control_pert' + + # Parse configuration + # ------------------- + window_type = self.config.window_type() + window_length = self.config.window_length() + forecast_length = self.config.forecast_length(window_length) + background_time_offset = self.config.background_time_offset() + number_of_iterations = self.config.number_of_iterations() + jedi_forecast_model = self.config.jedi_forecast_model(None) + generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + perhost = self.config.perhost(None) + + # Set the observing system records path + self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + + gsibec_nlats = self.config.gsibec_nlats(None) + gsibec_nlons = self.config.gsibec_nlons(None) + gsibec_configuration = self.config.gsibec_configuration(None) + npx_proc = self.config.npx_proc(None) + npy_proc = self.config.npy_proc(None) + npx = self.config.npx(None) + npy = self.config.npy(None) + + # Compute data assimilation window parameters + # -------------------------------------------- + background_time = self.da_window_params.background_time(background_time_offset) + local_background_time = self.da_window_params.local_background_time(window_length, + window_type) + local_background_time_iso = self.da_window_params.local_background_time_iso(window_length, + window_type) + window_begin = self.da_window_params.window_begin(window_length) + window_begin_iso = self.da_window_params.window_begin_iso(window_length) + window_end_iso = self.da_window_params.window_end_iso(window_length) + nmember = self.config.ensemble_num_members() + nchunk = self.config.ensemble_num_chunks() + ichunk = self.get_ensemble_ichunk() + + # exit execute if ichunk=-1: meaning reoder analysis files and return + # ------------------------------------------------------------------- + if ichunk == -1: + npert = int(nmember/nchunk) + istart = 0 + imem = 0 + for xchunk in range(1, nchunk+1): + if xchunk == 1: + iend = npert - 1 # mem: [0, 1 ... npert-1] : [ctrl, all pert] + else: + iend = npert # mem: [0, 1 ... npert] : [ctrl, all pert] + for imem_in_chunk in range(istart, iend+1): + # skip extra ctrl, because of ichunk=-1 + if xchunk > 1 and imem_in_chunk == 0: + continue + else: + imem += 1 + dir_a = f'analysis_chunk/chunk{xchunk:03d}/mem{imem_in_chunk:03d}' + dir_b = f'analysis/mem{imem:03d}' + dir_a_full = os.path.join(self.cycle_dir(), dir_a, + f'eda.ana.mem{imem_in_chunk:03d}.*.nc4') + fa_list = glob(dir_a_full) + if fa_list: + fa = fa_list[0] + else: + fa = '' + self.logger.error( + f"analysis files not found ichunk={ichunk}, " + f"imem_in_chunk={imem_in_chunk}") + tail = '.'.join(os.path.basename(fa).split('.')[-2:]) + fb = os.path.join(self.cycle_dir(), dir_b, f'eda.ana.mem{imem:03d}.{tail}') + # link fa to fb + # Convert strings to Path objects + fa = Path(fa) + fb = Path(fb) + # a1. Make the directory if it does not exist + fb.parent.mkdir(parents=True, exist_ok=True) + # a2. If fb exists (or is a broken symlink), safely remove it + if fb.is_symlink() or fb.exists(): + fb.unlink() + # a3. Create the symbolic link + # .resolve() gets the absolute path (like realpath in bash) + fb.symlink_to(fa.resolve()) + self.logger.info(f"Successfully linked {fa} to {fb}") + return + + # Populate jedi interface templates dictionary + # -------------------------------------------- + self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) + self.jedi_rendering.add_key('window_end_iso', window_end_iso) + self.jedi_rendering.add_key('window_length', window_length) + self.jedi_rendering.add_key('forecast_length', forecast_length) + self.jedi_rendering.add_key('minimizer', self.config.minimizer()) + self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) + self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) + self.jedi_rendering.add_key('saber_central_block', self.config.saber_central_block(None)) + self.jedi_rendering.add_key('saber_outer_block', self.config.saber_outer_block(None)) + self.jedi_rendering.add_key('gradient_norm_reduction', + self.config.gradient_norm_reduction()) + self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + + # Background + # ---------- + self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('local_background_time', local_background_time) + self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) + self.jedi_rendering.add_key('ensemble_num_members', nmember) + self.jedi_rendering.add_key('ensemble_num_chunks', nchunk) + self.jedi_rendering.add_key('ensemble_ichunk', ichunk) + + # Geometry + # -------- + self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) + self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) + self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) + self.jedi_rendering.add_key('npx_proc', npx_proc) + self.jedi_rendering.add_key('npy_proc', npy_proc) + self.jedi_rendering.add_key('npx', npx) + self.jedi_rendering.add_key('npy', npy) + self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + + # Observations + # ------------ + self.jedi_rendering.add_key('background_time', background_time) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('window_begin', window_begin) + + # Atmosphere background error model + # --------------------------------- + if gsibec_configuration is not None: + self.jedi_rendering.add_key('gsibec_configuration', gsibec_configuration) + self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) + self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) + self.jedi_rendering.add_key('gsibec_npx_proc', npx_proc) + self.jedi_rendering.add_key('gsibec_npy_proc', 6*npy_proc) + + # Model + # ----- + if window_type == '4D': + self.jedi_rendering.add_key('background_frequency', self.config.background_frequency()) + # Jedi configuration file + # ----------------------- + fname = f'jedi_{jedi_application}{window_type}_config_chunk{ichunk:03d}.yaml' + jedi_config_file = os.path.join(self.cycle_dir(), fname) + + # Output log file + # --------------- + output_log_file = os.path.join( + self.cycle_dir(), f"jedi_{jedi_application}{window_type}_log_chunk{ichunk:03d}.log") + + # Open the JEDI config file and fill initial templates + # ---------------------------------------------------- + jedi_config_dict = self.jedi_rendering.render_oops_file(f'{jedi_application}', + window_type, + jedi_forecast_model) + + npert = int(nmember/nchunk) + istart = 0 + if ichunk == 1: + iend = npert - 1 # mem: [0, 1 ... npert-1] : [ctrl, all pert] + else: + iend = npert # mem: [0, 1 ... npert] : [ctrl, all pert] + + # round-1: link bkg, copy obs, fv3jedi dir (B and R) + # ---------------------------------------------------- + mem_temp_dir = f'analysis_chunk/chunk{ichunk:003d}/mem%mem_pad%' + for imem in range(istart, iend+1): + if ichunk == 1: + id = imem + 1 + else: + if imem == 0: + id = 1 + else: + id = npert*(ichunk-1) + imem + + # link ebkg to ebkg_chunk + # ----------------------- + ebkg_chunk_dir = f'ebkg_chunk/chunk{ichunk:003d}/' + xdir = os.path.join(self.cycle_dir(), ebkg_chunk_dir, f'geos.mem{imem:03d}') + os.makedirs(xdir, exist_ok=True) + + # copy bkg files to imem dir + f1_list = glob(os.path.join(self.cycle_dir(), f'ebkg/mem{id:03d}/geos.mem*.nc4')) + if not f1_list: + self.logger.error(f"ebkg dir is empty for member id: {id}") + for f1 in f1_list: + f2 = os.path.basename(f1) + f2 = f2.split('.')[2:] + f2 = '.'.join(f2) + f2 = os.path.join(xdir, f2) + if os.path.lexists(f2): + os.remove(f2) + os.symlink(f1, f2) + + # analysis_chunk / chunk00x / mem00y will have its own obs, B and R + # ----------------------------------------------------------------- + mem_dir = f'analysis_chunk/chunk{ichunk:003d}/mem{imem:03d}' + xdir = os.path.join(self.cycle_dir(), mem_dir) + os.makedirs(xdir, exist_ok=True) + + for observer in jedi_config_dict['assimilation'][ + 'cost function']['observations']['observers']: + # Get observation name + observation = observer['observation_name'] + # copy obs input file to avoid multi MPI reading the same file + files = glob(os.path.join(self.cycle_dir(), f'{observation}.*')) + for src_file in files: + self.logger.info(f'f= {src_file} is copied to {xdir}') + shutil.copy(src_file, xdir) + + # copy fv3-jedi dir, update dir names + d1 = os.path.join(self.cycle_dir(), 'fv3-jedi') + d2 = os.path.join(self.cycle_dir(), mem_dir, 'fv3-jedi') + shutil.copytree(d1, d2, dirs_exist_ok=True) + + # round-2: modify dir keys in yaml to chunk00x / mem%mem_pad% + # ------------------------------------------------------------ + for observer in jedi_config_dict['assimilation'][ + 'cost function']['observations']['observers']: + # Get observation name + observation = observer['observation_name'] + + # for now delete obsdataout + del observer['obs space']['obsdataout'] + # hxout = observer['obs space']['obsdataout']['engine']['obsfile'] + # dir1, fname = os.path.split(hxout) + # hxout = os.path.join(dir1, mem_temp_dir, fname) + # observer['obs space']['obsdataout']['engine']['obsfile'] = hxout + + obsFileIn = observer['obs space']['obsdatain']['engine']['obsfile'] + dir1, fname = os.path.split(obsFileIn) + obsFileIn = os.path.join(dir1, mem_temp_dir, fname) + observer['obs space']['obsdatain']['engine']['obsfile'] = obsFileIn + + obs_bias = observer.get('obs bias') + if obs_bias is not None: + File = obs_bias['input file'] + dir1, fname = os.path.split(File) + File = os.path.join(dir1, mem_temp_dir, fname) + obs_bias['input file'] = File + # + File = obs_bias['output file'] + dir1, fname = os.path.split(File) + File = os.path.join(dir1, mem_temp_dir, fname) + obs_bias['output file'] = File + # + File = obs_bias.get('covariance', {}).get('output file') + if File is not None: + dir1, fname = os.path.split(File) + File = os.path.join(dir1, mem_temp_dir, fname) + obs_bias['covariance']['output file'] = File + # + File = obs_bias.get('covariance', {}).get('prior', {}).get('input file') + if File is not None: + dir1, fname = os.path.split(File) + File = os.path.join(dir1, mem_temp_dir, fname) + obs_bias['covariance']['prior']['input file'] = File + + observer['obs space']['obs perturbations seed shift'] = (ichunk-1)*npert + + # round-3: point dir to newly created chunks dir + # ---------------------------------------------------- + dir_list = ["bkg", "fv3files", "gsibec", "rcov"] + for i in dir_list: + j = f"fv3-jedi/{i}" + k = f"{mem_temp_dir}/{j}" # Result: analysis/chunk00x/mem_pad/fv3-jedi/rcov + jedi_config_dict = replace_string_value(jedi_config_dict, j, k) + # Result: e.g., analysis/mem002/fv3-jedi/rcov + + ruamel_yaml = YAML() + ruamel_yaml.default_flow_style = False + + # Write the ordered dictionary to YAML file + with open(jedi_config_file, 'w') as jedi_config_file_open: + ruamel_yaml.dump(jedi_config_dict, jedi_config_file_open) + + # Get the JEDI interface metadata + # ------------------------------- + model_component_meta = self.jedi_rendering.render_interface_meta() + + self.logger.info(f"num of processors = {model_component_meta['total_processors']}") + # Compute number of processors + # ---------------------------- + np = eval(str(model_component_meta['total_processors'])) + # modify np by (nmember / nchunk) + 1 + if ichunk == 1: + np = np * npert + else: + np = np * (npert + 1) + + # Jedi executable name + # -------------------- + jedi_executable = model_component_meta['executables']['edaControlPert'] + jedi_executable_path = os.path.join(self.experiment_path(), 'jedi_bundle', + 'build', 'bin', jedi_executable) + + # Run the JEDI executable + # ----------------------- + if not generate_yaml_and_exit: + self.logger.info('Running '+jedi_executable_path+' with '+str(np)+' processors.') + run_executable(self.logger, self.cycle_dir(), np, jedi_executable_path, + jedi_config_file, output_log_file, perhost) + else: + mpi_command = "mpirun" + if not (perhost is None or perhost == "None"): + mpi_command += f" -perhost {perhost}" + mpi_command += f" -np {np} {jedi_executable_path} {jedi_config_file} {output_log_file}" + print(f'intended mpi_command = {mpi_command}') + self.logger.info('YAML generated, now exiting.') + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index e04d03d33..4422ef7cf 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -852,6 +852,22 @@ class TaskQuestions(QuestionContainer, Enum): # -------------------------------------------------------------------------------------------------- + RunJediEdaControlPertExecutable = QuestionList( + list_name="RunJediEdaControlPertExecutable", + questions=[ + run_jedi_executable, + qd.ensemble_num_members(), + qd.ensemble_num_chunks(), + qd.obs_pert_amplitude(), + qd.obs_thinning_rej_fraction(), + qd.perhost(), + qd.comparison_log_type('variational'), + qd.mock_experiment() + ] + ) + + # -------------------------------------------------------------------------------------------------- + RunCompressForecast = QuestionList( list_name="RunCompressForecast", questions=[ diff --git a/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml b/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml index ebae35df6..358a11355 100644 --- a/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml +++ b/src/swell/test/jedi_configs/jedi_localensembleda_config.yaml @@ -25,7 +25,7 @@ increment variables: - rain_water - snow_water - mole_fraction_of_ozone_in_air -- geopotential_height_times_gravity_at_surface +- geopotential_at_surface - fraction_of_ocean - fraction_of_lake - fraction_of_ice diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 1ed626a62..ae827d8ff 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -542,6 +542,19 @@ class ensemble_hofx_strategy(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass + class ensemble_num_chunks(TaskQuestion): + default_value: str = "defer_to_model" + question_name: str = "ensemble_num_chunks" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "How many chunks shall be used in EDA control pert calculations?" + widget_type: WType = WType.INTEGER + + # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_num_members(TaskQuestion): default_value: str = "defer_to_model" diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 8c800f1b4..fb272dce3 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -85,6 +85,8 @@ def __init__( 'ensemble_hofx_packets', 'ensemble_hofx_strategy', 'ensemble_num_members', + 'ensemble_num_chunks', + 'ensemble_ichunk', 'ensemble_imember', 'ensmean_only', 'ensmeanvariance_only', diff --git a/src/swell/utilities/slurm.py b/src/swell/utilities/slurm.py index 0d3a0ccea..a5a158751 100644 --- a/src/swell/utilities/slurm.py +++ b/src/swell/utilities/slurm.py @@ -47,7 +47,9 @@ def prepare_scheduling_dict( "RunJediVariationalExecutable": {"all": {"nodes": 3}}, "RunJediUfoTestsExecutable": {"all": {"ntasks-per-node": 1}}, "RunJediConvertStateSoca2ciceExecutable": {"all": {"nodes": 1}}, - "RunJediEdaExecutable": {"all": {"ntasks-per-node": 126}} + "RunJediEdaExecutable": {"all": {"ntasks-per-node": 126}}, + "RunJediEdaControlPertExecutable": {"all": {"nodes": 3, "ntasks-per-node": 126}}, + "RunJediLocalEnsembleDaExecutable": {"all": {"nodes": 2}}, } # Global SLURM settings stored in $HOME/.swell/swell-slurm.yaml @@ -82,6 +84,7 @@ def prepare_scheduling_dict( 'RunCompressForecast', 'RunGeos', 'RunJediEdaExecutable', + 'RunJediEdaControlPertExecutable', 'RunJediEnsembleMeanVariance', 'RunJediDiffstates', 'RunJediConvertStateSoca2ciceExecutable', diff --git a/src/swell/utilities/yaml_utils.py b/src/swell/utilities/yaml_utils.py index c15810552..d404f33ed 100644 --- a/src/swell/utilities/yaml_utils.py +++ b/src/swell/utilities/yaml_utils.py @@ -1,6 +1,8 @@ +import yaml # -------------------------------------------------------------------------------------------------- + def replace_key(obj, old_key, new_key): """ Recursively replace dictionary keys in nested dictionaries/lists. @@ -41,3 +43,19 @@ def replace_string_value(data, sa, sb): return data # -------------------------------------------------------------------------------------------------- + + +def print_dict(*args, **kwargs): + return print_dict_as_yaml(*args, **kwargs) + + +def print_dict_as_yaml(data_dict): + # Convert the dictionary to a YAML string + # default_flow_style=False ensures block formatting instead of inline JSON-like formatting + # sort_keys=False preserves your dictionary's original key order (Python 3.7+) + yaml_string = yaml.dump(data_dict, default_flow_style=False, sort_keys=False) + + # Output to the terminal + print(yaml_string) + +# --------------------------------------------------------------------------------------------------