From 781814eb5e0dcf9b30f9708cf1adcb46892ea1c2 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 24 Aug 2026 13:31:34 -0400 Subject: [PATCH 01/15] Refactor how configs reference questions --- src/swell/configuration/question_defaults.py | 1646 ++++++++++++++++++ src/swell/tasks/clone_jedi.py | 1 + src/swell/utilities/config.py | 84 +- src/swell/utilities/swell_questions.py | 93 +- 4 files changed, 1713 insertions(+), 111 deletions(-) create mode 100644 src/swell/configuration/question_defaults.py diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py new file mode 100644 index 000000000..2faa3f3a3 --- /dev/null +++ b/src/swell/configuration/question_defaults.py @@ -0,0 +1,1646 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +from dataclasses import dataclass, field +from typing import List, Dict, Any + +from swell.utilities.swell_questions import SuiteQuestion, TaskQuestion +from swell.utilities.swell_questions import DataType as DType +from swell.utilities.dataclass_utils import mutable_field + +# -------------------------------------------------------------------------------------------------- +# Suite question defaults go here +# -------------------------------------------------------------------------------------------------- + +@dataclass +class comparison_experiment_paths(SuiteQuestion): + default_value: list = mutable_field([]) + prompt: str = "Provide paths to two experiments to run comparison tests on." + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class cycle_times(SuiteQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the cycle times for this model." + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class cycling_varbc(SuiteQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Do you want to use cycling VarBC option?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class download_convert_pipeline(SuiteQuestion): + default_value: bool = False + prompt: str = ("Run the DownloadObs and ConvertObsToIoda tasks?" + "(DownloadObs -> ConvertObsToIoda) -> IngestObs to R2D2") + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensemble_hofx_packets(SuiteQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the number of ensemble packets." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensemble_hofx_strategy(SuiteQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the ensemble hofx strategy." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class experiment_id(SuiteQuestion): + default_value: str = "defer_to_code" + prompt: str = "What is the experiment id?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class experiment_root(SuiteQuestion): + default_value: str = "defer_to_platform" + prompt: str = ("What is the experiment root (the directory where the " + "experiment will be stored)?") + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class final_cycle_point(SuiteQuestion): + default_value: str = "2023-10-10T06:00:00Z" + prompt: str = "What is the time of the final cycle (middle of the window)?" + data_type: DType = DType.ISO_DATETIME + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class marine_models(SuiteQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_marine" + ]) + prompt: str = "Select the active SOCA models for this model." + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class mock_experiment(SuiteQuestion): + default_value: bool = False + prompt: str = "Dry-run option for comparing configs." + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class model_components(SuiteQuestion): + default_value: str = "defer_to_code" + options: str = "defer_to_code" + prompt: str = "Enter the model components for this model." + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ingest_background_pipeline(SuiteQuestion): + default_value: bool = False + prompt: str = "Run the SaveBackground task to ingest background files into R2D2?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class parser_options(SuiteQuestion): + default_value: list = mutable_field(['fgrep_residual_norm']) + options: list = mutable_field(['fgrep_residual_norm']) + prompt: str = "List the test types to run on the JEDI oops log." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class r2d2_experiment_id(SuiteQuestion): + default_value: str = "defer_to_code" + prompt: str = "What experiment_id should r2d2 reference for experiment?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class r2d2_server(SuiteQuestion): + default_value: str | None = None + prompt: str = ( + "Server/profile name in ~/.swell/r2d2_credentials.yaml " + "(e.g. 'gmao_server'). Leave empty if credentials are at the root level." + ) + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class r2d2_datastore(SuiteQuestion): + default_value: str | None = None + prompt: str = ( + "Datastore name passed to R2D2 fetch and store operations " + "(e.g. a Discover directory store or an S3 bucket store). " + "Run scripts/discover_r2d2_datastores.py to list available datastores. " + "Leave empty to let R2D2 pick the highest-priority writable datastore " + "for your compute host." + ) + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class runahead_limit(SuiteQuestion): + default_value: str = "P4" + prompt: str = ("Set the Cylc runahead limit: the maximum number of cycles " + "that may be active ahead of the current cycle " + "(e.g. P1: up to 1 cycle ahead, P3: up to 3 cycles ahead, default P4).") + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class saber_central_block(SuiteQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which saber central block do you want to use?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class saber_outer_block(SuiteQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which saber outer blocks do you want to use?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class skip_ensemble_hofx(SuiteQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Do you want to run localensembleda observer and solver together?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class skip_r2d2(SuiteQuestion): + default_value: bool = False + prompt: str = "Skip registering and storing results of this experiment in R2D2?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class start_cycle_point(SuiteQuestion): + default_value: str = "2023-10-10T00:00:00Z" + prompt: str = "What is the time of the first cycle (middle of the window)?" + data_type: DType = DType.ISO_DATETIME + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class window_type(SuiteQuestion): + default_value: str = "defer_to_model" + options: List[str] = mutable_field([ + "3D", + "4D" + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter the window type for this model." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- +# Task question defaults go here +# -------------------------------------------------------------------------------------------------- + +@dataclass +class analysis_variables(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What are the analysis variables?" + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class background_error_model(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which background error model do you want to use?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class background_experiment(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the name of the name of the experiment providing the backgrounds?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class background_frequency(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + depends: Dict = mutable_field({ + "window_type": "4D" + }) + prompt: str = "What is the frequency of the background files?" + data_type: DType = DType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class background_time_offset(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = ("How long before the middle of the analysis window did" + " the background providing forecast begin?") + data_type: DType = DType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ebkg_time_offset(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = ("How long before the middle of the analysis window did" + " the ensemble background providing forecast begin?") + data_type: DType = DType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class rst_experiment(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the name of the experiment providing the restart files in R2D2?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class rst_file_types(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What are the restart file types to fetch/store from R2D2?" + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class rst_store_interval(TaskQuestion): + default_value: str = None + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = ("After how many cycles should restart files be stored as real files " + "(not symlinks)? E.g. 28 means every 28th cycle (and multiples) stores " + "real files. Leave unset to always store as symlinks.") + data_type: DType = DType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class bufr_obs_classes(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What BUFR observation classes will be used?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class bundles(TaskQuestion): + default_value: List[str] = mutable_field([ + "fv3-jedi", + "soca", + "iodaconv", + "ufo" + ]) + options: List[str] = mutable_field([ + "fv3-jedi", + "soca", + "iodaconv", + "ufo", + "ioda", + "oops", + "saber" + ]) + depends: Dict = mutable_field({ + "jedi_build_method": "create" + }) + prompt: str = "Which JEDI bundles do you wish to build?" + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class check_for_obs(TaskQuestion): + default_value: bool = True + options: List[bool] = mutable_field([True, False]) + models: List[str] = mutable_field([ + 'all_models' + ]) + prompt: str = "Perform check for observations? Set to false for debugging purposes." + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class clean_patterns(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Provide a list of patterns that you wish to remove from the cycle directory." + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class comparison_log_type(TaskQuestion): + default_value: str = "variational" + options: List[str] = mutable_field([ + 'variational', + 'fgat', + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Provide the log naming convention (e.g. 'variational', 'fgat')." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class crtm_coeff_dir(TaskQuestion): + default_value: str = "defer_to_platform" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the CRTM coefficient files?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensemble_hofx_packets(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter number of packets in which ensemble observers should be computed." + data_type: DType = DType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensemble_hofx_strategy(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter hofx strategy." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensemble_num_members(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "How many members comprise the ensemble?" + data_type: DType = DType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class obs_pert_amplitude(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Enter obs perturbation amplitude for EDA:" + data_type: DType = DType.FLOAT + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensmean_only(TaskQuestion): + default_value: bool = False + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Calculate ensemble mean only?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensmeanvariance_only(TaskQuestion): + default_value: bool = False + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Calculate ensemble mean and variance only?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ensmeanvariance_spec(TaskQuestion): + default_value: List[Dict[str, str]] = field(default_factory=lambda: [{}]) + models: List[str] = mutable_field([ + "all_models" + ]) + + prompt: str = "Configure the ensemble mean and variance specifications:" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class diffstates_spec(TaskQuestion): + default_value: Dict[str, Any] = field(default_factory=dict) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Configure the diffstates specifications: [state1, state2]" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_geos_gcm_build_path(TaskQuestion): + default_value: str = "defer_to_platform" + depends: Dict = mutable_field({ + "geos_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing GEOS build directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_geos_gcm_source_path(TaskQuestion): + default_value: str = "defer_to_platform" + depends: Dict = mutable_field({ + "geos_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing GEOS source code directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_jedi_build_directory(TaskQuestion): + default_value: str = "defer_to_platform" + depends: Dict = mutable_field({ + "jedi_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing JEDI build directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_jedi_build_directory_pinned(TaskQuestion): + default_value: str = "defer_to_platform" + depends: Dict = mutable_field({ + "jedi_build_method": "use_pinned_existing" + }) + prompt: str = "What is the path to the existing pinned JEDI build directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_jedi_source_directory(TaskQuestion): + default_value: str = "defer_to_platform" + depends: Dict = mutable_field({ + "jedi_build_method": "use_existing" + }) + prompt: str = "What is the path to the existing JEDI source code directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_jedi_source_directory_pinned(TaskQuestion): + default_value: str = "defer_to_platform" + depends: Dict = mutable_field({ + "jedi_build_method": "use_pinned_existing" + }) + prompt: str = "What is the path to the existing pinned JEDI source code directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class existing_perllib_path(TaskQuestion): + default_value: str = 'defer_to_platform' + question_name: str = 'existing_perllib_path' + prompt: str = "Provide a path to an existing location for GMAO_perllib." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class gmao_perllib_tag(TaskQuestion): + default_value: str = 'g1.0.1' + question_name: str = 'gmao_perllib_tag' + prompt: str = "Specify the tag at which GMAO_perllib should be cloned." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class forecast_duration(TaskQuestion): + default_value: str = "PT12H" + prompt: str = "GEOS forecast duration" + data_type: DType = DType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class forecast_length(TaskQuestion): + default_value: str = "PT12H" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "Duration of the GEOS-CF forecast (ISO 8601 duration, e.g. PT12H)" + data_type: DType = DType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class forecast_output_frequency(TaskQuestion): + default_value: str = "PT1H" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "Frequency of forecast output files (ISO 8601 duration, e.g. PT1H)" + data_type: DType = DType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class generate_yaml_and_exit(TaskQuestion): + default_value: bool = False + prompt: str = "Generate JEDI executable YAML and exit?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_build_method(TaskQuestion): + default_value: str = "create" + options: List[str] = mutable_field([ + "use_existing", + "create" + ]) + prompt: str = "Do you want to use an existing GEOS build or create a new build?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_homdir(TaskQuestion): + default_value: str = "defer_to_platform" + prompt: str = ("What is the location for the HOME Directory (HOMDIR in gcm_run and " + "gcm_setup) that contains model settings and RC files?") + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_expdir_different(TaskQuestion): + default_value: str = False + options: List[bool] = mutable_field([ + True, + False + ]) + prompt: str = ("Is your GEOS EXPERIMENT Directory, where restarts and scratch is located, " + "different than your GEOS HOME Directory?") + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_expdir(TaskQuestion): + default_value: str = "/dev/null/" + depends: Dict = mutable_field({ + "geos_expdir_different": True + }) + prompt: str = ("What is the location for the EXPERIMENT Directory (to contain model " + "output and restart files), if it is different than your GEOS HOME " + "Directory?") + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_cf_install_dir(TaskQuestion): + default_value: str = "defer_to_platform" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the path to the GEOS-CF install directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_cf_run_dir(TaskQuestion): + default_value: str = "defer_to_platform" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the path to the GEOS-CF model run directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geosfp_exp(TaskQuestion): + default_value: str = "f5295_fp" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the GEOS FP experiment ID used for IAU analysis files?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geosfp_path(TaskQuestion): + default_value: str = "defer_to_platform" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the path to the GEOS FP archive?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_gcm_tag(TaskQuestion): + default_value: str = "v11.6.0" + depends: Dict = mutable_field({ + "geos_build_method": "create" + }) + prompt: str = "Which GEOS tag do you wish to clone?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_x_background_directory(TaskQuestion): + default_value: str = "/dev/null/" + options: List[str] = mutable_field([ + "/dev/null/", + "/discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x" + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the path to the GEOS X-backgrounds directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geos_x_ensemble_directory(TaskQuestion): + default_value: str = "/dev/null/" + options: List[str] = mutable_field([ + "/dev/null/", + "/gpfsm/dnb05/projects/p139/rtodling/archive/" + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the GEOS X-backgrounds directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geovals_experiment(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the name of the R2D2 experiment providing the GeoVaLs?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class geovals_provider(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the name of the R2D2 database providing the GeoVaLs?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class gradient_norm_reduction(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What value of gradient norm reduction for convergence?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class gsibec_configuration(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which GSIBEC climatological or hybrid?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class gsibec_nlats(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "How many number of latutides in GSIBEC grid?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class gsibec_nlons(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "How many number of longitudes in GSIBEC grid?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class horizontal_resolution(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the horizontal resolution for the forecast model and backgrounds?" + data_type: DType = DType.STRING + +# ------------------------------------------------------------------------------------------------ + +@dataclass +class dry_run(TaskQuestion): + default_value: bool = True + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Dry-run mode: preview what would be ingested before storing to R2D2" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class store_as_symlink(TaskQuestion): + default_value: bool = True + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Store background files as symlinks in R2D2 instead of copying them?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class obs_rc_path(TaskQuestion): + default_value: str = 'GEOS_mksi/ObsClass/obsys-nccs.rc' + question_name: str = 'obs_rc_path' + prompt: str = "Filepath to observing system rc file within experiment directory." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class obs_to_ingest(TaskQuestion): + default_value: list = mutable_field([]) + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which observations do you want to ingest to R2D2?" + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class obs_to_download(TaskQuestion): + default_value: list = mutable_field([]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which observations do you want to download from remote servers?" + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class converter_path(TaskQuestion): + default_value: str = "" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = ("Path to directory containing ioda-converter scripts" + " (leave blank to use jedi_bin)") + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class initial_restarts_method(TaskQuestion): + default_value: str = "defer_to_platform" + options: List[str] = mutable_field([ + "geos_expdir", + "r2d2", + "hotstart", + ]) + prompt: str = "How should initial GEOS restarts be obtained?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ioda_locations_not_in_r2d2(TaskQuestion): + default_value: str = "defer_to_platform" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ( + "Provide a path that contains observation files not in r2d2.") + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class iau(TaskQuestion): + default_value: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "Use Incremental Analysis Update (IAU) in the GEOS-CF forecast?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class inc_template(TaskQuestion): + default_value: str = "defer_to_platform" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the path to the GEOS-CF increment template NetCDF file?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class jedi_build_method(TaskQuestion): + default_value: str = "use_existing" + options: List[str] = mutable_field([ + "use_existing", + "use_pinned_existing", + "create", + "pinned_create" + ]) + prompt: str = "Do you want to use an existing JEDI build or create a new build?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class jedi_forecast_model(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + depends: Dict = mutable_field({ + "window_type": "4D" + }) + prompt: str = "What forecast model should be used within JEDI for 4D window propagation?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_inflation_mult(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Specify the multiplicative prior inflation coefficient (0 inf]." + data_type: DType = DType.FLOAT + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_inflation_rtpp(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Specify the Relaxation To Prior Perturbation (RTPP) coefficient (0 1]." + data_type: DType = DType.FLOAT + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_inflation_rtps(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Specify the Relaxation To Prior Spread (RTPS) coefficient (0 1]." + data_type: DType = DType.FLOAT + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_save_posterior_ensemble(TaskQuestion): + default_value: bool = False + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Save the posterior ensemble members?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): + default_value: bool = False + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Save the posterior ensemble member increments?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_save_posterior_mean(TaskQuestion): + default_value: bool = False + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Save the posterior ensemble mean?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_save_posterior_mean_increment(TaskQuestion): + default_value: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Save the posterior ensemble mean increment?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_solver(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which local ensemble solver type should be implemented?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class local_ensemble_use_linear_observer(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Use linear observer in local ensemble solver?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class minimizer(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which data assimilation minimizer do you wish to use?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class mom6_iau(TaskQuestion): + default_value: str = "defer_to_model" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_marine", + ]) + prompt: str = "Do you wish to use IAU for MOM6?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class mom6_iau_nhours(TaskQuestion): + default_value: str = "defer_to_model" + options: List[str] = mutable_field([ + 'PT3H', + 'PT12H' + ]) + depends: dict = mutable_field({'mom6_iau': True}) + models: List[str] = mutable_field([ + "geos_marine", + ]) + prompt: str = "What is the IAU length (ODA_INCUPD_NHOURS) for MOM6?" + data_type: DType = DType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class ncdiag_experiments(TaskQuestion): + default_value: str = "defer_to_model" + options: List[str] = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which previously run experiments do you wish to use for the NCdiag?" + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class npx_proc(TaskQuestion): + default_value: str = "defer_to_model" + + models: List[str] = mutable_field([ + "geos_atmosphere", + "geos_cf" + ]) + prompt: str = "What number of processors do you wish to use in the x-direction?" + data_type: DType = DType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class npy_proc(TaskQuestion): + default_value: str = "defer_to_model" + + models: List[str] = mutable_field([ + "geos_atmosphere", + "geos_cf" + ]) + prompt: str = "What number of processors do you wish to use in the y-direction?" + data_type: DType = DType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class npx(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the number of grid points in the x-direction on each cube face?" + data_type: DType = DType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class npy(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_cf" + ]) + prompt: str = "What is the number of grid points in the y-direction on each cube face?" + data_type: DType = DType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class number_of_iterations(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = ( + "What number of iterations do you wish to use for each outer loop?" + " Provide a list of integers the same length as the number of outer loops.") + data_type: DType = DType.INTEGER_LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class obs_experiment(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the database providing the observations?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class observation_providers(TaskQuestion): + default_value: Dict[str, str] = mutable_field({}) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Map observation names to their R2D2 providers." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class obs_thinning_rej_fraction(TaskQuestion): + default_value: float = 0.75 + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the rejection fraction for obs thinning?" + data_type: DType = DType.FLOAT + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class observations(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Which observations do you want to include?" + data_type: DType = DType.LIST + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class observing_system_records_mksi_path(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the GSI formatted observing system records?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class observing_system_records_mksi_path_tag(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the GSI formatted observing system records tag?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class observing_system_records_path(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to the Swell formatted observing system records?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class path_to_ensemble(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_marine" + ]) + prompt: str = "What is the path to where ensemble members are stored?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class path_to_geos_adas_background(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ( + "What is the path for the GEOSadas cubed sphere backgrounds?") + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class path_to_gsi_bc_coefficients(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the location where GSI bias correction files can be found?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class path_to_gsi_nc_diags(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the path to where the GSI ncdiags are stored?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class perhost(TaskQuestion): + default_value: str = None + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the number of processors per host?" + data_type: DType = DType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class produce_geovals(TaskQuestion): + default_value: str = "defer_to_model" + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("When running the ncdiag to ioda converted do you " + "want to produce GeoVaLs files?") + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class publish_directory(TaskQuestion): + default_value: str = None + prompt: str = "Provide an external directory to publish relevant results to." + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class cache_fetch(TaskQuestion): + default_value: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + prompt: str = "Use cached observation files if they already exist?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class fetch_obs_from_public_s3(TaskQuestion): + default_value: bool = False + options: List[bool] = mutable_field([ + True, + False + ]) + prompt: str = "Fetch observations directly from a public S3 bucket if they are available?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class save_geovals(TaskQuestion): + default_value: bool = False + options: List[bool] = mutable_field([ + True, + False + ]) + prompt: str = "When running hofx do you want to output the GeoVaLs?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class single_observations(TaskQuestion): + default_value: bool = False + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Is it a single-observation test?" + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class swell_static_files(TaskQuestion): + default_value: str = "defer_to_platform" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the path to the Swell Static files directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class swell_static_files_user(TaskQuestion): + default_value: str = "None" + prompt: str = "What is the path to the user provided Swell Static Files directory?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class total_processors(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_marine", + ]) + prompt: str = "What is the number of processors for JEDI?" + data_type: DType = DType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_apply_log_transform(TaskQuestion): + default_value: bool = True + options: List[bool] = mutable_field([ + True, + False + ]) + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("Should a log (base 10) transformation be applied " + "to vertical coordinate when " + "constructing vertical localization?") + data_type: DType = DType.BOOLEAN + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_function(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which localization scheme should be applied in the vertical?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_ioda_vertical_coord(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "Which coordinate should be used in constructing vertical localization?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_ioda_vertical_coord_group(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("Which vertical coordinate group should be used " + "in constructing vertical localization?") + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_lengthscale(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = "What is the length scale for vertical covariance localization?" + data_type: DType = DType.INTEGER + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_localization_method(TaskQuestion): + default_value: str = "defer_to_model" + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "geos_atmosphere" + ]) + prompt: str = ("What localization scheme should be applied in " + "constructing a vertical localization?") + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class vertical_resolution(TaskQuestion): + default_value: str = "defer_to_model" + + options: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the vertical resolution for the forecast model and background?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class window_length(TaskQuestion): + default_value: str = "defer_to_model" + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "What is the duration for the data assimilation window?" + data_type: DType = DType.ISO_DURATION + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class window_type(TaskQuestion): + default_value: str = "defer_to_model" + options: List[str] = mutable_field([ + "3D", + "4D" + ]) + models: List[str] = mutable_field([ + "all_models" + ]) + prompt: str = "Do you want to use a 3D or 4D (including FGAT) window?" + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- + +@dataclass +class background_source_path(TaskQuestion): + default_value: str = ( + '/css/gmao/geos-cf/NRTv2/priv/ana/Y%Y/M%m/D%d/' + 'GEOS.cf.ana.jdi_inst_1hr_glo_C360x360x6_v72.%Y%m%d_%H%Mz.R0.nc4' + ) + models: List[str] = mutable_field(['geos_cf']) + prompt: str = ("Path template for background files. Uses Python strftime format codes, " + "e.g. Y%Y/M%m/D%d gives Y2025/M10/D02 and %Y%m%d_%H%Mz gives " + "20251002_0900z.") + data_type: DType = DType.STRING + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 043975437..d9dd7ffe5 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -16,6 +16,7 @@ from swell.tasks.base.task_base import taskBase from swell.utilities.pinned_versions.check_hashes import check_hashes from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs +from swell.configuration.question_defaults import * # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 649e6dd0a..3da9d27a2 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -13,7 +13,7 @@ from swell.tasks.task_questions import TaskQuestions as task_questions from swell.utilities.logger import Logger from swell.suites.all_suites import AllSuites - +from swell.utilities.swell_questions import SwellQuestion # -------------------------------------------------------------------------------------------------- # @package configuration @@ -86,13 +86,9 @@ def __init__(self, input_file: str, logger: Logger, task_name: str, model: str) else: model_config = {} + self.all_model_configs = {} if 'models' in experiment_dict.keys(): - - # Add a dictionary tracking all model-dependent values - setattr(self, f'__all_model_configs__', experiment_dict['models']) - - # Add a method to access the model-dependent dictionary - setattr(self, f'all_model_configs', self.get('all_model_configs')) + self.all_model_configs = experiment_dict['models'] # Remove the model specific part from the full config if 'models' in experiment_dict.keys(): @@ -109,84 +105,56 @@ def __init__(self, input_file: str, logger: Logger, task_name: str, model: str) # supposed to act upon. experiment_dict.update(model_config) + self.experiment_dict = experiment_dict + # Step 2: create variables in the object with the keys/values in the config # ------------------------------------------------------------------------- # Check for suite questions suite_questions = AllSuites.get_config( self.__suite_to_run__).get_all_question_names('suite') - question_list = [] + self.question_list = [] # Add suite questions if they aren't already set for question in suite_questions: - if not self.has_attr(question): - question_list.append(question) + if question not in suite_questions: + self.question_list.append(question) # Find the questions associated with the task if task_name in task_questions.get_all(): - question_list.extend(task_questions[task_name].value.get_all_question_names()) - - # Loop through the experiment dictionary - for exp_key, exp_val in experiment_dict.items(): - - # Assign the value if needed by the task - if exp_key in question_list: - - # Set as a variable to config - setattr(self, f'__{exp_key}__', exp_val) - - # Add a get method to access variable - setattr(self, f'{exp_key}', self.get(exp_key)) + self.question_list.extend(task_questions[task_name].value.get_all_question_names()) # ---------------------------------------------------------------------------------------------- - def get(self, experiment_key: str) -> Callable: + def resolve(self, question: SwellQuestion, default='LrZRExPGcQ'): + name = question.question_name - def getter(default='None'): - return getattr(self, f'__{experiment_key}__') - return getter - - # ---------------------------------------------------------------------------------------------- + if name in self.question_list: + default = self.experiment_dict[name] + elif name in self.experiment_dict: + raise KeyError(f'Value {name} is present in config but this task has not been assigned' + ' it in `task_questions.py`') - # Implementation to check if object has attribute that works with custom __get_attr__ - def has_attr(self, name: str): - attr = getattr(self, f'__{name}__') - - if hasattr(attr, '__name__') and attr.__name__ == 'variable_not_found': - return False - - return True - - # ---------------------------------------------------------------------------------------------- - - # Implementation of __getattr__ to ensure there is no crash when a task requests a variable that - # does not exist. This is valid so long as the task provides a default value. - def __getattr__(self, name: str) -> Callable: - def variable_not_found(default='LrZRExPGcQ'): - if default == 'LrZRExPGcQ': - self.__logger__.abort(f'In config class, trying to get variable \'{name}\' but ' + - f'this variable was not created. Ensure that the variable ' + - f'is in the experiment configuration and specified in ' + - f'the question list for the task (task_questions.py).') - else: - return default - return variable_not_found + if default == 'LrZRExPGcQ': + raise KeyError(f'Trying to reference value {name} in config but this key does not ' + 'exist and no default has been provided') + + return default # ---------------------------------------------------------------------------------------------- def get_key_for_model(self, name: str, model: str, default='LrZRExPGcQ'): """ Access keys in any model component. Provide a default to avoid errors. """ - if hasattr(self, 'all_model_configs'): - all_configs = self.all_model_configs() - - if model in all_configs.keys() and name in all_configs[model].keys(): - default = all_configs[model][name] + try: + default = self.all_model_configs[model][name] + except KeyError: + pass if default == 'LrZRExPGcQ': self.__logger__.abort(f"In config class, trying to reference value '{name}'" + f" for model '{model}', but config key does not exist and no" + f" default has been provided.") return default - +# self.config.resolve(skip_ensemble_hofx, True) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index 1d0e2f45c..331ab1d0b 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -12,6 +12,7 @@ from dataclasses import dataclass, asdict, field from typing import List, Optional, Self, Union, Literal from enum import Enum +from collections.abc import Mapping from swell.utilities.datetime_util import is_datetime, is_duration from swell.swell_path import get_swell_path @@ -19,69 +20,52 @@ # -------------------------------------------------------------------------------------------------- -class WidgetType(Enum): +class DataType(Enum): STRING = "string" - STRING_CHECK_LIST = "string-check-list" - STRING_DROP_LIST = "string-drop-list" BOOLEAN = "boolean" ISO_DURATION = "iso-duration" ISO_DATETIME = "iso-datetime" INTEGER = "integer" INTEGER_LIST = "integer-list" - FILE_CHECK_LIST = "file-check-list" FLOAT = "float" + LIST = "list" + MAPPING = "mapping" - @property - def is_drop_list(self) -> bool: - return 'drop-list' in self.value - - @property - def is_check_list(self) -> bool: - return 'check-list' in self.value - - @property - def base_type(self) -> type: - """ Get the base type of the value based on the widget type. """ - if 'string' in self.value: - return str - if 'boolean' in self.value: - return bool - if 'integer' in self.value: - return int - if 'float' in self.value: - return float - if 'iso-' in self.value: - return str - - def validate_value(self, value) -> bool: - """ Validate that the value matches the type and format of the widget type. """ - base_type = self.base_type() - - # Check that the answer fits the base type - if base_type == float: - try: - float(value) - except ValueError: - return False - elif base_type == int: - try: - int(value) - except ValueError: - return False - else: - try: - str(value) - except ValueError: - return False + def is_type(self, value) -> bool: + """ Validate that the value matches the data type. """ - # If the widget is a datetime, ensure it is in the right format - if self == WidgetType.ISO_DATETIME: + # Ensure value is in ISO datetime format + if self == DataType.ISO_DATETIME: return is_datetime(value) - # Ensure the value is a duration - if self == WidgetType.ISO_DURATION: + # Ensure the value is in ISO duration format + if self == DataType.ISO_DURATION: return is_duration(value) + if self == DataType.STRING: + return isinstance(value, str) + + if self == DataType.BOOLEAN: + return isinstance(value, bool) + + if self == DataType.INTEGER: + return isinstance(value, int) + + if self == DataType.INTEGER_LIST: + if isinstance(value, list): + return all([isinstance(item, int) for item in value]) + else: + return False + + if self == DataType.FLOAT: + return isinstance(value, float) + + if self == DataType.LIST: + return isinstance(value, list) + + if self == DataType.MAPPING: + return isinstance(value, Mapping) + return True @@ -91,13 +75,16 @@ def validate_value(self, value) -> bool: class SwellQuestion: """Basic dataclass for defining Swell questions for suites and tasks""" default_value: str - question_name: str - widget_type: WidgetType + data_type: DataType prompt: str + question_name: str | None = None question_type: str = None - ask_question: bool = False options: Optional[str] = None + def __post_init__(self) -> None: + if self.question_name is None: + self.question_name = self.__class__.__name__ + # -------------------------------------------------------------------------------------------------- From b1f9e5bbceb68029a4b0dfd228f30c1bf7f8e322 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 24 Aug 2026 13:43:29 -0400 Subject: [PATCH 02/15] Update suite configs --- src/swell/suites/3dfgat_atmos/suite_config.py | 2 +- .../suites/3dfgat_marine_cycle/suite_config.py | 2 +- src/swell/suites/3dvar_atmos/suite_config.py | 2 +- src/swell/suites/3dvar_cf/suite_config.py | 2 +- src/swell/suites/3dvar_cf_cycle/suite_config.py | 2 +- src/swell/suites/3dvar_marine/suite_config.py | 2 +- .../suites/3dvar_marine_cycle/suite_config.py | 2 +- src/swell/suites/build_geos/suite_config.py | 1 + src/swell/suites/build_jedi/suite_config.py | 1 + src/swell/suites/compare/suite_config.py | 2 +- src/swell/suites/convert_bufr/suite_config.py | 2 +- src/swell/suites/convert_ncdiags/suite_config.py | 2 +- src/swell/suites/eda_atmos/suite_config.py | 2 +- src/swell/suites/eva_capabilities/suite_config.py | 2 +- .../suites/forecast_coupled_geos/suite_config.py | 2 +- src/swell/suites/geosadas/suite_config.py | 2 +- src/swell/suites/hofx/suite_config.py | 2 +- src/swell/suites/hofx_cf/suite_config.py | 2 +- src/swell/suites/letkf_marine/suite_config.py | 2 +- src/swell/suites/localensembleda/suite_config.py | 2 +- src/swell/suites/r2d2_ingest/suite_config.py | 2 +- src/swell/suites/ufo_testing/suite_config.py | 2 +- src/swell/tasks/clone_jedi.py | 14 +++++++------- 23 files changed, 29 insertions(+), 27 deletions(-) diff --git a/src/swell/suites/3dfgat_atmos/suite_config.py b/src/swell/suites/3dfgat_atmos/suite_config.py index 58122cbf9..12382d5b5 100644 --- a/src/swell/suites/3dfgat_atmos/suite_config.py +++ b/src/swell/suites/3dfgat_atmos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dfgat_marine_cycle/suite_config.py b/src/swell/suites/3dfgat_marine_cycle/suite_config.py index da954da5c..28630f9a2 100644 --- a/src/swell/suites/3dfgat_marine_cycle/suite_config.py +++ b/src/swell/suites/3dfgat_marine_cycle/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar_atmos/suite_config.py b/src/swell/suites/3dvar_atmos/suite_config.py index 6dbe4cb6f..212aa07b0 100644 --- a/src/swell/suites/3dvar_atmos/suite_config.py +++ b/src/swell/suites/3dvar_atmos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar_cf/suite_config.py b/src/swell/suites/3dvar_cf/suite_config.py index 93dd6118c..78d975d0c 100644 --- a/src/swell/suites/3dvar_cf/suite_config.py +++ b/src/swell/suites/3dvar_cf/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar_cf_cycle/suite_config.py b/src/swell/suites/3dvar_cf_cycle/suite_config.py index 7cc828346..b4d023908 100644 --- a/src/swell/suites/3dvar_cf_cycle/suite_config.py +++ b/src/swell/suites/3dvar_cf_cycle/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar_marine/suite_config.py b/src/swell/suites/3dvar_marine/suite_config.py index 002432e0c..d572c0595 100644 --- a/src/swell/suites/3dvar_marine/suite_config.py +++ b/src/swell/suites/3dvar_marine/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/3dvar_marine_cycle/suite_config.py b/src/swell/suites/3dvar_marine_cycle/suite_config.py index 0a6967502..3ecff5134 100644 --- a/src/swell/suites/3dvar_marine_cycle/suite_config.py +++ b/src/swell/suites/3dvar_marine_cycle/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/build_geos/suite_config.py b/src/swell/suites/build_geos/suite_config.py index a61136668..d7a420730 100644 --- a/src/swell/suites/build_geos/suite_config.py +++ b/src/swell/suites/build_geos/suite_config.py @@ -10,6 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.suites.suite_questions import SuiteQuestions as sq +# import swell.configuration.question_defaults as qd from enum import Enum diff --git a/src/swell/suites/build_jedi/suite_config.py b/src/swell/suites/build_jedi/suite_config.py index 4da92ab6b..b3d1b5d15 100644 --- a/src/swell/suites/build_jedi/suite_config.py +++ b/src/swell/suites/build_jedi/suite_config.py @@ -10,6 +10,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList from swell.suites.suite_questions import SuiteQuestions as sq +# import swell.configuration.question_defaults as qd from enum import Enum diff --git a/src/swell/suites/compare/suite_config.py b/src/swell/suites/compare/suite_config.py index 7847c366a..404311420 100644 --- a/src/swell/suites/compare/suite_config.py +++ b/src/swell/suites/compare/suite_config.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- from swell.utilities.swell_questions import QuestionContainer, QuestionList, WidgetType -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/convert_bufr/suite_config.py b/src/swell/suites/convert_bufr/suite_config.py index 27e9d3e79..8587340b8 100644 --- a/src/swell/suites/convert_bufr/suite_config.py +++ b/src/swell/suites/convert_bufr/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/convert_ncdiags/suite_config.py b/src/swell/suites/convert_ncdiags/suite_config.py index 268312c49..11463ad61 100644 --- a/src/swell/suites/convert_ncdiags/suite_config.py +++ b/src/swell/suites/convert_ncdiags/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/eda_atmos/suite_config.py b/src/swell/suites/eda_atmos/suite_config.py index a632d8b86..d84bd1b82 100644 --- a/src/swell/suites/eda_atmos/suite_config.py +++ b/src/swell/suites/eda_atmos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/eva_capabilities/suite_config.py b/src/swell/suites/eva_capabilities/suite_config.py index f00b13ea3..6a4425a53 100644 --- a/src/swell/suites/eva_capabilities/suite_config.py +++ b/src/swell/suites/eva_capabilities/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/forecast_coupled_geos/suite_config.py b/src/swell/suites/forecast_coupled_geos/suite_config.py index 8d2ad5e48..540c44b30 100644 --- a/src/swell/suites/forecast_coupled_geos/suite_config.py +++ b/src/swell/suites/forecast_coupled_geos/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/geosadas/suite_config.py b/src/swell/suites/geosadas/suite_config.py index 32c1e75b8..2fcaa1a97 100644 --- a/src/swell/suites/geosadas/suite_config.py +++ b/src/swell/suites/geosadas/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/hofx/suite_config.py b/src/swell/suites/hofx/suite_config.py index ad48d0054..25a052d98 100644 --- a/src/swell/suites/hofx/suite_config.py +++ b/src/swell/suites/hofx/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/hofx_cf/suite_config.py b/src/swell/suites/hofx_cf/suite_config.py index 9fa8ad6d1..a7dc928bf 100644 --- a/src/swell/suites/hofx_cf/suite_config.py +++ b/src/swell/suites/hofx_cf/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/letkf_marine/suite_config.py b/src/swell/suites/letkf_marine/suite_config.py index 1aec9e4fd..7aff1c391 100644 --- a/src/swell/suites/letkf_marine/suite_config.py +++ b/src/swell/suites/letkf_marine/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/localensembleda/suite_config.py b/src/swell/suites/localensembleda/suite_config.py index e80f67cb0..4187e1c26 100644 --- a/src/swell/suites/localensembleda/suite_config.py +++ b/src/swell/suites/localensembleda/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/r2d2_ingest/suite_config.py b/src/swell/suites/r2d2_ingest/suite_config.py index 95bb6eedf..7764eb9f9 100644 --- a/src/swell/suites/r2d2_ingest/suite_config.py +++ b/src/swell/suites/r2d2_ingest/suite_config.py @@ -6,7 +6,7 @@ """ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/suites/ufo_testing/suite_config.py b/src/swell/suites/ufo_testing/suite_config.py index 5482a39cf..fd64f5004 100644 --- a/src/swell/suites/ufo_testing/suite_config.py +++ b/src/swell/suites/ufo_testing/suite_config.py @@ -9,7 +9,7 @@ from swell.utilities.swell_questions import QuestionContainer, QuestionList -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq from enum import Enum diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index d9dd7ffe5..b3f875927 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -37,17 +37,17 @@ def execute(self) -> None: # Choice to link to existing build or build JEDI using jedi_bundle # ---------------------------------------------------------------- - if self.config.jedi_build_method() == 'use_existing': + if self.config.resolve(jedi_build_method) == 'use_existing': # Link the source code directory link_path(self.config.existing_jedi_source_directory(), jedi_bundle_source_path) - elif self.config.jedi_build_method() == 'use_pinned_existing': + elif self.config.resolve(jedi_build_method) == 'use_pinned_existing': # Check hashes before proceeding - check_hashes(self.config.existing_jedi_source_directory_pinned(), self.logger) + check_hashes(self.config.resolve(existing_jedi_source_directory_pinned), self.logger) # Link the pinned source code directory - link_path(self.config.existing_jedi_source_directory_pinned(), jedi_bundle_source_path) + link_path(self.config.resolve(existing_jedi_source_directory_pinned), jedi_bundle_source_path) - elif self.config.jedi_build_method() in ('create', 'pinned_create'): + elif self.config.resolve(jedi_build_method) in ('create', 'pinned_create'): # Determine which bundles need to be build model_components = self.get_model_components() if model_components is not None: @@ -64,11 +64,11 @@ def execute(self) -> None: # Determine whether to use pinned versions or not use_pinned = False - if self.config.jedi_build_method() == 'pinned_create': + if self.config.resolve(jedi_build_method) == 'pinned_create': use_pinned = True # Generate the build dictionary - jedi_bundle_dict = set_jedi_bundle_config(self.config.bundles(bundles), + jedi_bundle_dict = set_jedi_bundle_config(self.config.resolve(bundles, default=bundles), jedi_bundle_source_path, jedi_bundle_build_path, self.platform(), From a7952d45a0c7a96c8d00abe7a955f5368fc59d8b Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Mon, 24 Aug 2026 16:31:09 -0400 Subject: [PATCH 03/15] Fixes --- .../prepare_config_and_suite.py | 1 - .../question_and_answer_cli.py | 102 ------------------ src/swell/suites/compare/suite_config.py | 8 +- src/swell/suites/suite_questions.py | 2 +- src/swell/tasks/clone_jedi.py | 2 +- src/swell/tasks/task_questions.py | 2 +- src/swell/utilities/config.py | 2 +- 7 files changed, 8 insertions(+), 111 deletions(-) delete mode 100644 src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py diff --git a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py index 69987bf5f..d6398a04a 100644 --- a/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py +++ b/src/swell/deployment/prepare_config_and_suite/prepare_config_and_suite.py @@ -15,7 +15,6 @@ from typing import Tuple, Optional from swell.swell_path import get_swell_path -from swell.deployment.prepare_config_and_suite.question_and_answer_cli import GetAnswerCli from swell.deployment.prepare_config_and_suite.question_and_answer_defaults import GetAnswerDefaults from swell.utilities.dictionary import dict_get from swell.utilities.logger import Logger diff --git a/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py b/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py deleted file mode 100644 index bc6075020..000000000 --- a/src/swell/deployment/prepare_config_and_suite/question_and_answer_cli.py +++ /dev/null @@ -1,102 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import questionary -from questionary import Choice -from typing import Optional - -from swell.utilities.logger import Logger -from swell.utilities.swell_questions import WidgetType - - -# -------------------------------------------------------------------------------------------------- - - -class GetAnswerCli: - - def get_answer(self, logger: Logger, key: str, val: dict, model: Optional[str] = None): - prompt = val['prompt'] - default = val['default_value'] - widget_type = val['widget_type'] - options = val['options'] - - if model is not None: - prompt = f'[{model}] {prompt}' - - print('\n') - if widget_type.is_drop_list: - answer = self.make_drop_widget(prompt, default, options, widget_type) - elif widget_type.is_check_list: - answer = self.make_check_widget(prompt, default, options, widget_type) - elif widget_type == WidgetType.BOOLEAN: - answer = self.make_boolean_widget(prompt, default) - - else: - answer = self.make_generic_widget(prompt, default, options, widget_type) - - if answer in ['', []] and widget_type != WidgetType.FILE_CHECK_LIST: - answer = default - - if answer == 'EXIT' or answer is None: - logger.abort('Exiting Swell prepper...') - - return answer - - # -------------------------------------------------------------------------------------------------- - - def make_generic_widget(self, prompt: str, default, options: list, widget_type: WidgetType): - default = str(default) - - answer = questionary.text(f"{prompt} [{default}]", validate=lambda text: True - if widget_type.validate_value(text) - else f'Please enter a value of type: {widget_type.value}', - default=default).ask() - - return answer - - # -------------------------------------------------------------------------------------------------- - - def make_drop_widget(self, prompt: str, default, options: list, widget_type: WidgetType): - default = str(default) - choices = [str(x) for x in options] - - answer = questionary.select(prompt, choices=choices, default=default).ask() - - return answer - - # -------------------------------------------------------------------------------------------------- - - def make_check_widget(self, prompt: str, default, options: list, widget_type: WidgetType): - choices = options.copy() - - if isinstance(default, list): - for i, c in enumerate(choices): - if c in default: - choices[i] = Choice(c, checked=True) - else: - choices[i] = Choice(c, checked=False) - default = None - - answer = questionary.checkbox(prompt, choices=choices, default=default, - validate=lambda text: True - if (widget_type.validate_value(text) and text != []) - else 'Select at least one option').ask() - - return answer - - # -------------------------------------------------------------------------------------------------- - - def make_boolean_widget(self, prompt: str, default: bool): - - answer = questionary.confirm(prompt, default=default, auto_enter=False).ask() - - return answer - - # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/suites/compare/suite_config.py b/src/swell/suites/compare/suite_config.py index 404311420..2763087f8 100644 --- a/src/swell/suites/compare/suite_config.py +++ b/src/swell/suites/compare/suite_config.py @@ -7,7 +7,7 @@ # # -------------------------------------------------------------------------------------------------- -from swell.utilities.swell_questions import QuestionContainer, QuestionList, WidgetType +from swell.utilities.swell_questions import QuestionContainer, QuestionList import swell.configuration.question_defaults as qd from swell.suites.suite_questions import SuiteQuestions as sq @@ -25,9 +25,9 @@ class SuiteConfig(QuestionContainer, Enum): questions=[ sq.all_suites, qd.comparison_experiment_paths(), - qd.start_cycle_point(default_value=None, widget_type=WidgetType.STRING), - qd.final_cycle_point(default_value=None, widget_type=WidgetType.STRING), - qd.cycle_times(default_value=[None], widget_type=WidgetType.STRING_CHECK_LIST), + qd.start_cycle_point(default_value=None), + qd.final_cycle_point(default_value=None), + qd.cycle_times(default_value=[None]), qd.model_components(), qd.runahead_limit(), ] diff --git a/src/swell/suites/suite_questions.py b/src/swell/suites/suite_questions.py index 31a58acdc..8ef4c1a7f 100644 --- a/src/swell/suites/suite_questions.py +++ b/src/swell/suites/suite_questions.py @@ -11,7 +11,7 @@ from enum import Enum from swell.utilities.swell_questions import QuestionList, QuestionContainer -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index b3f875927..e1cee67c9 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -39,7 +39,7 @@ def execute(self) -> None: # ---------------------------------------------------------------- if self.config.resolve(jedi_build_method) == 'use_existing': # Link the source code directory - link_path(self.config.existing_jedi_source_directory(), jedi_bundle_source_path) + link_path(self.config.resolve(existing_jedi_source_directory), jedi_bundle_source_path) elif self.config.resolve(jedi_build_method) == 'use_pinned_existing': # Check hashes before proceeding diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 71fca1608..48c5e61ad 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -12,7 +12,7 @@ from enum import Enum from swell.utilities.swell_questions import QuestionList, QuestionContainer -from swell.utilities.question_defaults import QuestionDefaults as qd +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 3da9d27a2..4f7d457f7 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -127,7 +127,7 @@ def __init__(self, input_file: str, logger: Logger, task_name: str, model: str) # ---------------------------------------------------------------------------------------------- def resolve(self, question: SwellQuestion, default='LrZRExPGcQ'): - name = question.question_name + name = question().question_name if name in self.question_list: default = self.experiment_dict[name] From 346c97eb8d107581365e4524841ba5d67a8416b7 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 25 Aug 2026 13:31:39 -0400 Subject: [PATCH 04/15] Updates for experiment creation --- .../question_and_answer_defaults.py | 6 +- src/swell/utilities/config.py | 9 +- src/swell/utilities/question_defaults.py | 1861 ----------------- src/swell/utilities/swell_questions.py | 1 + 4 files changed, 12 insertions(+), 1865 deletions(-) delete mode 100644 src/swell/utilities/question_defaults.py diff --git a/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py b/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py index f600c4198..e70ae4872 100644 --- a/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py +++ b/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py @@ -18,11 +18,11 @@ class GetAnswerDefaults: def get_answer(self, logger: Logger, key: str, val: dict, model: Optional[str] = None) -> Union[int, float, str]: default = val['default_value'] - widget_type = val['widget_type'] + data_type = val['data_type'] - if not widget_type.validate_value(default): + if not data_type.is_type(default): logger.abort(f'Default value for {key}, {default}, does not conform to type ' - f'{widget_type.base_type.__name__}, check the override file or ' + f'{data_type.base_type.__name__}, check the override file or ' 'suite configuration.') return default diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 4f7d457f7..4ed3af6ab 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -127,7 +127,8 @@ def __init__(self, input_file: str, logger: Logger, task_name: str, model: str) # ---------------------------------------------------------------------------------------------- def resolve(self, question: SwellQuestion, default='LrZRExPGcQ'): - name = question().question_name + question_obj = question() + name = question_obj.question_name if name in self.question_list: default = self.experiment_dict[name] @@ -139,6 +140,12 @@ def resolve(self, question: SwellQuestion, default='LrZRExPGcQ'): raise KeyError(f'Trying to reference value {name} in config but this key does not ' 'exist and no default has been provided') + data_type = question_obj.data_type + + if not data_type.is_type(default): + self.__logger__.warning(f'Warning: Experiment key {name} does not conform to expected' + f' type <{data_type.value}>.') + return default # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py deleted file mode 100644 index f05e59afb..000000000 --- a/src/swell/utilities/question_defaults.py +++ /dev/null @@ -1,1861 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -from dataclasses import dataclass, field -from typing import List, Dict, Any - -from swell.utilities.swell_questions import SuiteQuestion, TaskQuestion -from swell.utilities.swell_questions import WidgetType as WType -from swell.utilities.dataclass_utils import mutable_field - - -# -------------------------------------------------------------------------------------------------- - -class QuestionDefaults(): - - # -------------------------------------------------------------------------------------------------- - # Suite question defaults go here - # -------------------------------------------------------------------------------------------------- - - @dataclass - class comparison_experiment_paths(SuiteQuestion): - default_value: list = mutable_field([]) - question_name: str = "comparison_experiment_paths" - ask_question: bool = True - prompt: str = "Provide paths to two experiments to run comparison tests on." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class cycle_times(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "cycle_times" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the cycle times for this model." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class cycling_varbc(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "cycling_varbc" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Do you want to use cycling VarBC option?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_packets(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_packets" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the number of ensemble packets." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_strategy(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_strategy" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the ensemble hofx strategy." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class experiment_id(SuiteQuestion): - default_value: str = "defer_to_code" - question_name: str = "experiment_id" - ask_question: bool = True - prompt: str = "What is the experiment id?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class experiment_root(SuiteQuestion): - default_value: str = "defer_to_platform" - question_name: str = "experiment_root" - ask_question: bool = True - prompt: str = ("What is the experiment root (the directory where the " - "experiment will be stored)?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class final_cycle_point(SuiteQuestion): - default_value: str = "2023-10-10T06:00:00Z" - question_name: str = "final_cycle_point" - ask_question: bool = True - prompt: str = "What is the time of the final cycle (middle of the window)?" - widget_type: WType = WType.ISO_DATETIME - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class marine_models(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "marine_models" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_marine" - ]) - prompt: str = "Select the active SOCA models for this model." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class mock_experiment(SuiteQuestion): - default_value: bool = False - question_name: str = "mock_experiment" - ask_question: bool = False - prompt: str = "Dry-run option for comparing configs." - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class model_components(SuiteQuestion): - default_value: str = "defer_to_code" - question_name: str = "model_components" - ask_question: bool = True - options: str = "defer_to_code" - prompt: str = "Enter the model components for this model." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class parser_options(SuiteQuestion): - default_value: list = mutable_field(['fgrep_residual_norm']) - question_name: str = "parser_options" - ask_question: bool = True - options: list = mutable_field(['fgrep_residual_norm']) - prompt: str = "List the test types to run on the JEDI oops log." - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class r2d2_experiment_id(SuiteQuestion): - default_value: str = "defer_to_code" - question_name: str = "r2d2_experiment_id" - prompt: str = "What experiment_id should r2d2 reference for experiment?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class r2d2_server(SuiteQuestion): - default_value: str | None = None - question_name: str = "r2d2_server" - ask_question: bool = False - prompt: str = ( - "Server/profile name in ~/.swell/r2d2_credentials.yaml " - "(e.g. 'gmao_server'). Leave empty if credentials are at the root level." - ) - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class r2d2_datastore(SuiteQuestion): - default_value: str | None = None - question_name: str = "r2d2_datastore" - ask_question: bool = False - prompt: str = ( - "Datastore name passed to R2D2 fetch and store operations " - "(e.g. a Discover directory store or an S3 bucket store). " - "Run scripts/discover_r2d2_datastores.py to list available datastores. " - "Leave empty to let R2D2 pick the highest-priority writable datastore " - "for your compute host." - ) - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class runahead_limit(SuiteQuestion): - default_value: str = "P4" - question_name: str = "runahead_limit" - ask_question: bool = True - prompt: str = ("Set the Cylc runahead limit: the maximum number of cycles " - "that may be active ahead of the current cycle " - "(e.g. P1: up to 1 cycle ahead, P3: up to 3 cycles ahead, default P4).") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class saber_central_block(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "saber_central_block" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which saber central block do you want to use?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class saber_outer_block(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "saber_outer_block" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which saber outer blocks do you want to use?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class skip_ensemble_hofx(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "skip_ensemble_hofx" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Do you want to run localensembleda observer and solver together?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class skip_r2d2(SuiteQuestion): - default_value: bool = False - question_name: str = "skip_r2d2" - prompt: str = "Skip registering and storing results of this experiment in R2D2?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class start_cycle_point(SuiteQuestion): - default_value: str = "2023-10-10T00:00:00Z" - question_name: str = "start_cycle_point" - ask_question: bool = True - prompt: str = "What is the time of the first cycle (middle of the window)?" - widget_type: WType = WType.ISO_DATETIME - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class window_type(SuiteQuestion): - default_value: str = "defer_to_model" - question_name: str = "window_type" - options: List[str] = mutable_field([ - "3D", - "4D" - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter the window type for this model." - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - # Task question defaults go here - # -------------------------------------------------------------------------------------------------- - - @dataclass - class analysis_variables(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "analysis_variables" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What are the analysis variables?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_error_model(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_error_model" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which background error model do you want to use?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_experiment(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_experiment" - ask_question: bool = True - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the name of the name of the experiment providing the backgrounds?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_frequency(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_frequency" - models: List[str] = mutable_field([ - "all_models" - ]) - depends: Dict = mutable_field({ - "window_type": "4D" - }) - prompt: str = "What is the frequency of the background files?" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_time_offset(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "background_time_offset" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = ("How long before the middle of the analysis window did" - " the background providing forecast begin?") - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ebkg_time_offset(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ebkg_time_offset" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = ("How long before the middle of the analysis window did" - " the ensemble background providing forecast begin?") - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class rst_experiment(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "rst_experiment" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the name of the experiment providing the restart files in R2D2?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class rst_file_types(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "rst_file_types" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What are the restart file types to fetch/store from R2D2?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class rst_store_interval(TaskQuestion): - default_value: str = None - question_name: str = "rst_store_interval" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = ("After how many cycles should restart files be stored as real files " - "(not symlinks)? E.g. 28 means every 28th cycle (and multiples) stores " - "real files. Leave unset to always store as symlinks.") - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class bufr_obs_classes(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "bufr_obs_classes" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What BUFR observation classes will be used?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class bundles(TaskQuestion): - default_value: List[str] = mutable_field([ - "fv3-jedi", - "soca", - "iodaconv", - "ufo" - ]) - question_name: str = "bundles" - ask_question: bool = True - options: List[str] = mutable_field([ - "fv3-jedi", - "soca", - "iodaconv", - "ufo", - "ioda", - "oops", - "saber" - ]) - depends: Dict = mutable_field({ - "jedi_build_method": "create" - }) - prompt: str = "Which JEDI bundles do you wish to build?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class check_for_obs(TaskQuestion): - default_value: bool = True - question_name: str = "check_for_obs" - options: List[bool] = mutable_field([True, False]) - models: List[str] = mutable_field([ - 'all_models' - ]) - prompt: str = "Perform check for observations? Set to false for debugging purposes." - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class clean_patterns(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "clean_patterns" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Provide a list of patterns that you wish to remove from the cycle directory." - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class comparison_log_type(TaskQuestion): - default_value: str = "variational" - question_name: str = "comparison_log_type" - options: List[str] = mutable_field([ - 'variational', - 'fgat', - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Provide the log naming convention (e.g. 'variational', 'fgat')." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class crtm_coeff_dir(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "crtm_coeff_dir" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the CRTM coefficient files?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_packets(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_packets" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter number of packets in which ensemble observers should be computed." - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_hofx_strategy(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_hofx_strategy" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter hofx strategy." - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensemble_num_members(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ensemble_num_members" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "How many members comprise the ensemble?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_pert_amplitude(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "obs_pert_amplitude" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Enter obs perturbation amplitude for EDA:" - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensmean_only(TaskQuestion): - default_value: bool = False - question_name: str = "ensmean_only" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Calculate ensemble mean only?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensmeanvariance_only(TaskQuestion): - default_value: bool = False - question_name: str = "ensmeanvariance_only" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Calculate ensemble mean and variance only?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ensmeanvariance_spec(TaskQuestion): - default_value: List[Dict[str, str]] = field(default_factory=lambda: [{}]) - question_name: str = "ensmeanvariance_spec" - models: List[str] = mutable_field([ - "all_models" - ]) - ask_question: bool = True - prompt: str = "Configure the ensemble mean and variance specifications:" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class diffstates_spec(TaskQuestion): - default_value: Dict[str, Any] = field(default_factory=dict) - question_name: str = "diffstates_spec" - models: List[str] = mutable_field([ - "all_models" - ]) - ask_question: bool = True - prompt: str = "Configure the diffstates specifications: [state1, state2]" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_geos_gcm_build_path(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_geos_gcm_build_path" - ask_question: bool = True - depends: Dict = mutable_field({ - "geos_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing GEOS build directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_geos_gcm_source_path(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_geos_gcm_source_path" - ask_question: bool = True - depends: Dict = mutable_field({ - "geos_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing GEOS source code directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_build_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_build_directory" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing JEDI build directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_build_directory_pinned(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_build_directory_pinned" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_pinned_existing" - }) - prompt: str = "What is the path to the existing pinned JEDI build directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_source_directory(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_source_directory" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_existing" - }) - prompt: str = "What is the path to the existing JEDI source code directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_jedi_source_directory_pinned(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "existing_jedi_source_directory_pinned" - ask_question: bool = True - depends: Dict = mutable_field({ - "jedi_build_method": "use_pinned_existing" - }) - prompt: str = "What is the path to the existing pinned JEDI source code directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class existing_perllib_path(TaskQuestion): - default_value: str = 'defer_to_platform' - question_name: str = 'existing_perllib_path' - prompt: str = "Provide a path to an existing location for GMAO_perllib." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gmao_perllib_tag(TaskQuestion): - default_value: str = 'g1.0.1' - question_name: str = 'gmao_perllib_tag' - prompt: str = "Specify the tag at which GMAO_perllib should be cloned." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class forecast_duration(TaskQuestion): - default_value: str = "PT12H" - question_name: str = "forecast_duration" - ask_question: bool = True - prompt: str = "GEOS forecast duration" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class forecast_length(TaskQuestion): - default_value: str = "PT12H" - question_name: str = "forecast_length" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "Duration of the GEOS-CF forecast (ISO 8601 duration, e.g. PT12H)" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class forecast_output_frequency(TaskQuestion): - default_value: str = "PT1H" - question_name: str = "forecast_output_frequency" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "Frequency of forecast output files (ISO 8601 duration, e.g. PT1H)" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class generate_yaml_and_exit(TaskQuestion): - default_value: bool = False - question_name: str = "generate_yaml_and_exit" - prompt: str = "Generate JEDI executable YAML and exit?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_build_method(TaskQuestion): - default_value: str = "create" - question_name: str = "geos_build_method" - ask_question: bool = True - options: List[str] = mutable_field([ - "use_existing", - "create" - ]) - prompt: str = "Do you want to use an existing GEOS build or create a new build?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_homdir(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "geos_homdir" - ask_question: bool = True - prompt: str = ("What is the location for the HOME Directory (HOMDIR in gcm_run and " - "gcm_setup) that contains model settings and RC files?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_expdir_different(TaskQuestion): - default_value: str = False - question_name: str = "geos_expdir_different" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - prompt: str = ("Is your GEOS EXPERIMENT Directory, where restarts and scratch is located, " - "different than your GEOS HOME Directory?") - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_expdir(TaskQuestion): - default_value: str = "/dev/null/" - question_name: str = "geos_expdir" - depends: Dict = mutable_field({ - "geos_expdir_different": True - }) - prompt: str = ("What is the location for the EXPERIMENT Directory (to contain model " - "output and restart files), if it is different than your GEOS HOME " - "Directory?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_cf_install_dir(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "geos_cf_install_dir" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the path to the GEOS-CF install directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_cf_run_dir(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "geos_cf_run_dir" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the path to the GEOS-CF model run directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geosfp_exp(TaskQuestion): - default_value: str = "f5295_fp" - question_name: str = "geosfp_exp" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the GEOS FP experiment ID used for IAU analysis files?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geosfp_path(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "geosfp_path" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the path to the GEOS FP archive?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_gcm_tag(TaskQuestion): - default_value: str = "v11.6.0" - question_name: str = "geos_gcm_tag" - depends: Dict = mutable_field({ - "geos_build_method": "create" - }) - prompt: str = "Which GEOS tag do you wish to clone?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_x_background_directory(TaskQuestion): - default_value: str = "/dev/null/" - question_name: str = "geos_x_background_directory" - ask_question: bool = True - options: List[str] = mutable_field([ - "/dev/null/", - "/discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x" - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the path to the GEOS X-backgrounds directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geos_x_ensemble_directory(TaskQuestion): - default_value: str = "/dev/null/" - question_name: str = "geos_x_ensemble_directory" - ask_question: bool = True - options: List[str] = mutable_field([ - "/dev/null/", - "/gpfsm/dnb05/projects/p139/rtodling/archive/" - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the GEOS X-backgrounds directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geovals_experiment(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "geovals_experiment" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the name of the R2D2 experiment providing the GeoVaLs?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class geovals_provider(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "geovals_provider" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the name of the R2D2 database providing the GeoVaLs?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gradient_norm_reduction(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gradient_norm_reduction" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What value of gradient norm reduction for convergence?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gsibec_configuration(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gsibec_configuration" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which GSIBEC climatological or hybrid?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gsibec_nlats(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gsibec_nlats" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "How many number of latutides in GSIBEC grid?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class gsibec_nlons(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "gsibec_nlons" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "How many number of longitudes in GSIBEC grid?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class horizontal_resolution(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "horizontal_resolution" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the horizontal resolution for the forecast model and backgrounds?" - widget_type: WType = WType.STRING_DROP_LIST - - # ------------------------------------------------------------------------------------------------ - - @dataclass - class dry_run(TaskQuestion): - default_value: bool = True - question_name: str = "dry_run" - ask_question: bool = False - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Dry-run mode: preview what would be ingested before storing to R2D2" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class store_as_symlink(TaskQuestion): - default_value: bool = True - question_name: str = "store_as_symlink" - ask_question: bool = True - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Store background files as symlinks in R2D2 instead of copying them?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_rc_path(TaskQuestion): - default_value: str = 'GEOS_mksi/ObsClass/obsys-nccs.rc' - question_name: str = 'obs_rc_path' - ask_question: bool = True - prompt: str = "Filepath to observing system rc file within experiment directory." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_to_ingest(TaskQuestion): - default_value: list = mutable_field([]) - question_name: str = "obs_to_ingest" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which observations do you want to ingest to R2D2?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_to_download(TaskQuestion): - default_value: list = mutable_field([]) - question_name: str = "obs_to_download" - ask_question: bool = True - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which observations do you want to download from remote servers?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class converter_path(TaskQuestion): - default_value: str = "" - question_name: str = "converter_path" - ask_question: bool = True - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = ("Path to directory containing ioda-converter scripts" - " (leave blank to use jedi_bin)") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class initial_restarts_method(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "initial_restarts_method" - ask_question: bool = True - options: List[str] = mutable_field([ - "geos_expdir", - "r2d2", - "hotstart", - ]) - prompt: str = "How should initial GEOS restarts be obtained?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ioda_locations_not_in_r2d2(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "ioda_locations_not_in_r2d2" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ( - "Provide a path that contains observation files not in r2d2.") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class iau(TaskQuestion): - default_value: bool = True - question_name: str = "iau" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "Use Incremental Analysis Update (IAU) in the GEOS-CF forecast?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class inc_template(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "inc_template" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the path to the GEOS-CF increment template NetCDF file?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class jedi_build_method(TaskQuestion): - default_value: str = "use_existing" - question_name: str = "jedi_build_method" - ask_question: bool = True - options: List[str] = mutable_field([ - "use_existing", - "use_pinned_existing", - "create", - "pinned_create" - ]) - prompt: str = "Do you want to use an existing JEDI build or create a new build?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class jedi_forecast_model(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "jedi_forecast_model" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - depends: Dict = mutable_field({ - "window_type": "4D" - }) - prompt: str = "What forecast model should be used within JEDI for 4D window propagation?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_inflation_mult(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_inflation_mult" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Specify the multiplicative prior inflation coefficient (0 inf]." - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_inflation_rtpp(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_inflation_rtpp" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Specify the Relaxation To Prior Perturbation (RTPP) coefficient (0 1]." - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_inflation_rtps(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_inflation_rtps" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Specify the Relaxation To Prior Spread (RTPS) coefficient (0 1]." - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_ensemble(TaskQuestion): - default_value: bool = False - question_name: str = "local_ensemble_save_posterior_ensemble" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Save the posterior ensemble members?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): - default_value: bool = False - question_name: str = "local_ensemble_save_posterior_ensemble_increments" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Save the posterior ensemble member increments?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_mean(TaskQuestion): - default_value: bool = False - question_name: str = "local_ensemble_save_posterior_mean" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Save the posterior ensemble mean?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_save_posterior_mean_increment(TaskQuestion): - default_value: bool = True - question_name: str = "local_ensemble_save_posterior_mean_increment" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Save the posterior ensemble mean increment?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_solver(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_solver" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which local ensemble solver type should be implemented?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class local_ensemble_use_linear_observer(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "local_ensemble_use_linear_observer" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Use linear observer in local ensemble solver?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class minimizer(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "minimizer" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which data assimilation minimizer do you wish to use?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class mom6_iau(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "mom6_iau" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_marine", - ]) - prompt: str = "Do you wish to use IAU for MOM6?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class mom6_iau_nhours(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "mom6_iau_nhours" - options: List[str] = mutable_field([ - 'PT3H', - 'PT12H' - ]) - depends: dict = mutable_field({'mom6_iau': True}) - models: List[str] = mutable_field([ - "geos_marine", - ]) - prompt: str = "What is the IAU length (ODA_INCUPD_NHOURS) for MOM6?" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ncdiag_experiments(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "ncdiag_experiments" - options: List[str] = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which previously run experiments do you wish to use for the NCdiag?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npx_proc(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npx_proc" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere", - "geos_cf" - ]) - prompt: str = "What number of processors do you wish to use in the x-direction?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npy_proc(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npy_proc" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere", - "geos_cf" - ]) - prompt: str = "What number of processors do you wish to use in the y-direction?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npx(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npx" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the number of grid points in the x-direction on each cube face?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class npy(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "npy" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_cf" - ]) - prompt: str = "What is the number of grid points in the y-direction on each cube face?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class number_of_iterations(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "number_of_iterations" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = ( - "What number of iterations do you wish to use for each outer loop?" - " Provide a list of integers the same length as the number of outer loops.") - widget_type: WType = WType.INTEGER_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_experiment(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "obs_experiment" - ask_question: bool = True - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the database providing the observations?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observation_providers(TaskQuestion): - default_value: Dict[str, str] = mutable_field({}) - question_name: str = "observation_providers" - ask_question: bool = False - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Map observation names to their R2D2 providers." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class obs_thinning_rej_fraction(TaskQuestion): - default_value: float = 0.75 - question_name: str = "obs_thinning_rej_fraction" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the rejection fraction for obs thinning?" - widget_type: WType = WType.FLOAT - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observations(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observations" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Which observations do you want to include?" - widget_type: WType = WType.STRING_CHECK_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observing_system_records_mksi_path(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observing_system_records_mksi_path" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the GSI formatted observing system records?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observing_system_records_mksi_path_tag(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observing_system_records_mksi_path_tag" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the GSI formatted observing system records tag?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class observing_system_records_path(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "observing_system_records_path" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to the Swell formatted observing system records?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_ensemble(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_ensemble" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_marine" - ]) - prompt: str = "What is the path to where ensemble members are stored?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_geos_adas_background(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_geos_adas_background" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ( - "What is the path for the GEOSadas cubed sphere backgrounds?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_gsi_bc_coefficients(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_gsi_bc_coefficients" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the location where GSI bias correction files can be found?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class path_to_gsi_nc_diags(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "path_to_gsi_nc_diags" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the path to where the GSI ncdiags are stored?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class perhost(TaskQuestion): - default_value: str = None - question_name: str = "perhost" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the number of processors per host?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class produce_geovals(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "produce_geovals" - ask_question: bool = True - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("When running the ncdiag to ioda converted do you " - "want to produce GeoVaLs files?") - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class publish_directory(TaskQuestion): - default_value: str = None - question_name: str = "publish_directory" - ask_question: bool = False - prompt: str = "Provide an external directory to publish relevant results to." - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class cache_fetch(TaskQuestion): - default_value: bool = True - question_name: str = "cache_fetch" - options: List[bool] = mutable_field([ - True, - False - ]) - prompt: str = "Use cached observation files if they already exist?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class fetch_obs_from_public_s3(TaskQuestion): - default_value: bool = False - question_name: str = "fetch_obs_from_public_s3" - options: List[bool] = mutable_field([ - True, - False - ]) - prompt: str = "Fetch observations directly from a public S3 bucket if they are available?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class save_geovals(TaskQuestion): - default_value: bool = False - question_name: str = "save_geovals" - options: List[bool] = mutable_field([ - True, - False - ]) - prompt: str = "When running hofx do you want to output the GeoVaLs?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class single_observations(TaskQuestion): - default_value: bool = False - question_name: str = "single_observations" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Is it a single-observation test?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class swell_static_files(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "swell_static_files" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the path to the Swell Static files directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class swell_static_files_user(TaskQuestion): - default_value: str = "None" - question_name: str = "swell_static_files_user" - prompt: str = "What is the path to the user provided Swell Static Files directory?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class total_processors(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "total_processors" - ask_question: bool = True - models: List[str] = mutable_field([ - "geos_marine", - ]) - prompt: str = "What is the number of processors for JEDI?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_apply_log_transform(TaskQuestion): - default_value: bool = True - question_name: str = "vertical_localization_apply_log_transform" - options: List[bool] = mutable_field([ - True, - False - ]) - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("Should a log (base 10) transformation be applied " - "to vertical coordinate when " - "constructing vertical localization?") - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_function(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_function" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which localization scheme should be applied in the vertical?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_ioda_vertical_coord(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_ioda_vertical_coord" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "Which coordinate should be used in constructing vertical localization?" - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_ioda_vertical_coord_group(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_ioda_vertical_coord_group" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("Which vertical coordinate group should be used " - "in constructing vertical localization?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_lengthscale(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_lengthscale" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = "What is the length scale for vertical covariance localization?" - widget_type: WType = WType.INTEGER - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_localization_method(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_localization_method" - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "geos_atmosphere" - ]) - prompt: str = ("What localization scheme should be applied in " - "constructing a vertical localization?") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class vertical_resolution(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "vertical_resolution" - ask_question: bool = True - options: str = "defer_to_model" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the vertical resolution for the forecast model and background?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class window_length(TaskQuestion): - default_value: str = "defer_to_model" - question_name: str = "window_length" - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "What is the duration for the data assimilation window?" - widget_type: WType = WType.ISO_DURATION - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class window_type(TaskQuestion): - question_name: str = "window_type" - default_value: str = "defer_to_model" - ask_question: bool = True - options: List[str] = mutable_field([ - "3D", - "4D" - ]) - models: List[str] = mutable_field([ - "all_models" - ]) - prompt: str = "Do you want to use a 3D or 4D (including FGAT) window?" - widget_type: WType = WType.STRING_DROP_LIST - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class background_source_path(TaskQuestion): - default_value: str = ( - '/css/gmao/geos-cf/NRTv2/priv/ana/Y%Y/M%m/D%d/' - 'GEOS.cf.ana.jdi_inst_1hr_glo_C360x360x6_v72.%Y%m%d_%H%Mz.R0.nc4' - ) - question_name: str = "background_source_path" - ask_question: bool = True - models: List[str] = mutable_field(['geos_cf']) - prompt: str = ("Path template for background files. Uses Python strftime format codes, " - "e.g. Y%Y/M%m/D%d gives Y2025/M10/D02 and %Y%m%d_%H%Mz gives " - "20251002_0900z.") - widget_type: WType = WType.STRING - - # -------------------------------------------------------------------------------------------------- - - @dataclass - class ingest_background_pipeline(SuiteQuestion): - default_value: bool = False - question_name: str = "ingest_background_pipeline" - ask_question: bool = False - prompt: str = "Run the SaveBackground task to ingest background files into R2D2?" - widget_type: WType = WType.BOOLEAN - - # -------------------------------------------------------------------------------------------------- - @dataclass - class download_convert_pipeline(SuiteQuestion): - default_value: bool = False - question_name: str = "download_convert_pipeline" - ask_question: bool = False - prompt: str = ("Run the DownloadObs and ConvertObsToIoda tasks?" - "(DownloadObs -> ConvertObsToIoda) -> IngestObs to R2D2") - widget_type: WType = WType.BOOLEAN diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index 331ab1d0b..2c1b9b2b8 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -16,6 +16,7 @@ from swell.utilities.datetime_util import is_datetime, is_duration from swell.swell_path import get_swell_path +from swell.utilities.logger import Logger # -------------------------------------------------------------------------------------------------- From 34b9c67c559e38f0cefe6b37e8327c7540d6c587 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 25 Aug 2026 14:05:41 -0400 Subject: [PATCH 05/15] Experiment creation functional --- src/swell/configuration/question_defaults.py | 18 +++++++----------- .../question_and_answer_defaults.py | 11 ++++++++--- src/swell/utilities/config.py | 7 ++++++- src/swell/utilities/swell_questions.py | 4 ++++ 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py index 2faa3f3a3..1776f8028 100644 --- a/src/swell/configuration/question_defaults.py +++ b/src/swell/configuration/question_defaults.py @@ -167,7 +167,7 @@ class r2d2_server(SuiteQuestion): "Server/profile name in ~/.swell/r2d2_credentials.yaml " "(e.g. 'gmao_server'). Leave empty if credentials are at the root level." ) - data_type: DType = DType.STRING + data_type: list = mutable_field([DType.STRING, DType.NONE]) # -------------------------------------------------------------------------------------------------- @@ -181,7 +181,7 @@ class r2d2_datastore(SuiteQuestion): "Leave empty to let R2D2 pick the highest-priority writable datastore " "for your compute host." ) - data_type: DType = DType.STRING + data_type: DType = mutable_field([DType.STRING, DType.NONE]) # -------------------------------------------------------------------------------------------------- @@ -843,7 +843,7 @@ class gradient_norm_reduction(TaskQuestion): "all_models" ]) prompt: str = "What value of gradient norm reduction for convergence?" - data_type: DType = DType.STRING + data_type: DType = DType.FLOAT # -------------------------------------------------------------------------------------------------- @@ -865,7 +865,7 @@ class gsibec_nlats(TaskQuestion): "geos_atmosphere" ]) prompt: str = "How many number of latutides in GSIBEC grid?" - data_type: DType = DType.STRING + data_type: DType = DType.INTEGER # -------------------------------------------------------------------------------------------------- @@ -876,7 +876,7 @@ class gsibec_nlons(TaskQuestion): "geos_atmosphere" ]) prompt: str = "How many number of longitudes in GSIBEC grid?" - data_type: DType = DType.STRING + data_type: DType = DType.INTEGER # -------------------------------------------------------------------------------------------------- @@ -1289,7 +1289,7 @@ class observation_providers(TaskQuestion): "all_models" ]) prompt: str = "Map observation names to their R2D2 providers." - data_type: DType = DType.STRING + data_type: DType = DType.MAPPING # -------------------------------------------------------------------------------------------------- @@ -1397,15 +1397,11 @@ class path_to_gsi_nc_diags(TaskQuestion): @dataclass class perhost(TaskQuestion): default_value: str = None - options: List[bool] = mutable_field([ - True, - False - ]) models: List[str] = mutable_field([ "geos_atmosphere" ]) prompt: str = "What is the number of processors per host?" - data_type: DType = DType.INTEGER + data_type: DType = mutable_field([DType.INTEGER, DType.NONE]) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py b/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py index e70ae4872..2601f78c2 100644 --- a/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py +++ b/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py @@ -20,9 +20,14 @@ def get_answer(self, logger: Logger, key: str, val: dict, default = val['default_value'] data_type = val['data_type'] - if not data_type.is_type(default): - logger.abort(f'Default value for {key}, {default}, does not conform to type ' - f'{data_type.base_type.__name__}, check the override file or ' + if isinstance(data_type, list): + if not any([dtype.is_type(default) for dtype in data_type]): + self.__logger__.warning(f'Warning: Experiment key {name} does not conform to any expected' + f' types {" ".join(data_type)}') + + elif not data_type.is_type(default): + logger.warning(f'Default value for {key}, {default}, does not conform to type ' + f'{data_type.value}, check the override file or ' 'suite configuration.') return default diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 4ed3af6ab..dea439908 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -142,7 +142,12 @@ def resolve(self, question: SwellQuestion, default='LrZRExPGcQ'): data_type = question_obj.data_type - if not data_type.is_type(default): + if isinstance(data_type, list): + if not any([dtype.is_type(default) for dtype in data_type]): + self.__logger__.warning(f'Warning: Experiment key {name} does not conform to any expected' + f' types {" ".join(data_type)}') + + elif not data_type.is_type(default): self.__logger__.warning(f'Warning: Experiment key {name} does not conform to expected' f' type <{data_type.value}>.') diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index 2c1b9b2b8..ecc02d3ec 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -31,10 +31,14 @@ class DataType(Enum): FLOAT = "float" LIST = "list" MAPPING = "mapping" + NONE = "none" def is_type(self, value) -> bool: """ Validate that the value matches the data type. """ + if self == DataType.NONE: + return value is None + # Ensure value is in ISO datetime format if self == DataType.ISO_DATETIME: return is_datetime(value) From adade36108893b3ac3de91fa0cf2c769b5d9df95 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 25 Aug 2026 15:24:16 -0400 Subject: [PATCH 06/15] Change for all tasks --- src/swell/tasks/bufr_to_ioda.py | 1 + src/swell/tasks/build_geos.py | 3 +- src/swell/tasks/build_geos_by_linking.py | 11 ++- src/swell/tasks/build_jedi.py | 9 +- src/swell/tasks/build_jedi_by_linking.py | 11 ++- src/swell/tasks/clean_cycle.py | 5 +- src/swell/tasks/clean_eda.py | 55 +++++------ src/swell/tasks/clone_geos.py | 10 +- src/swell/tasks/clone_geos_mksi.py | 7 +- src/swell/tasks/clone_gmao_perllib.py | 5 +- src/swell/tasks/clone_jedi.py | 17 ++-- src/swell/tasks/convert_obs_to_ioda.py | 7 +- src/swell/tasks/download_obs.py | 7 +- src/swell/tasks/eva_comparison_increment.py | 3 +- src/swell/tasks/eva_comparison_jedi_log.py | 5 +- .../tasks/eva_comparison_observations.py | 7 +- src/swell/tasks/eva_increment.py | 9 +- src/swell/tasks/eva_jedi_log.py | 1 + src/swell/tasks/eva_observations.py | 13 +-- src/swell/tasks/eva_timeseries.py | 15 +-- src/swell/tasks/generate_b_climatology.py | 29 +++--- .../generate_b_climatology_by_linking.py | 17 ++-- .../generate_observing_system_records.py | 7 +- src/swell/tasks/get_background.py | 19 ++-- .../tasks/get_background_geos_experiment.py | 7 +- src/swell/tasks/get_bufr.py | 5 +- src/swell/tasks/get_coupled_geos_restart.py | 9 +- .../tasks/get_ensemble_geos_experiment.py | 7 +- src/swell/tasks/get_geos_adas_background.py | 3 +- src/swell/tasks/get_geovals.py | 15 +-- src/swell/tasks/get_gsi_bc.py | 5 +- src/swell/tasks/get_gsi_ncdiag.py | 3 +- src/swell/tasks/get_ncdiags.py | 17 ++-- src/swell/tasks/get_obs_not_in_r2d2.py | 3 +- src/swell/tasks/get_observations.py | 29 +++--- src/swell/tasks/get_restart_cf.py | 11 ++- src/swell/tasks/gsi_bc_to_ioda.py | 11 ++- src/swell/tasks/gsi_ncdiag_to_ioda.py | 9 +- src/swell/tasks/ingest_obs.py | 13 +-- src/swell/tasks/jedi_log_comparison.py | 5 +- src/swell/tasks/jedi_oops_log_parser.py | 5 +- src/swell/tasks/link_coupled_geos_output.py | 9 +- src/swell/tasks/link_geos_output.py | 9 +- src/swell/tasks/move_da_restart.py | 9 +- src/swell/tasks/move_forecast_restart.py | 3 +- src/swell/tasks/prep_coupled_geos_run_dir.py | 15 +-- src/swell/tasks/prep_forecast_cf.py | 21 ++-- src/swell/tasks/prepare_analysis.py | 11 ++- src/swell/tasks/publish_comparisons.py | 5 +- src/swell/tasks/render_jedi_observations.py | 17 ++-- ...jedi_convert_state_soca2cice_executable.py | 17 ++-- src/swell/tasks/run_jedi_diffstates.py | 25 ++--- src/swell/tasks/run_jedi_eda_executable.py | 61 ++++++------ .../tasks/run_jedi_ensemble_mean_variance.py | 29 +++--- src/swell/tasks/run_jedi_fgat_executable.py | 45 ++++----- .../run_jedi_hofx_ensemble_executable.py | 33 +++---- src/swell/tasks/run_jedi_hofx_executable.py | 43 +++++---- .../run_jedi_local_ensemble_da_executable.py | 95 ++++++++++--------- .../tasks/run_jedi_obsfilters_executable.py | 31 +++--- .../tasks/run_jedi_ufo_tests_executable.py | 17 ++-- .../tasks/run_jedi_variational_executable.py | 57 +++++------ src/swell/tasks/save_background.py | 11 ++- src/swell/tasks/save_forecast_cf.py | 11 ++- src/swell/tasks/save_obs_diags.py | 19 ++-- src/swell/tasks/save_restart.py | 1 + src/swell/tasks/save_restart_cf.py | 11 ++- src/swell/tasks/stage_jedi.py | 21 ++-- 67 files changed, 562 insertions(+), 494 deletions(-) diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 0ea370754..da2d54238 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -13,6 +13,7 @@ import subprocess import shutil +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 614db7023..5ffdf93ef 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -10,6 +10,7 @@ import os +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.build import build_and_source_dirs from swell.utilities.shell_commands import run_subprocess, create_executable_file @@ -34,7 +35,7 @@ def execute(self) -> None: # Check that the choice is to create build # ---------------------------------------- - if not self.config.geos_build_method() == 'create': + if not self.config.resolve(geos_build_method) == 'create': self.logger.abort(f'Found \'{jedi_build_method}\' for jedi_build_method in the ' f'experiment dictionary. Must be \'create\'.') diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index 16fc91ee6..0e0077a40 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -10,6 +10,7 @@ import os +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.build import build_and_source_dirs, link_path @@ -32,16 +33,16 @@ def execute(self) -> None: # Choice to link to existing build or build GEOS # ---------------------------------------------- - if not self.config.geos_build_method() == 'use_existing': - self.logger.abort(f'Found \'{self.config.geos_build_method()}\' for ' + + if not self.config.resolve(geos_build_method) == 'use_existing': + self.logger.abort(f'Found \'{self.config.resolve(geos_build_method)}\' for ' + f'geos_build_method in the experiment dictionary. Must be ' + f'\'use_existing\'.') # Assert that the existing build directory contains a bin directory - if not os.path.exists(os.path.join(self.config.existing_geos_gcm_build_path(), 'bin')): + if not os.path.exists(os.path.join(self.config.resolve(existing_geos_gcm_build_path), 'bin')): self.logger.abort(f'Existing GEOS build directory is provided but a bin ' + f'directory is not found in the path ' + - f'\'{self.config.existing_geos_gcm_build_path()}\'') + f'\'{self.config.resolve(existing_geos_gcm_build_path)}\'') # Write warning to user self.logger.info('Suitable GEOS build found, linking build directory. Warning: ' + @@ -50,7 +51,7 @@ def execute(self) -> None: 'this experiment may not be reproducible if the build changes.') # Link the source code directory - link_path(self.config.existing_geos_gcm_build_path(), geos_gcm_build_path) + link_path(self.config.resolve(existing_geos_gcm_build_path), geos_gcm_build_path) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index d64bd0b62..b9b53c8a9 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -12,6 +12,7 @@ from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs @@ -33,8 +34,8 @@ def execute(self) -> None: # Choice to link to existing build or build JEDI using jedi_bundle # ---------------------------------------------------------------- - if self.config.jedi_build_method() not in ('create', 'pinned_create'): - self.logger.abort(f'Found \'{self.config.jedi_build_method()}\' for ' + + if self.config.resolve(jedi_build_method) not in ('create', 'pinned_create'): + self.logger.abort(f'Found \'{self.config.resolve(jedi_build_method)}\' for ' + f'jedi_build_method in the experiment dictionary. Must be ' + f'\'create\' or \'pinned_create\'.') @@ -53,11 +54,11 @@ def execute(self) -> None: bundles = get_bundles() use_pinned = False - if self.config.jedi_build_method() == 'pinned_create': + if self.config.resolve(jedi_build_method) == 'pinned_create': use_pinned = True # Generate the build dictionary - jedi_bundle_dict = set_jedi_bundle_config(self.config.bundles(bundles), + jedi_bundle_dict = set_jedi_bundle_config(self.config.resolve(bundles, default=bundles), jedi_bundle_source_path, jedi_bundle_build_path, self.platform(), use_pinned, 24) diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index f746a70f8..381cde5b4 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -10,6 +10,7 @@ import os +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.build import build_and_source_dirs, link_path @@ -21,8 +22,8 @@ class BuildJediByLinking(taskBase): def execute(self) -> None: # Abort if jedi build method isn't use_existing or use_pinned_existing - if self.config.jedi_build_method() not in ('use_existing', 'use_pinned_existing'): - self.logger.abort(f'Found \'{self.config.jedi_build_method()}\' for ' + + if self.config.resolve(jedi_build_method) not in ('use_existing', 'use_pinned_existing'): + self.logger.abort(f'Found \'{self.config.resolve(jedi_build_method)}\' for ' + f'jedi_build_method in the experiment dictionary. Must be ' + f'\'use_existing\' or \'use_pinned_existing\'.') @@ -34,10 +35,10 @@ def execute(self) -> None: jedi_bundle_build_path, jedi_bundle_source_path = build_and_source_dirs(jedi_bundle_path) # Set existing jedi build directory based on jedi build method - if self.config.jedi_build_method() == 'use_existing': - existing_jedi_build_directory = self.config.existing_jedi_build_directory() + if self.config.resolve(jedi_build_method) == 'use_existing': + existing_jedi_build_directory = self.config.resolve(existing_jedi_build_directory) else: - existing_jedi_build_directory = self.config.existing_jedi_build_directory_pinned() + existing_jedi_build_directory = self.config.resolve(existing_jedi_build_directory_pinned) # Assert that the existing build directory contains a bin directory if not os.path.exists(os.path.join(existing_jedi_build_directory, 'bin')): diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index f552bbd5c..a7ba54242 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -10,6 +10,7 @@ import isodate import os import shutil +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats from datetime import datetime as dt @@ -32,7 +33,7 @@ class CleanCycle(taskBase): def execute(self) -> None: # Parse config - clean_patterns = self.config.clean_patterns(None) + clean_patterns = self.config.resolve(clean_patterns, default=None) # If no cleaning requested then exit if clean_patterns is None: @@ -91,7 +92,7 @@ def execute(self) -> None: # Clean the previous cycle's scratch directory now that this cycle has retrieved # any restart/checkpoint files it needed (which may have been symlinked from there) - window_length = self.config.window_length(None) + window_length = self.config.resolve(window_length, default=None) if window_length is not None and self.cycle_time_dto() != self.start_cycle_point_dto(): prev_cycle_time = self.cycle_time_dto() - isodate.parse_duration(window_length) diff --git a/src/swell/tasks/clean_eda.py b/src/swell/tasks/clean_eda.py index 7b745ffda..bb8dec60b 100644 --- a/src/swell/tasks/clean_eda.py +++ b/src/swell/tasks/clean_eda.py @@ -10,6 +10,7 @@ import os import glob import shutil +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase @@ -28,23 +29,23 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_type = self.config.window_type() - window_length = self.config.window_length() - forecast_length = self.config.forecast_length(window_length) - background_time_offset = self.config.background_time_offset() - number_of_iterations = self.config.number_of_iterations() - jedi_forecast_model = self.config.jedi_forecast_model(None) + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) + forecast_length = self.config.resolve(forecast_length, default=window_length) + background_time_offset = self.config.resolve(background_time_offset) + number_of_iterations = self.config.resolve(number_of_iterations) + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) - gsibec_nlats = self.config.gsibec_nlats(None) - gsibec_nlons = self.config.gsibec_nlons(None) - gsibec_configuration = self.config.gsibec_configuration(None) - npx_proc = self.config.npx_proc(None) - npy_proc = self.config.npy_proc(None) - npx = self.config.npx(None) - npy = self.config.npy(None) + gsibec_nlats = self.config.resolve(gsibec_nlats, default=None) + gsibec_nlons = self.config.resolve(gsibec_nlons, default=None) + gsibec_configuration = self.config.resolve(gsibec_configuration, default=None) + npx_proc = self.config.resolve(npx_proc, default=None) + npy_proc = self.config.resolve(npy_proc, default=None) + npx = self.config.resolve(npx, default=None) + npy = self.config.resolve(npy, default=None) # Compute data assimilation window parameters # -------------------------------------------- @@ -56,7 +57,7 @@ def execute(self) -> None: window_begin = self.da_window_params.window_begin(window_length) window_begin_iso = self.da_window_params.window_begin_iso(window_length) window_end_iso = self.da_window_params.window_end_iso(window_length) - nmember = self.config.ensemble_num_members() + nmember = self.config.resolve(ensemble_num_members) # imember = self.get_ensemble_imember() # Populate jedi interface templates dictionary @@ -65,38 +66,38 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) self.jedi_rendering.add_key('forecast_length', forecast_length) - self.jedi_rendering.add_key('minimizer', self.config.minimizer()) + self.jedi_rendering.add_key('minimizer', self.config.resolve(minimizer)) self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) - self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) - self.jedi_rendering.add_key('saber_central_block', self.config.saber_central_block(None)) - self.jedi_rendering.add_key('saber_outer_block', self.config.saber_outer_block(None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) + self.jedi_rendering.add_key('saber_central_block', self.config.resolve(saber_central_block, default=None)) + self.jedi_rendering.add_key('saber_outer_block', self.config.resolve(saber_outer_block, default=None)) self.jedi_rendering.add_key('gradient_norm_reduction', - self.config.gradient_norm_reduction()) - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.config.resolve(gradient_norm_reduction)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) - self.jedi_rendering.add_key('ensemble_num_members', self.config.ensemble_num_members()) + self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(ensemble_num_members)) self.jedi_rendering.add_key('ensemble_imember', 1) # pass an integer to jinja2 # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) self.jedi_rendering.add_key('npx', npx) self.jedi_rendering.add_key('npy', npy) - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Atmosphere background error model @@ -111,7 +112,7 @@ def execute(self) -> None: # Model # ----- if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.background_frequency()) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) # Open the JEDI config file and fill initial templates # ---------------------------------------------------- diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 6190a81a0..4b9d48f1e 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -10,6 +10,8 @@ import os +from swell.configuration.question_defaults import * +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.build import build_and_source_dirs, link_path from swell.utilities.git_utils import git_clone @@ -33,15 +35,15 @@ def execute(self) -> None: # Choice to link to existing build or build GEOS # ---------------------------------------------- - if self.config.geos_build_method() == 'use_existing': + if self.config.resolve(resolve, default=geos_build_method) == 'use_existing': # Link the source code directory - link_path(self.config.existing_geos_gcm_source_path(), geos_gcm_source_path) + link_path(self.config.resolve(resolve, default=existing_geos_gcm_source_path), geos_gcm_source_path) - elif self.config.geos_build_method() == 'create': + elif self.config.resolve(resolve, default=geos_build_method) == 'create': # Get tag to build - geos_gcm_tag = self.config.geos_gcm_tag() + geos_gcm_tag = self.config.resolve(resolve, default=geos_gcm_tag) # Make sure tag is prepended with 'v' if geos_gcm_tag[0] != 'v': diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index 555b6638a..ff7c15a34 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -9,6 +9,7 @@ import os +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.build import link_path @@ -31,8 +32,8 @@ def execute(self) -> None: # Parse config # ------------ - path_to_geos_mksi = self.config.observing_system_records_mksi_path() - tag = self.config.observing_system_records_mksi_path_tag() + path_to_geos_mksi = self.config.resolve(observing_system_records_mksi_path) + tag = self.config.resolve(observing_system_records_mksi_path_tag) # If observing_system_records_mksi_path is None, clone GEOS_mksi repo to experiment # directory @@ -46,7 +47,7 @@ def execute(self) -> None: + os.path.join(self.experiment_path(), 'GEOS_mksi')) else: # Link the source code directory - link_path(self.config.observing_system_records_mksi_path(), + link_path(self.config.resolve(observing_system_records_mksi_path), os.path.join(self.experiment_path(), 'GEOS_mksi')) diff --git a/src/swell/tasks/clone_gmao_perllib.py b/src/swell/tasks/clone_gmao_perllib.py index 573876822..bf3f6884b 100644 --- a/src/swell/tasks/clone_gmao_perllib.py +++ b/src/swell/tasks/clone_gmao_perllib.py @@ -11,6 +11,7 @@ import os import subprocess +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -25,8 +26,8 @@ def execute(self) -> None: experiment_perllib_path = os.path.join(swell_exp_path, 'GMAO_perllib') # Get the existing location of GMAO_perllib - existing_perllib_path = self.config.gmao_perllib_path(None) - gmao_perllib_tag = self.config.gmao_perllib_tag(None) + existing_perllib_path = self.config.resolve(gmao_perllib_path, default=None) + gmao_perllib_tag = self.config.resolve(gmao_perllib_tag, default=None) # Set the default tag to g1.0.1 if not specified if gmao_perllib_tag is None: diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index e1cee67c9..a0dd1ce33 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -13,6 +13,7 @@ from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles from swell.utilities.build import link_path +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.pinned_versions.check_hashes import check_hashes from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs @@ -37,17 +38,17 @@ def execute(self) -> None: # Choice to link to existing build or build JEDI using jedi_bundle # ---------------------------------------------------------------- - if self.config.resolve(jedi_build_method) == 'use_existing': + if self.config.resolve(resolve, default=jedi_build_method) == 'use_existing': # Link the source code directory - link_path(self.config.resolve(existing_jedi_source_directory), jedi_bundle_source_path) + link_path(self.config.resolve(resolve, default=existing_jedi_source_directory), jedi_bundle_source_path) - elif self.config.resolve(jedi_build_method) == 'use_pinned_existing': + elif self.config.resolve(resolve, default=jedi_build_method) == 'use_pinned_existing': # Check hashes before proceeding - check_hashes(self.config.resolve(existing_jedi_source_directory_pinned), self.logger) + check_hashes(self.config.resolve(resolve, default=existing_jedi_source_directory_pinned), self.logger) # Link the pinned source code directory - link_path(self.config.resolve(existing_jedi_source_directory_pinned), jedi_bundle_source_path) + link_path(self.config.resolve(resolve, default=existing_jedi_source_directory_pinned), jedi_bundle_source_path) - elif self.config.resolve(jedi_build_method) in ('create', 'pinned_create'): + elif self.config.resolve(resolve, default=jedi_build_method) in ('create', 'pinned_create'): # Determine which bundles need to be build model_components = self.get_model_components() if model_components is not None: @@ -64,11 +65,11 @@ def execute(self) -> None: # Determine whether to use pinned versions or not use_pinned = False - if self.config.resolve(jedi_build_method) == 'pinned_create': + if self.config.resolve(resolve, default=jedi_build_method) == 'pinned_create': use_pinned = True # Generate the build dictionary - jedi_bundle_dict = set_jedi_bundle_config(self.config.resolve(bundles, default=bundles), + jedi_bundle_dict = set_jedi_bundle_config(self.config.resolve(resolve, default=bundles, bundles), jedi_bundle_source_path, jedi_bundle_build_path, self.platform(), diff --git a/src/swell/tasks/convert_obs_to_ioda.py b/src/swell/tasks/convert_obs_to_ioda.py index e6ea1998c..97c23f281 100644 --- a/src/swell/tasks/convert_obs_to_ioda.py +++ b/src/swell/tasks/convert_obs_to_ioda.py @@ -20,6 +20,7 @@ import yaml from datetime import datetime +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase @@ -62,13 +63,13 @@ class ConvertObsToIoda(taskBase): def execute(self) -> None: - obs_to_convert = self.config.obs_to_download([]) - dry_run = self.config.dry_run(True) + obs_to_convert = self.config.resolve(obs_to_download, default=[]) + dry_run = self.config.resolve(dry_run, default=True) if dry_run: self.logger.info('DRY RUN MODE - No converters will be run') - converter_path = self.config.converter_path('') + converter_path = self.config.resolve(converter_path, default='') if converter_path: converter_bin = converter_path else: diff --git a/src/swell/tasks/download_obs.py b/src/swell/tasks/download_obs.py index 1056e177c..9d2b0ea76 100644 --- a/src/swell/tasks/download_obs.py +++ b/src/swell/tasks/download_obs.py @@ -35,6 +35,7 @@ import isodate import requests +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities import s3 as swell_s3 @@ -79,9 +80,9 @@ class DownloadObs(taskBase): def execute(self) -> None: - obs_to_download = self.config.obs_to_download([]) - window_length = self.config.window_length() - dry_run = self.config.dry_run(True) + obs_to_download = self.config.resolve(obs_to_download, default=[]) + window_length = self.config.resolve(window_length) + dry_run = self.config.resolve(dry_run, default=True) if dry_run: self.logger.info('DRY RUN MODE - No files will be downloaded') diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 8cfc16918..10c65aa30 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -14,6 +14,7 @@ from eva.eva_driver import eva +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams @@ -48,7 +49,7 @@ def execute(self) -> None: eva_str_template = eva_config_file_open.read() # Get the paths for the two experiments - experiment_paths = self.config.comparison_experiment_paths() + experiment_paths = self.config.resolve(comparison_experiment_paths) print(experiment_paths) experiment_tag_paths = comparison_tags(experiment_paths, self.logger) diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 7c96a4155..46f5de944 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -13,6 +13,7 @@ from eva.eva_driver import eva +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.comparisons import comparison_tags @@ -31,7 +32,7 @@ def execute(self) -> None: # Get the log type # ---------------- - log_type = self.config.comparison_log_type() + log_type = self.config.resolve(comparison_log_type) # Read Eva template file into dictionary # -------------------------------------- @@ -41,7 +42,7 @@ def execute(self) -> None: eva_str_template = eva_config_file_open.read() # Get the paths for the two experiments - experiment_paths = self.config.comparison_experiment_paths() + experiment_paths = self.config.resolve(comparison_experiment_paths) experiment_tag_paths = comparison_tags(experiment_paths, self.logger) diff --git a/src/swell/tasks/eva_comparison_observations.py b/src/swell/tasks/eva_comparison_observations.py index 59a318cbe..9716a2bc5 100644 --- a/src/swell/tasks/eva_comparison_observations.py +++ b/src/swell/tasks/eva_comparison_observations.py @@ -16,6 +16,7 @@ from swell.swell_path import get_swell_path from swell.deployment.platforms.platforms import login_or_compute +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 @@ -41,11 +42,11 @@ def execute(self) -> None: # Comparison log type # ------------------- - log_type = self.config.comparison_log_type() + log_type = self.config.resolve(comparison_log_type) # Get the experiment paths # ------------------------ - experiment_paths = self.config.comparison_experiment_paths() + experiment_paths = self.config.resolve(comparison_experiment_paths) experiment_tag_paths = comparison_tags(experiment_paths, self.logger) @@ -125,7 +126,7 @@ def execute(self) -> None: eva_dicts = [] # Empty list of dictionaries # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) for observation in observations: if self.get_model() == 'geos_atmosphere': diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index 03c420b26..da2df4eca 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -13,6 +13,7 @@ from eva.eva_driver import eva +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.jinja2 import template_string_jinja2 @@ -26,10 +27,10 @@ def execute(self) -> None: # Get the model and window type # ----------------------------- model = self.get_model() - window_type = self.config.window_type() + window_type = self.config.resolve(window_type) if model == 'geos_marine': - marine_models = self.config.marine_models() + marine_models = self.config.resolve(marine_models) # Read Eva template file into dictionary # -------------------------------------- @@ -46,12 +47,12 @@ def execute(self) -> None: # Create time strings for eva_override directory cycle_time_reformat = self.cycle_time_dto().strftime('%Y%m%d_%H%M%Sz') - window_begin_dto = self.da_window_params.window_begin(self.config.window_length(), + window_begin_dto = self.da_window_params.window_begin(self.config.resolve(window_length), dto=True) window_begin = window_begin_dto.strftime('%Y%m%d_%H%M%Sz') local_bkg_dir, local_bkg_dto = self.da_window_params.local_background_time( - self.config.window_length(), self.config.window_type(), dto=True) + self.config.window_length(), self.config.resolve(window_length), dto=True) local_bkg_time = local_bkg_dto.strftime('%Y%m%d_%H%M%Sz') # Define the increment filename and path diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index a648606d0..3b2fc5465 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -13,6 +13,7 @@ from eva.eva_driver import eva +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.jinja2 import template_string_jinja2 diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index a545e2ad3..455f84db8 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -15,6 +15,7 @@ from eva.eva_driver import eva from swell.deployment.platforms.platforms import login_or_compute +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 @@ -36,24 +37,24 @@ class EvaObservations(taskBase): def execute(self) -> None: - window_length = self.config.window_length() + window_length = self.config.resolve(window_length) # Compute window beginning time # ----------------------------- window_begin = self.da_window_params.window_begin(window_length) background_time = self.da_window_params.background_time( - self.config.background_time_offset()) + self.config.resolve(background_time_offset)) # Create JEDI interface config templates dictionary # ------------------------------------------------- self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Get the model # ------------- model = self.get_model() - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) # Determine if running on login or compute node and set workers # ------------------------------------------------------------- @@ -94,11 +95,11 @@ def execute(self) -> None: eva_dicts = [] # Empty list of dictionaries # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) yaml = YAML(typ='safe') - for observation in self.config.observations(): + for observation in self.config.resolve(observations): # Load the observation dictionary observation_dict = self.jedi_rendering.render_interface_observations(observation) diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index 0610654f6..545e65152 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -17,6 +17,7 @@ from eva.eva_driver import eva from swell.deployment.platforms.platforms import login_or_compute +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary @@ -38,15 +39,15 @@ class EvaTimeseries(taskBase): def execute(self) -> None: - window_length = self.config.window_length() + window_length = self.config.resolve(window_length) # Compute window beginning time # ----------------------------- window_begin = self.da_window_params.window_begin(window_length) background_time = self.da_window_params.background_time( - self.config.background_time_offset()) + self.config.resolve(background_time_offset)) - ncdiag_experiments = self.config.ncdiag_experiments() + ncdiag_experiments = self.config.resolve(ncdiag_experiments) # Use built-in methods to get the start and end cycle points # ---------------------------------------------------------- @@ -70,14 +71,14 @@ def execute(self) -> None: # Create JEDI interface config templates dictionary # ------------------------------------------------- self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) self.jedi_rendering.add_key('window_length', window_length) # Get the model # ------------- model = self.get_model() - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) # Determine if running on login or compute node and set workers # ------------------------------------------------------------- @@ -116,11 +117,11 @@ def execute(self) -> None: eva_dicts = [] # Empty list of dictionaries # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) yaml = YAML(typ='safe') - for observation in self.config.observations(): + for observation in self.config.resolve(observations): # Load the observation dictionary observation_dict = self.jedi_rendering.render_interface_observations(observation) diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index 0dc8f7872..269871794 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -8,6 +8,7 @@ import os from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.shell_commands import run_track_log_subprocess from swell.utilities.file_system_operations import check_if_files_exist_in_path @@ -222,12 +223,12 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_length = self.config.window_length() - window_type = self.config.window_type() - background_error_model = self.config.background_error_model() + window_length = self.config.resolve(window_length) + window_type = self.config.resolve(window_type) + background_error_model = self.config.resolve(background_error_model) - swell_static_files_user = self.config.swell_static_files_user(None) - self.swell_static_files = self.config.swell_static_files() + swell_static_files_user = self.config.resolve(swell_static_files_user, default=None) + self.swell_static_files = self.config.resolve(swell_static_files) # Use static_files_user if present in config and contains files # ------------------------------------------------------------- @@ -237,18 +238,18 @@ def execute(self) -> None: self.logger.info(f'Using swell static files in {swell_static_files_user}') self.swell_static_files = swell_static_files_user - self.horizontal_resolution = self.config.horizontal_resolution() - self.vertical_resolution = self.config.vertical_resolution() - self.generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + self.horizontal_resolution = self.config.resolve(horizontal_resolution) + self.vertical_resolution = self.config.resolve(vertical_resolution) + self.generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) # Get the JEDI interface for this model component # ----------------------------------------------- - self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) - self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) - self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) - self.jedi_rendering.add_key('background_error_model', self.config.background_error_model()) - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.jedi_rendering.add_key('npx_proc', self.config.resolve(npx_proc, default=None)) + self.jedi_rendering.add_key('npy_proc', self.config.resolve(npy_proc, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) + self.jedi_rendering.add_key('background_error_model', self.config.resolve(background_error_model)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) # Compute data assimilation window parameters # ------------------------------------------- local_background_time = self.da_window_params.local_background_time(window_length, diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 11ffe21c1..0fe4b53fb 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -7,6 +7,7 @@ # -------------------------------------------------------------------------------------------------- import os +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import link_all_files_from_first_in_hierarchy_of_sources @@ -41,16 +42,16 @@ def execute(self) -> None: # Get the flavor of static background error model # ----------------------------------------------- - background_error_model = self.config.background_error_model() + background_error_model = self.config.resolve(background_error_model) # Get the JEDI interface for this model component # ----------------------------------------------- - self.jedi_rendering.add_key('background_error_model', self.config.background_error_model()) + self.jedi_rendering.add_key('background_error_model', self.config.resolve(background_error_model)) # Extract general parts of the config # ----------------------------------- - swell_static_files_main = self.config.swell_static_files() - swell_static_files_user = self.config.swell_static_files_user(None) + swell_static_files_main = self.config.resolve(swell_static_files) + swell_static_files_user = self.config.resolve(swell_static_files_user, default=None) # Set the destination directory # ------------------------------ @@ -64,8 +65,8 @@ def execute(self) -> None: # Compute data assimilation window parameters to obtain the local background time # ------------------------------------------------------------------------------- - window_length = self.config.window_length() - window_type = self.config.window_type() + window_length = self.config.resolve(window_length) + window_type = self.config.resolve(window_type) local_background_time = self.da_window_params.local_background_time(window_length, window_type) @@ -79,8 +80,8 @@ def execute(self) -> None: # long run. # ------------------------------------------------------------------------------ if background_error_model == 'explicit_diffusion': - horizontal_resolution = self.config.horizontal_resolution() - vertical_resolution = self.config.vertical_resolution() + horizontal_resolution = self.config.resolve(horizontal_resolution) + vertical_resolution = self.config.resolve(vertical_resolution) res_path = horizontal_resolution + 'x' + vertical_resolution correlation_files = 'vt.' + local_background_time + '.nc' diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index bf8d71fdc..6dc678496 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -10,6 +10,7 @@ import os +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.observing_system_records import ObservingSystemRecords @@ -32,21 +33,21 @@ def execute(self) -> None: # Parse GSI records and save yamls # -------------------------------- - observing_system_records_path = self.config.observing_system_records_path(None) + observing_system_records_path = self.config.resolve(observing_system_records_path, default=None) if observing_system_records_path == 'None': cycle_dir = self.cycle_dir() observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') observing_system_records_path = os.path.expanduser(observing_system_records_path) - path_to_geos_mksi = os.path.expanduser(self.config.observing_system_records_mksi_path()) + path_to_geos_mksi = os.path.expanduser(self.config.resolve(observing_system_records_mksi_path)) if path_to_geos_mksi == 'None': path_to_geos_mksi = os.path.join(self.experiment_path(), 'GEOS_mksi') # Parse sidb # ---------- record_type = 'channel' - observations = self.config.observations() + observations = self.config.resolve(observations) path_to_gsi_records = os.path.join(path_to_geos_mksi, 'sidb') sat_records = ObservingSystemRecords(record_type) sat_records.parse_records(path_to_gsi_records) diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index afb92f1d1..b964cd91a 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -8,6 +8,7 @@ # -------------------------------------------------------------------------------------------------- +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials, get_r2d2_model_name @@ -55,29 +56,29 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.r2d2_server(default=None), + r2d2_server=self.config.resolve(r2d2_server, default=None), ) - r2d2_datastore = self.config.r2d2_datastore(default=None) + r2d2_datastore = self.config.resolve(r2d2_datastore, default=None) # Get duration into forecast for first background file # ---------------------------------------------------- bkg_steps = [] # Parse config - background_frequency = self.config.background_frequency(None) - horizontal_resolution = self.config.horizontal_resolution() - window_length = self.config.window_length() - window_type = self.config.window_type() + background_frequency = self.config.resolve(background_frequency, default=None) + horizontal_resolution = self.config.resolve(horizontal_resolution) + window_length = self.config.resolve(window_length) + window_type = self.config.resolve(window_type) # For experiments with cycle in the suite name: # for the first cycle, use background_experiment in config # as the experiment id for fetching from r2d2 for cycles after # the first, use the current experiment id for fetching from r2d2 if self.cycle_time_dto() != self.start_cycle_point_dto() and 'cycle' in self.suite_name(): - background_experiment = self.config.r2d2_experiment_id() + background_experiment = self.config.resolve(r2d2_experiment_id) else: - background_experiment = self.config.background_experiment() + background_experiment = self.config.resolve(background_experiment) self.logger.info(f'Fetching background from experiment {background_experiment}') @@ -88,7 +89,7 @@ def execute(self) -> None: # Add to jedi config rendering dictionary self.jedi_rendering.add_key('local_background_time', local_background_time) - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) self.jedi_rendering.add_key('analysis_time_iso', analysis_time_iso) # Convert to datetime durations diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index a08406c52..2417e6494 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -13,6 +13,7 @@ import os import tarfile +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats @@ -44,9 +45,9 @@ def execute(self): # Parse config # ------------ - background_experiment = self.config.background_experiment() - geos_x_background_directory = self.config.geos_x_background_directory() - background_time_offset = self.config.background_time_offset() + background_experiment = self.config.resolve(background_experiment) + geos_x_background_directory = self.config.resolve(geos_x_background_directory) + background_time_offset = self.config.resolve(background_time_offset) # Since this is an optional task, check if the geos_x_background_directory is # set to /dev/null, if so fail the task diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index 357839fd3..1c406b143 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -13,6 +13,7 @@ from datetime import datetime as dt from swell.utilities.datetime_util import datetime_formats +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -43,7 +44,7 @@ def execute(self) -> None: perl_executable_path = os.path.join(self.experiment_path(), 'GMAO_perllib') geos_mksi_obsysrc_path = os.path.join(self.experiment_path(), - self.config.obs_rc_path()) + self.config.resolve(obs_rc_path)) # Environment variables for acquire_obsys env_dict = os.environ @@ -59,7 +60,7 @@ def execute(self) -> None: # Get BUFR obs classes from mksi and acquire them # ----------------------------------------------- - obsclasses = self.config.bufr_obs_classes() + obsclasses = self.config.resolve(bufr_obs_classes) for obsclass in obsclasses: diff --git a/src/swell/tasks/get_coupled_geos_restart.py b/src/swell/tasks/get_coupled_geos_restart.py index 75c1d57c0..891c40360 100644 --- a/src/swell/tasks/get_coupled_geos_restart.py +++ b/src/swell/tasks/get_coupled_geos_restart.py @@ -10,6 +10,7 @@ import os import glob +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import copy_to_dst_dir @@ -47,7 +48,7 @@ def execute(self) -> None: swell_exp_path = self.experiment_path() # Obtain GEOS HOMDIR from user input - self.geos_homdir = self.config.geos_homdir() + self.geos_homdir = self.config.resolve(geos_homdir) self.logger.info(f'GEOS HOME directory: {self.geos_homdir}') # Create GEOSgcm directory in the experiment folder if it doesn't exist yet @@ -69,8 +70,8 @@ def execute(self) -> None: geos_expdir_path = geos_homdir_path # If GEOS expdir is set to be different to homdir, create a link to expdir - if self.config.geos_expdir_different(): - self.geos_expdir = self.config.geos_expdir() + if self.config.resolve(geos_expdir_different): + self.geos_expdir = self.config.resolve(geos_expdir) self.logger.info(f'GEOS EXPERIMENT directory: {self.geos_expdir}') if not os.path.exists(self.geos_expdir): @@ -108,7 +109,7 @@ def initial_restarts(self, geos_expdir_path: str) -> None: # 3) Hotstart, just use the existing restarts in the forecast directory (e.g., from a # previous run or manually placed there) # ---------------------------------------------------- - initial_restarts_method = self.config.initial_restarts_method('geos_expdir') + initial_restarts_method = self.config.resolve(initial_restarts_method, default='geos_expdir') if initial_restarts_method == 'geos_expdir': self.initial_restarts_from_directory(geos_expdir_path) diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index 178967987..ea234471b 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -12,6 +12,7 @@ import os import tarfile +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats @@ -43,9 +44,9 @@ def execute(self): # Parse config # ------------ - background_experiment = self.config.background_experiment() - geos_x_ensemble_directory = self.config.geos_x_ensemble_directory() - background_time_offset = self.config.ebkg_time_offset() + background_experiment = self.config.resolve(background_experiment) + geos_x_ensemble_directory = self.config.resolve(geos_x_ensemble_directory) + background_time_offset = self.config.resolve(ebkg_time_offset) # Since this is an optional task, check if the geos_x_ensemble_directory is # set to /dev/null, if so fail the task diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index 354d53515..ec83cf31e 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -13,6 +13,7 @@ import os import re +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase @@ -25,7 +26,7 @@ def execute(self) -> None: # Get the path and pattern for the background files # ------------------------------------------------- - background_path = self.config.path_to_geos_adas_background() + background_path = self.config.resolve(path_to_geos_adas_background) # Get list of ncdiags to test with # -------------------------------- diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index 2f04c884f..257cd2526 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -10,6 +10,7 @@ import os +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials from r2d2 import fetch @@ -26,17 +27,17 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.r2d2_server(default=None), + r2d2_server=self.config.resolve(r2d2_server, default=None), ) # Parse config # ------------ - geovals_experiment = self.config.geovals_experiment() - geovals_provider = self.config.geovals_provider() - background_time_offset = self.config.background_time_offset() - observations = self.config.observations() - window_length = self.config.window_length() - crtm_coeff_dir = self.config.crtm_coeff_dir(None) + geovals_experiment = self.config.resolve(geovals_experiment) + geovals_provider = self.config.resolve(geovals_provider) + background_time_offset = self.config.resolve(background_time_offset) + observations = self.config.resolve(observations) + window_length = self.config.resolve(window_length) + crtm_coeff_dir = self.config.resolve(crtm_coeff_dir, default=None) # Get window begin time window_begin = self.da_window_params.window_begin(window_length) diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index 9ca84ae20..86050c2fe 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -14,6 +14,7 @@ import shutil import tarfile +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase @@ -26,8 +27,8 @@ def execute(self) -> None: # Get the build method # -------------------- - gsi_bc_location = self.config.path_to_gsi_bc_coefficients() - window_length = self.config.window_length() + gsi_bc_location = self.config.resolve(path_to_gsi_bc_coefficients) + window_length = self.config.resolve(window_length) # Time of GSI analysis providing the bias correction coefficients # --------------------------------------------------------------- diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index 3567e9581..b58d4796f 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -11,6 +11,7 @@ import glob import os +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase @@ -23,7 +24,7 @@ def execute(self) -> None: # Get the build method # -------------------- - gsi_diag_path = self.config.path_to_gsi_nc_diags() + gsi_diag_path = self.config.resolve(path_to_gsi_nc_diags) # Replace gsi_diag_path datetime string with the actual datetime # -------------------------------------------------------------- diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index 9e0a0aff7..97a275592 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -8,6 +8,7 @@ # -------------------------------------------------------------------------------------------------- import os +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from r2d2 import fetch from swell.utilities.r2d2 import load_r2d2_credentials @@ -28,15 +29,15 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.r2d2_server(default=None), + r2d2_server=self.config.resolve(r2d2_server, default=None), ) # Parse config # ------------ - ncdiag_experiments = self.config.ncdiag_experiments() - observations = self.config.observations() - window_length = self.config.window_length() - background_time_offset = self.config.background_time_offset() + ncdiag_experiments = self.config.resolve(ncdiag_experiments) + observations = self.config.resolve(observations) + window_length = self.config.resolve(window_length) + background_time_offset = self.config.resolve(background_time_offset) # Compute data assimilation window parameters # -------------------------------------------- @@ -46,11 +47,11 @@ def execute(self) -> None: self.jedi_rendering.add_key('background_time', background_time) # Set the JEDI rendering parameters. Model specific ones have None as default - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) # Loop over ncdiag experiments # ------------------------------- diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index 2c218bc30..e09c139a6 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -12,6 +12,7 @@ import os import subprocess +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase @@ -28,7 +29,7 @@ def execute(self) -> None: # Get the path and pattern for the observation files # ------------------------------------------------- - existing_path = self.config.ioda_locations_not_in_r2d2() + existing_path = self.config.resolve(ioda_locations_not_in_r2d2) # Point to the model directory # ---------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index b792500f1..0f7161701 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -18,6 +18,7 @@ from concurrent.futures import ThreadPoolExecutor from datetime import timedelta, datetime as dt +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities import s3 as swell_s3 from swell.utilities.r2d2 import load_r2d2_credentials, get_r2d2_model_name @@ -210,32 +211,32 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.r2d2_server(default=None), + r2d2_server=self.config.resolve(r2d2_server, default=None), ) - r2d2_datastore = self.config.r2d2_datastore(default=None) + r2d2_datastore = self.config.resolve(r2d2_datastore, default=None) # Parse config # ------------ - obs_experiment = self.config.obs_experiment() - background_time_offset = self.config.background_time_offset() - observations = self.config.observations() - observation_providers = self.config.observation_providers(default={}) - window_length = self.config.window_length() - crtm_coeff_dir = self.config.crtm_coeff_dir(None) - window_length = self.config.window_length() - cycling_varbc = self.config.cycling_varbc(None) - cache_fetch = self.config.cache_fetch(True) + obs_experiment = self.config.resolve(obs_experiment) + background_time_offset = self.config.resolve(background_time_offset) + observations = self.config.resolve(observations) + observation_providers = self.config.resolve(observation_providers, default={}) + window_length = self.config.resolve(window_length) + crtm_coeff_dir = self.config.resolve(crtm_coeff_dir, default=None) + window_length = self.config.resolve(window_length) + cycling_varbc = self.config.resolve(cycling_varbc, default=None) + cache_fetch = self.config.resolve(cache_fetch, default=True) # When True, observations that have a fetch_observations_s3 # registry entry are pulled directly from a public S3 bucket instead of # R2D2. When False (default) every observation goes through R2D2 as before. - fetch_obs_from_public_s3 = self.config.fetch_obs_from_public_s3(False) + fetch_obs_from_public_s3 = self.config.resolve(fetch_obs_from_public_s3, default=False) # Get model component and translate to R2D2 model name model_component = self.get_model() r2d2_model = get_r2d2_model_name(model_component) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) # Get window begin time window_begin = self.da_window_params.window_begin(window_length) @@ -259,7 +260,7 @@ def execute(self) -> None: self.jedi_rendering.add_key('background_time', background_time) self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) self.jedi_rendering.add_key('window_begin', window_begin) - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) # Read observation ioda names ioda_names_list = get_ioda_names_list() diff --git a/src/swell/tasks/get_restart_cf.py b/src/swell/tasks/get_restart_cf.py index 93f768811..4caa701da 100644 --- a/src/swell/tasks/get_restart_cf.py +++ b/src/swell/tasks/get_restart_cf.py @@ -8,6 +8,7 @@ # -------------------------------------------------------------------------------------------------- +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials @@ -29,15 +30,15 @@ def execute(self) -> None: # --------------------- load_r2d2_credentials(self.logger, self.platform()) - rst_file_types = self.config.rst_file_types() + rst_file_types = self.config.resolve(rst_file_types) cycle_dir = self.cycle_dir() scratch_dir = os.path.join(cycle_dir, 'scratch') model = self.__model__ - horizontal_resolution = self.config.horizontal_resolution() + horizontal_resolution = self.config.resolve(horizontal_resolution) - window_length = self.config.window_length() + window_length = self.config.resolve(window_length) window_begin = self.da_window_params.window_begin(window_length, dto=True) window_begin_prev = window_begin - isodate.parse_duration(window_length) @@ -45,9 +46,9 @@ def execute(self) -> None: # Use rst_experiment for first cycle if self.cycle_time_dto() == self.start_cycle_point_dto(): - rst_exp = self.config.rst_experiment() + rst_exp = self.config.resolve(rst_experiment) else: - rst_exp = self.config.r2d2_experiment_id() + rst_exp = self.config.resolve(r2d2_experiment_id) self.logger.info(f'Fetching rst files from experiment {rst_exp}') diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index e86e61921..859a69785 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -12,6 +12,7 @@ import os +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.dictionary import write_dict_to_yaml from swell.utilities.shell_commands import run_track_log_subprocess @@ -26,10 +27,10 @@ def execute(self) -> None: # Parse configuration # ------------------- - observations = self.config.observations() - background_time_offset = self.config.background_time_offset() - window_length = self.config.window_length() - crtm_coeff_dir = self.config.crtm_coeff_dir(None) + observations = self.config.resolve(observations) + background_time_offset = self.config.resolve(background_time_offset) + window_length = self.config.resolve(window_length) + crtm_coeff_dir = self.config.resolve(crtm_coeff_dir, default=None) # Get window beginning time window_begin = self.da_window_params.window_begin(window_length) @@ -47,7 +48,7 @@ def execute(self) -> None: sensors_tlapse = [] # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) for observation in observations: diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 2e036f5a5..9cda64983 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -18,6 +18,7 @@ import pyiodaconv.gsi_ncdiag as gsid from pyiodaconv.combine_obsspace import combine_obsspace +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats from swell.utilities.shell_commands import run_subprocess, create_executable_file @@ -32,10 +33,10 @@ def execute(self) -> None: # Parse configuration # ------------------- - observations = self.config.observations() - single_observations = self.config.single_observations() - produce_geovals = self.config.produce_geovals() - window_length = self.config.window_length() + observations = self.config.resolve(observations) + single_observations = self.config.resolve(single_observations) + produce_geovals = self.config.resolve(produce_geovals) + window_length = self.config.resolve(window_length) # Get window beginning time window_begin = self.da_window_params.window_begin(window_length) diff --git a/src/swell/tasks/ingest_obs.py b/src/swell/tasks/ingest_obs.py index c47a12702..697b5f1bb 100644 --- a/src/swell/tasks/ingest_obs.py +++ b/src/swell/tasks/ingest_obs.py @@ -17,6 +17,7 @@ import requests +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials from swell.utilities.observations import get_ioda_names_list, get_provider_for_observation @@ -72,19 +73,19 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.r2d2_server(default=None), + r2d2_server=self.config.resolve(r2d2_server, default=None), ) - r2d2_datastore = self.config.r2d2_datastore(default=None) + r2d2_datastore = self.config.resolve(r2d2_datastore, default=None) # Get list of observations to ingest (strings) - obs_to_ingest = self.config.obs_to_ingest([]) + obs_to_ingest = self.config.resolve(obs_to_ingest, default=[]) # Read observation ioda names (for provider lookup) self.ioda_names_list = get_ioda_names_list() # Get window parameters - window_length = self.config.window_length() + window_length = self.config.resolve(window_length) # Beginning of the DA window window_start = self.da_window_params.window_begin_iso(window_length) @@ -93,13 +94,13 @@ def execute(self) -> None: cycle_time = self.cycle_time() # Check for dry-run mode (default True for safety) - dry_run = self.config.dry_run(True) + dry_run = self.config.resolve(dry_run, default=True) if dry_run: self.logger.info( "DRY RUN MODE - No files will be ingested to R2D2") - store_as_symlink = self.config.store_as_symlink(False) + store_as_symlink = self.config.resolve(store_as_symlink, default=False) total_ingested = 0 total_failed = 0 diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index ebb081d6f..11324937b 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -13,6 +13,7 @@ import re import numpy as np +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.comparisons import comparison_tags @@ -27,7 +28,7 @@ class JediLogComparison(taskBase): def execute(self): - experiment_paths = self.config.comparison_experiment_paths() + experiment_paths = self.config.resolve(comparison_experiment_paths) experiment_tag_paths = comparison_tags(experiment_paths, self.logger) @@ -59,7 +60,7 @@ def execute(self): # Boolean for whether fields fall within tolerances passed = True - log_type = self.config.comparison_log_type() + log_type = self.config.resolve(comparison_log_type) for exp_tag, experiment_path in experiment_tag_paths.items(): diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index 543f77dda..abd75dd5e 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -11,6 +11,7 @@ import os import subprocess +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -25,7 +26,7 @@ def fgrep_residual_norm(self, output_file): cycle_time = self.__datetime__.string_directory() model = self.get_model() - log_type = self.config.comparison_log_type() + log_type = self.config.resolve(comparison_log_type) # Build the command command = ['fgrep', '"Residual norm"'] + [ @@ -48,7 +49,7 @@ def execute(self) -> None: output_file = os.path.join(self.cycle_dir(), 'jedi_log_analysis.txt') - for parser_option in self.config.parser_options(['fgrep_residual_norm']): + for parser_option in self.config.resolve(parser_options, default=['fgrep_residual_norm']): if parser_option == 'fgrep_residual_norm': self.fgrep_residual_norm(output_file) diff --git a/src/swell/tasks/link_coupled_geos_output.py b/src/swell/tasks/link_coupled_geos_output.py index fb566c6c1..f0847c576 100644 --- a/src/swell/tasks/link_coupled_geos_output.py +++ b/src/swell/tasks/link_coupled_geos_output.py @@ -16,6 +16,7 @@ from typing import Tuple from swell.utilities.datetime_util import datetime_formats +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -36,13 +37,13 @@ def execute(self) -> None: # Parse configuration # ------------------- - self.marine_models = self.config.marine_models(None) or [] - self.window_type = self.config.window_type() - self.window_length = self.config.window_length() + self.marine_models = self.config.resolve(marine_models, default=None) or [] + self.window_type = self.config.resolve(window_type) + self.window_length = self.config.resolve(window_length) self.window_begin_iso = self.da_window_params.window_begin_iso(self.window_length) if self.window_type == '4D' or 'fgat' in self.suite_name(): - self.background_frequency = self.config.background_frequency() + self.background_frequency = self.config.resolve(background_frequency) self.bkgr_time_iso, self.bkgr_time_dto = self.da_window_params.local_background_time( self.window_length, diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index f2853728a..fca1bb95c 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -16,6 +16,7 @@ from typing import Tuple from swell.utilities.datetime_util import datetime_formats +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -35,13 +36,13 @@ def execute(self) -> None: # Parse configuration # ------------------- - self.marine_models = self.config.marine_models(None) or [] - self.window_type = self.config.window_type() - self.window_length = self.config.window_length() + self.marine_models = self.config.resolve(marine_models, default=None) or [] + self.window_type = self.config.resolve(window_type) + self.window_length = self.config.resolve(window_length) self.window_begin_iso = self.da_window_params.window_begin_iso(self.window_length) if self.window_type == '4D' or 'fgat' in self.suite_name(): - self.background_frequency = self.config.background_frequency() + self.background_frequency = self.config.resolve(background_frequency) self.bkgr_time_iso, self.bkgr_time_dto = self.da_window_params.local_background_time( self.window_length, diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index deec7de29..2ac88a570 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -12,6 +12,7 @@ import os import re +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import move_files @@ -37,8 +38,8 @@ def execute(self) -> None: # Obtain MOM6 IAU bool # ---------------------- - self.mom6_iau = self.config.mom6_iau() - self.jedi_rendering.add_key('mom6_iau', self.config.mom6_iau(False)) + self.mom6_iau = self.config.resolve(mom6_iau) + self.jedi_rendering.add_key('mom6_iau', self.config.resolve(mom6_iau, default=False)) # Current and restart time objects # -------------------------------- @@ -76,7 +77,7 @@ def move_restarts(self) -> None: if int(agcm_dict.get('RECORD_FREQUENCY', '0')) > 0: self.logger.info('RECORD_FREQUENCY is turned on in AGCM.rc. Using _checkpoint files ' 'with timestamps.') - window_length = self.config.window_length() + window_length = self.config.resolve(window_length) an_fcst_offset = self.da_window_params.analysis_forecast_window_offset(window_length) rst_dto = self.cc_dto + isodate.parse_duration(an_fcst_offset) @@ -118,7 +119,7 @@ def move_marine_restarts(self) -> None: if int(agcm_dict.get('RECORD_FREQUENCY', '0')) > 0: - window_length = self.config.window_length() + window_length = self.config.resolve(window_length) an_fcst_offset = self.da_window_params.analysis_forecast_window_offset(window_length) rst_dto = self.cc_dto + isodate.parse_duration(an_fcst_offset) diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index e82f62565..e5a32da82 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -10,6 +10,7 @@ import os import glob +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import move_files @@ -32,7 +33,7 @@ def execute(self) -> None: # Next cycle folder name # ----------------------- - self.forecast_duration = self.config.forecast_duration() + self.forecast_duration = self.config.resolve(forecast_duration) # Create cycle_dir and RESTART # ---------------------------- diff --git a/src/swell/tasks/prep_coupled_geos_run_dir.py b/src/swell/tasks/prep_coupled_geos_run_dir.py index 84f1bb1c6..7ce19e8f4 100644 --- a/src/swell/tasks/prep_coupled_geos_run_dir.py +++ b/src/swell/tasks/prep_coupled_geos_run_dir.py @@ -11,6 +11,7 @@ import isodate import re +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import copy_to_dst_dir @@ -43,12 +44,12 @@ def execute(self) -> None: # config.s to the cycle forecast directory self.geos_homdir = os.path.join(self.experiment_path(), 'GEOSgcm', 'GEOS_homdir') - if self.config.geos_expdir_different(): + if self.config.resolve(geos_expdir_different): self.geos_expdir = os.path.join(self.experiment_path(), 'GEOSgcm', 'GEOS_expdir') else: self.geos_expdir = self.geos_homdir - self.geos_build = self.config.existing_geos_gcm_build_path() + self.geos_build = self.config.resolve(existing_geos_gcm_build_path) self.logger.info('Preparing GEOS Forecast directory......') self.logger.info('Some steps involve modifying input files and replacing file contents.') @@ -59,7 +60,7 @@ def execute(self) -> None: # context. So forecast start will be assigned as cycle_date - forecast_duration * 3/4 # Need to convert forecast_duration to a datetime object first # ------------------------------------------------------------- - self.forecast_duration = self.config.forecast_duration() + self.forecast_duration = self.config.resolve(forecast_duration) self.fc_dto = self.cycle_time_dto() - isodate.parse_duration(self.forecast_duration) * 3 / 4 # Get static files @@ -196,7 +197,7 @@ def mom6_iau(self) -> None: # file is located inside the INPUT directory. At the first cycle, mom6_increment.nc may not # be present in the INPUT directory, so this step is skipped. # -------------------------------------------------------------------------- - if self.config.get_key_for_model('mom6_iau', 'geos_marine', False): + if self.config.resolve(get_key_for_model, default='mom6_iau', 'geos_marine', False): if os.path.exists(self.forecast_dir('RESTART/mom6_increment.nc')): self.logger.info('MOM6 Increment file found in RESTART directory') @@ -206,8 +207,8 @@ def mom6_iau(self) -> None: mom_oda_incupd = self.forecast_dir('MOM_oda_incupd') mom6_config = self.geos.parse_mom6_input(mom_oda_incupd) # P50D is just a random input for get_key_for_model to function - mom6_iau_nhours = self.config.get_key_for_model('mom6_iau_nhours', 'geos_marine', - 'PT50D') +self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)6self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)hself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)=self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)lself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',).self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)cself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',).self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)tself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)kself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)yself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)dself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)lself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)(self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)6self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)hself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',),self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',),self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) +self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) 'PT50D') # convert ISO to 3.0 duration = isodate.parse_duration(mom6_iau_nhours) @@ -261,7 +262,7 @@ def rewrite_cap(self, rcdict: dict, rcfile: str) -> dict: # This method returns rcdict with the bool fix # --------------------------------------------- self.logger.info('Modifying CAP.rc') - [time_string, days, _] = self.geos.iso_to_time_str(self.config.forecast_duration()) + [time_string, days, _] = self.geos.iso_to_time_str(self.config.resolve(forecast_duration)) # Prepend day information # ----------------------- diff --git a/src/swell/tasks/prep_forecast_cf.py b/src/swell/tasks/prep_forecast_cf.py index 0b10766e2..7da2291af 100644 --- a/src/swell/tasks/prep_forecast_cf.py +++ b/src/swell/tasks/prep_forecast_cf.py @@ -15,6 +15,7 @@ import xarray as xr from swell.configuration.jedi.interfaces.geos_cf.model.r2d2 import forecast_history +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.shell_commands import run_subprocess @@ -47,20 +48,20 @@ def execute(self) -> None: # Gather config values # -------------------- self.expid = self.experiment_id() - self.window_length = self.config.window_length() - self.forecast_length = self.config.forecast_length() - self.forecast_output_frequency = self.config.forecast_output_frequency() - self.resolution = self.config.horizontal_resolution() - self.an_vars_long = self.config.analysis_variables() + self.window_length = self.config.resolve(window_length) + self.forecast_length = self.config.resolve(forecast_length) + self.forecast_output_frequency = self.config.resolve(forecast_output_frequency) + self.resolution = self.config.resolve(horizontal_resolution) + self.an_vars_long = self.config.resolve(analysis_variables) - self.geos_cf_run_dir = self.config.geos_cf_run_dir() - self.geos_cf_install_dir = self.config.geos_cf_install_dir() + self.geos_cf_run_dir = self.config.resolve(geos_cf_run_dir) + self.geos_cf_install_dir = self.config.resolve(geos_cf_install_dir) self.namelists_dir = os.path.join(self.experiment_config_path(), 'jedi', 'interfaces', 'geos_cf', 'namelists') - self.fp_exp = self.config.geosfp_exp() - self.fp_loc = self.config.geosfp_path() + self.fp_exp = self.config.resolve(geosfp_exp) + self.fp_loc = self.config.resolve(geosfp_path) # Derive window times # ------------------- @@ -110,7 +111,7 @@ def replace_string(self, filename: str, string1: str, string2: str) -> None: def create_geos_cf_increments(self) -> None: """Convert JEDI increment files to GEOS-CF format using the increment template.""" - inc_template = self.config.inc_template() + inc_template = self.config.resolve(inc_template) tstring_date = (self.parse_andate.strftime('%Y-%m-%d') + ' ' + self.parse_andate.strftime('%H:%M:%S')) diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index 50738c963..eb032411d 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -15,6 +15,7 @@ from typing import Union from swell.utilities.shell_commands import run_subprocess +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -40,8 +41,8 @@ def execute(self) -> None: # This will change with different model types # -------------------------------- - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) - self.jedi_rendering.add_key('mom6_iau', self.config.mom6_iau(False)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('mom6_iau', self.config.resolve(mom6_iau, default=False)) model_component_meta = self.jedi_rendering.render_interface_meta() @@ -49,7 +50,7 @@ def execute(self) -> None: # -------------------------------- self.cc_dto = self.cycle_time_dto() - window_length = self.config.window_length() + window_length = self.config.resolve(window_length) # GEOS restarts have seconds in their filename # We want to use rst_dto at the beginning of the DA window (window offset is negative) @@ -75,7 +76,7 @@ def execute(self) -> None: # Obtain MOM6 IAU bool # ---------------------- - mom6_iau = self.config.mom6_iau() + mom6_iau = self.config.resolve(mom6_iau) # Generic rst file format # ------------------------ @@ -90,7 +91,7 @@ def execute(self) -> None: f_rst = self.forecast_dir(['scratch', 'RESTART', rst_dto.strftime('MOM.res_Y%Y_D%j_S') + seconds + '.nc']) - self.soca_ana = self.config.analysis_variables() + self.soca_ana = self.config.resolve(analysis_variables) if mom6_iau: self.mom6_increment(f_rst, ana_path, incr_path) diff --git a/src/swell/tasks/publish_comparisons.py b/src/swell/tasks/publish_comparisons.py index 553066c7d..d6663fba4 100644 --- a/src/swell/tasks/publish_comparisons.py +++ b/src/swell/tasks/publish_comparisons.py @@ -11,6 +11,7 @@ import shutil from pathlib import Path +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -26,7 +27,7 @@ class PublishComparisons(taskBase): def execute(self) -> None: # Output path base to copy files to - publish_directory = self.config.publish_directory(None) + publish_directory = self.config.resolve(publish_directory, default=None) # Skip this task if there is no publish directory if publish_directory is None: @@ -40,7 +41,7 @@ def execute(self) -> None: experiment_id = github_run_id / experiment_id # Name the location after the experiment ID - publish_location = Path(self.config.publish_directory()) / experiment_id + publish_location = Path(self.config.resolve(publish_directory)) / experiment_id self.logger.info(f'Copying comparison test results to {publish_location}') diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index a2c43f26a..2602886c4 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -11,6 +11,7 @@ import os from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import check_obs @@ -22,26 +23,26 @@ class RenderJediObservations(taskBase): def execute(self) -> None: # List of observations - obs_list = self.config.observations() + obs_list = self.config.resolve(observations) # Whether to run get_channels for obs - check_for_obs = self.config.check_for_obs(True) + check_for_obs = self.config.resolve(check_for_obs, default=True) # Observing system records paths - observing_system_records_path = self.config.observing_system_records_path(None) + observing_system_records_path = self.config.resolve(observing_system_records_path, default=None) self.jedi_rendering.set_obs_records_path(observing_system_records_path) # Marine models - marine_models = self.config.marine_models(None) + marine_models = self.config.resolve(marine_models, default=None) # Window parameters - window_length = self.config.window_length() - background_time_offset = self.config.background_time_offset() + window_length = self.config.resolve(window_length) + background_time_offset = self.config.resolve(background_time_offset) # Window parameters for observations window_begin = self.da_window_params.window_begin(window_length) background_time = self.da_window_params.background_time(background_time_offset) - crtm_coeff_dir = self.config.crtm_coeff_dir(None) + crtm_coeff_dir = self.config.resolve(crtm_coeff_dir, default=None) # Set fields for obs files self.jedi_rendering.add_key('window_begin', window_begin) @@ -51,7 +52,7 @@ def execute(self) -> None: cwd = os.getcwd() - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('experiment_root', 'experiment_root') diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index 297f0bfb3..400ee6151 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -11,6 +11,7 @@ import os from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -28,10 +29,10 @@ def execute(self) -> None: # --------------------- jedi_application = 'convert_state_soca2cice' - jedi_forecast_model = self.config.jedi_forecast_model(None) - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - window_type = self.config.window_type() - window_length = self.config.window_length() + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) # Compute data assimilation window parameters # -------------------------------------------- @@ -44,8 +45,8 @@ def execute(self) -> None: # Populate jedi interface templates dictionary # -------------------------------------------- - self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) # Background and analysis times # ----------------------------- @@ -56,14 +57,14 @@ def execute(self) -> None: # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') # Geometry # -------- - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) # Jedi configuration file # ----------------------- diff --git a/src/swell/tasks/run_jedi_diffstates.py b/src/swell/tasks/run_jedi_diffstates.py index e86fc7319..fb1424388 100644 --- a/src/swell/tasks/run_jedi_diffstates.py +++ b/src/swell/tasks/run_jedi_diffstates.py @@ -11,6 +11,7 @@ import os from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -30,14 +31,14 @@ def execute(self) -> None: # Parse configuration # ------------------- - jedi_forecast_model = self.config.jedi_forecast_model(None) - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - npx_proc = self.config.npx_proc(None) - npy_proc = self.config.npy_proc(None) + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + npx_proc = self.config.resolve(npx_proc, default=None) + npy_proc = self.config.resolve(npy_proc, default=None) # Compute data assimilation window parameters - window_type = self.config.window_type() - window_length = self.config.window_length() + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) local_background_time = self.da_window_params.local_background_time(window_length, window_type) local_background_time_iso = self.da_window_params.local_background_time_iso(window_length, @@ -46,33 +47,33 @@ def execute(self) -> None: window_end_iso = self.da_window_params.window_end_iso(window_length) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) # Populate jedi interface templates dictionary # -------------------------------------------- self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) - self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) # Background - self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) # Diffstates - spec_dict = self.config.diffstates_spec() + spec_dict = self.config.resolve(diffstates_spec) self.jedi_rendering.add_key('diffstates_spec', spec_dict) self.logger.info(f'diffstates = {spec_dict}') # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/run_jedi_eda_executable.py b/src/swell/tasks/run_jedi_eda_executable.py index a9011f723..5c5c48ae0 100644 --- a/src/swell/tasks/run_jedi_eda_executable.py +++ b/src/swell/tasks/run_jedi_eda_executable.py @@ -12,6 +12,7 @@ import shutil from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable from swell.utilities.yaml_utils import replace_string_value @@ -31,25 +32,25 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_type = self.config.window_type() - window_length = self.config.window_length() - forecast_length = self.config.forecast_length(window_length) - background_time_offset = self.config.background_time_offset() - number_of_iterations = self.config.number_of_iterations() - jedi_forecast_model = self.config.jedi_forecast_model(None) - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - perhost = self.config.perhost(None) + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) + forecast_length = self.config.resolve(forecast_length, default=window_length) + background_time_offset = self.config.resolve(background_time_offset) + number_of_iterations = self.config.resolve(number_of_iterations) + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + perhost = self.config.resolve(perhost, default=None) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) - gsibec_nlats = self.config.gsibec_nlats(None) - gsibec_nlons = self.config.gsibec_nlons(None) - gsibec_configuration = self.config.gsibec_configuration(None) - npx_proc = self.config.npx_proc(None) - npy_proc = self.config.npy_proc(None) - npx = self.config.npx(None) - npy = self.config.npy(None) + gsibec_nlats = self.config.resolve(gsibec_nlats, default=None) + gsibec_nlons = self.config.resolve(gsibec_nlons, default=None) + gsibec_configuration = self.config.resolve(gsibec_configuration, default=None) + npx_proc = self.config.resolve(npx_proc, default=None) + npy_proc = self.config.resolve(npy_proc, default=None) + npx = self.config.resolve(npx, default=None) + npy = self.config.resolve(npy, default=None) # Compute data assimilation window parameters # -------------------------------------------- @@ -61,8 +62,8 @@ def execute(self) -> None: window_begin = self.da_window_params.window_begin(window_length) window_begin_iso = self.da_window_params.window_begin_iso(window_length) window_end_iso = self.da_window_params.window_end_iso(window_length) - obs_pert_amplitude = self.config.obs_pert_amplitude() - nmember = self.config.ensemble_num_members() + obs_pert_amplitude = self.config.resolve(obs_pert_amplitude) + nmember = self.config.resolve(ensemble_num_members) imember = self.get_ensemble_imember() # Populate jedi interface templates dictionary @@ -71,38 +72,38 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) self.jedi_rendering.add_key('forecast_length', forecast_length) - self.jedi_rendering.add_key('minimizer', self.config.minimizer()) + self.jedi_rendering.add_key('minimizer', self.config.resolve(minimizer)) self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) - self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) - self.jedi_rendering.add_key('saber_central_block', self.config.saber_central_block(None)) - self.jedi_rendering.add_key('saber_outer_block', self.config.saber_outer_block(None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) + self.jedi_rendering.add_key('saber_central_block', self.config.resolve(saber_central_block, default=None)) + self.jedi_rendering.add_key('saber_outer_block', self.config.resolve(saber_outer_block, default=None)) self.jedi_rendering.add_key('gradient_norm_reduction', - self.config.gradient_norm_reduction()) - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.config.resolve(gradient_norm_reduction)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) - self.jedi_rendering.add_key('ensemble_num_members', self.config.ensemble_num_members()) + self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(ensemble_num_members)) self.jedi_rendering.add_key('ensemble_imember', imember) # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) self.jedi_rendering.add_key('npx', npx) self.jedi_rendering.add_key('npy', npy) - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Atmosphere background error model @@ -117,7 +118,7 @@ def execute(self) -> None: # Model # ----- if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.background_frequency()) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) # Jedi configuration file # ----------------------- diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index d57444650..2e6ec265d 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -11,6 +11,7 @@ import os from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -30,14 +31,14 @@ def execute(self) -> None: # Parse configuration # ------------------- - jedi_forecast_model = self.config.jedi_forecast_model(None) - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - npx_proc = self.config.npx_proc(None) - npy_proc = self.config.npy_proc(None) + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + npx_proc = self.config.resolve(npx_proc, default=None) + npy_proc = self.config.resolve(npy_proc, default=None) # Compute data assimilation window parameters - window_type = self.config.window_type() - window_length = self.config.window_length() + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) local_background_time = self.da_window_params.local_background_time(window_length, window_type) local_background_time_iso = self.da_window_params.local_background_time_iso(window_length, @@ -46,7 +47,7 @@ def execute(self) -> None: window_end_iso = self.da_window_params.window_end_iso(window_length) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) # Ensemble # ------------------------ @@ -56,27 +57,27 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) - self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) # Background - self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) # Ensemble - self.jedi_rendering.add_key('ensemble_num_members', self.config.ensemble_num_members(None)) + self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(ensemble_num_members, default=None)) - self.logger.info(f'self.config.ensmeanvariance_spec = {self.config.ensmeanvariance_spec()}') - meanvar_spec_dict = self.config.ensmeanvariance_spec() + self.logger.info(f'ensmeanvariance_spec = {self.config.resolve(ensmeanvariance_spec)}') + meanvar_spec_dict = self.config.resolve(ensmeanvariance_spec) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index 0c2eaed80..6c6462c2f 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -10,6 +10,7 @@ import os from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -29,23 +30,23 @@ def execute(self) -> None: # Parse configuration # ------------------- - marine_models = self.config.marine_models() - window_type = self.config.window_type() - window_length = self.config.window_length() - background_time_offset = self.config.background_time_offset() - number_of_iterations = self.config.number_of_iterations() - jedi_forecast_model = self.config.jedi_forecast_model(None) - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + marine_models = self.config.resolve(marine_models) + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) + background_time_offset = self.config.resolve(background_time_offset) + number_of_iterations = self.config.resolve(number_of_iterations) + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) # Atmosphere specific settings # ---------------------------- # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) - gsibec_nlats = self.config.gsibec_nlats(None) - gsibec_nlons = self.config.gsibec_nlons(None) - npx_proc = self.config.npx_proc(None) - npy_proc = self.config.npy_proc(None) + gsibec_nlats = self.config.resolve(gsibec_nlats, default=None) + gsibec_nlons = self.config.resolve(gsibec_nlons, default=None) + npx_proc = self.config.resolve(npx_proc, default=None) + npy_proc = self.config.resolve(npy_proc, default=None) # Compute data assimilation window parameters # -------------------------------------------- @@ -64,37 +65,37 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) - self.jedi_rendering.add_key('minimizer', self.config.minimizer()) + self.jedi_rendering.add_key('minimizer', self.config.resolve(minimizer)) self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) - self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) self.jedi_rendering.add_key('gradient_norm_reduction', - self.config.gradient_norm_reduction()) + self.config.resolve(gradient_norm_reduction)) # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Atmosphere background error model # --------------------------------- if npx_proc is not None and npy_proc is not None: - self.jedi_rendering.add_key('gsibec_configuration', self.config.gsibec_configuration()) + self.jedi_rendering.add_key('gsibec_configuration', self.config.resolve(gsibec_configuration)) self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) self.jedi_rendering.add_key('gsibec_npx_proc', npx_proc) @@ -102,12 +103,12 @@ def execute(self) -> None: # Background frequency is required for FGAT irrespective of the model type # ------------------------------------------------------------------------ - background_frequency = self.config.background_frequency() + background_frequency = self.config.resolve(background_frequency) self.jedi_rendering.add_key('background_frequency', background_frequency) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index 2a5be2c7a..5b3cef852 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -11,6 +11,7 @@ import os from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable from swell.tasks.run_jedi_hofx_executable import RunJediHofxExecutable @@ -35,11 +36,11 @@ def execute(self) -> None: # Parse configuration ... despite same block in RunJediHofxExecutable # ------------------------------------------------------------------- - window_type = self.config.window_type() - window_length = self.config.window_length() - background_time_offset = self.config.background_time_offset() - jedi_forecast_model = self.config.jedi_forecast_model(None) - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) + background_time_offset = self.config.resolve(background_time_offset) + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) # Compute data assimilation window parameters background_time = self.da_window_params.background_time(background_time_offset) @@ -52,9 +53,9 @@ def execute(self) -> None: # Ensemble hofx components # ------------------------ - ensemble_hofx_packets = self.config.ensemble_hofx_packets() - ensemble_hofx_strategy = self.config.ensemble_hofx_strategy() - ensemble_num_members = self.config.ensemble_num_members() + ensemble_hofx_packets = self.config.resolve(ensemble_hofx_packets) + ensemble_hofx_strategy = self.config.resolve(ensemble_hofx_strategy) + ensemble_num_members = self.config.resolve(ensemble_num_members) # Force packets of equal size (i.e., members handled) # --------------------------------------------------- @@ -83,24 +84,24 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_end_iso', window_end_iso) # Background - self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) - self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) - self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) + self.jedi_rendering.add_key('npx_proc', self.config.resolve(npx_proc, default=None)) + self.jedi_rendering.add_key('npy_proc', self.config.resolve(npy_proc, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) # Observations self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Model if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.background_frequency()) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) # Populate remaining entries of jedi interface templates dictionary # ----------------------------------------------------------------- @@ -109,7 +110,7 @@ def execute(self) -> None: # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 46e039811..0fd2f16d0 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -13,6 +13,7 @@ from ruamel.yaml import YAML from typing import Optional +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.netcdf_files import combine_files_without_groups from swell.utilities.run_jedi_executables import run_executable @@ -33,17 +34,17 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Parse configuration # ------------------- - window_type = self.config.window_type() - window_length = self.config.window_length() - forecast_length = self.config.forecast_length(window_length) - background_time_offset = self.config.background_time_offset() - observations = self.config.observations() - jedi_forecast_model = self.config.jedi_forecast_model(None) - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - save_geovals = self.config.save_geovals(False) + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) + forecast_length = self.config.resolve(forecast_length, default=window_length) + background_time_offset = self.config.resolve(background_time_offset) + observations = self.config.resolve(observations) + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + save_geovals = self.config.resolve(save_geovals, default=False) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) # Compute data assimilation window parameters # -------------------------------------------- @@ -65,28 +66,28 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) - self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) - self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) - self.jedi_rendering.add_key('npx', self.config.npx(None)) - self.jedi_rendering.add_key('npy', self.config.npy(None)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) + self.jedi_rendering.add_key('npx_proc', self.config.resolve(npx_proc, default=None)) + self.jedi_rendering.add_key('npy_proc', self.config.resolve(npy_proc, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('npx', self.config.resolve(npx, default=None)) + self.jedi_rendering.add_key('npy', self.config.resolve(npy, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') @@ -94,7 +95,7 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Model # ----- if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.background_frequency()) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) # Get the JEDI interface metadata # ------------------------------- @@ -132,7 +133,7 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: 'time interpolation': 'linear' } self.jedi_rendering.add_key('forecast_length', - self.config.forecast_length(window_length)) + self.config.resolve(forecast_length, default=window_length)) # Update config filters to save the GeoVaLs from the model interface. # Add GOMsaver to either obs filters OR obs prior filters, if neither @@ -270,7 +271,7 @@ def append_gomsaver( # Add mem to the filename if it is not None mem_str = f'_mem{mem}' if mem is not None else '' - if not self.config.mock_experiment(False): + if not self.config.resolve(mock_experiment, default=False): cycle_dir = self.cycle_dir() else: cycle_dir = 'cycle_dir' diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 20a687df3..7211e5d19 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -12,6 +12,7 @@ from importlib import resources from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable from swell.utilities.yaml_utils import replace_key @@ -32,18 +33,18 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_type = self.config.window_type() - window_length = self.config.window_length() - background_time_offset = self.config.background_time_offset() + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) + background_time_offset = self.config.resolve(background_time_offset) - jedi_forecast_model = self.config.jedi_forecast_model(None) - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - ensmean_only = self.config.ensmean_only() - ensmeanvariance_only = self.config.ensmeanvariance_only() - perhost = self.config.perhost(None) + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + ensmean_only = self.config.resolve(ensmean_only) + ensmeanvariance_only = self.config.resolve(ensmeanvariance_only) + perhost = self.config.resolve(perhost, default=None) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) # Compute data assimilation window parameters background_time = self.da_window_params.background_time(background_time_offset) @@ -60,74 +61,74 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) self.jedi_rendering.add_key('window_length', window_length) self.jedi_rendering.add_key('window_end_iso', window_end_iso) - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) - self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) # Background - self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) - self.jedi_rendering.add_key('ensemble_num_members', self.config.ensemble_num_members()) + self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(ensemble_num_members)) # Geometry - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) - self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) - self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) + self.jedi_rendering.add_key('npx_proc', self.config.resolve(npx_proc, default=None)) + self.jedi_rendering.add_key('npy_proc', self.config.resolve(npy_proc, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) # Observations self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Ensemble hofx components - self.jedi_rendering.add_key('ensemble_hofx_strategy', self.config.ensemble_hofx_strategy()) - self.jedi_rendering.add_key('ensemble_hofx_packets', self.config.ensemble_hofx_packets()) + self.jedi_rendering.add_key('ensemble_hofx_strategy', self.config.resolve(ensemble_hofx_strategy)) + self.jedi_rendering.add_key('ensemble_hofx_packets', self.config.resolve(ensemble_hofx_packets)) # Ensemble Localizations # ------------------------------ if self.get_model() == 'geos_atmosphere': self.jedi_rendering.add_key('vertical_localization_method', - self.config.vertical_localization_method()) + self.config.resolve(vertical_localization_method)) self.jedi_rendering.add_key('vertical_localization_apply_log_transform', - self.config.vertical_localization_apply_log_transform()) + self.config.resolve(vertical_localization_apply_log_transform)) self.jedi_rendering.add_key('vertical_localization_lengthscale', - self.config.vertical_localization_lengthscale()) + self.config.resolve(vertical_localization_lengthscale)) self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord', - self.config.vertical_localization_ioda_vertical_coord()) + self.config.resolve(vertical_localization_ioda_vertical_coord)) self.jedi_rendering.add_key( 'vertical_localization_ioda_vertical_coord_group', - self.config.vertical_localization_ioda_vertical_coord_group()) + self.config.resolve(vertical_localization_ioda_vertical_coord_group)) self.jedi_rendering.add_key('vertical_localization_function', - self.config.vertical_localization_function()) + self.config.resolve(vertical_localization_function)) # Driver - self.jedi_rendering.add_key('local_ensemble_solver', self.config.local_ensemble_solver()) + self.jedi_rendering.add_key('local_ensemble_solver', self.config.resolve(local_ensemble_solver)) self.jedi_rendering.add_key('local_ensemble_inflation_rtps', - self.config.local_ensemble_inflation_rtps()) + self.config.resolve(local_ensemble_inflation_rtps)) self.jedi_rendering.add_key('local_ensemble_inflation_rtpp', - self.config.local_ensemble_inflation_rtpp()) + self.config.resolve(local_ensemble_inflation_rtpp)) self.jedi_rendering.add_key('local_ensemble_inflation_mult', - self.config.local_ensemble_inflation_mult()) + self.config.resolve(local_ensemble_inflation_mult)) self.jedi_rendering.add_key('local_ensemble_save_posterior_mean', - self.config.local_ensemble_save_posterior_mean()) + self.config.resolve(local_ensemble_save_posterior_mean)) self.jedi_rendering.add_key('local_ensemble_save_posterior_ensemble', - self.config.local_ensemble_save_posterior_ensemble()) + self.config.resolve(local_ensemble_save_posterior_ensemble)) self.jedi_rendering.add_key('local_ensemble_save_posterior_mean_increment', - self.config.local_ensemble_save_posterior_mean_increment()) + self.config.resolve(local_ensemble_save_posterior_mean_increment)) self.jedi_rendering.add_key('local_ensemble_save_posterior_ensemble_increments', - self.config.local_ensemble_save_posterior_ensemble_increments()) + self.config.resolve(local_ensemble_save_posterior_ensemble_increments)) self.jedi_rendering.add_key('ensmean_only', - self.config.ensmean_only()) + self.config.resolve(ensmean_only)) self.jedi_rendering.add_key('ensmeanvariance_only', - self.config.ensmeanvariance_only()) + self.config.resolve(ensmeanvariance_only)) self.jedi_rendering.add_key('local_ensemble_use_linear_observer', - self.config.local_ensemble_use_linear_observer()) - self.jedi_rendering.add_key('skip_ensemble_hofx', self.config.skip_ensemble_hofx()) + self.config.resolve(local_ensemble_use_linear_observer)) + self.jedi_rendering.add_key('skip_ensemble_hofx', self.config.resolve(skip_ensemble_hofx)) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') @@ -135,8 +136,8 @@ def execute(self) -> None: # Prevent both 'local_ensemble_save_posterior_mean' and # 'local_ensemble_save_posterior_ensemble' from being true # -------------------------------------------------------- - if self.config.local_ensemble_save_posterior_mean() and \ - self.config.local_ensemble_save_posterior_ensemble(): + if self.config.resolve(local_ensemble_save_posterior_mean) and \ + self.config.resolve(local_ensemble_save_posterior_ensemble): raise ValueError("'local_ensemble_save_posterior_mean' and\ 'local_ensemble_save_posterior_ensemble' cannot be both true!") @@ -157,15 +158,15 @@ def execute(self) -> None: # Assemble localizations # ---------------------- # # Vertical localizations have bug(s) - Commented out for now... - # vertLoc = {'localization method': self.config.vertical_localization_method(), + # vertLoc = {'localization method': self.config.resolve(vertical_localization_method), # 'apply log transformation': - # self.config.vertical_localization_apply_log_transform(), - # 'vertical lengthscale': self.config.vertical_localization_lengthscale(), + # self.config.resolve(vertical_localization_apply_log_transform), + # 'vertical lengthscale': self.config.resolve(vertical_localization_lengthscale), # 'ioda vertical coordinate': - # self.config.vertical_localization_ioda_vertical_coord(), + # self.config.resolve(vertical_localization_ioda_vertical_coord), # 'ioda vertical coordinate group': - # self.config.vertical_localization_ioda_vertical_coord_group(), - # 'localization function': self.config.vertical_localization_function()} + # self.config.resolve(vertical_localization_ioda_vertical_coord_group), + # 'localization function': self.config.resolve(vertical_localization_function)} # localizations = [horizLoc, vertLoc] if len(vertLoc) != 0 else [horizLoc] # Include ensemble localizations and halo types with each observation diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index d2c6009ed..27772e0ee 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -12,6 +12,7 @@ from ruamel.yaml import YAML from typing import Optional import random +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -35,15 +36,15 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Parse configuration # ------------------- - window_type = self.config.window_type() - window_length = self.config.window_length() - background_time_offset = self.config.background_time_offset() - jedi_forecast_model = self.config.jedi_forecast_model(None) - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - obs_thinning_rej_fraction = self.config.obs_thinning_rej_fraction() + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) + background_time_offset = self.config.resolve(background_time_offset) + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + obs_thinning_rej_fraction = self.config.resolve(obs_thinning_rej_fraction) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) # Compute data assimilation window parameters # -------------------------------------------- @@ -64,26 +65,26 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) - self.jedi_rendering.add_key('npx_proc', self.config.npx_proc(None)) - self.jedi_rendering.add_key('npy_proc', self.config.npy_proc(None)) - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) + self.jedi_rendering.add_key('npx_proc', self.config.resolve(npx_proc, default=None)) + self.jedi_rendering.add_key('npy_proc', self.config.resolve(npy_proc, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') @@ -91,7 +92,7 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Model # ----- if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.background_frequency()) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) # Get the JEDI interface metadata # ------------------------------- diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 308d5c8ee..62842e037 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -12,6 +12,7 @@ import os from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.dictionary import update_dict from swell.utilities.run_jedi_executables import run_executable @@ -32,14 +33,14 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_length = self.config.window_length() - bkg_time_offset = self.config.background_time_offset() - observations = self.config.observations() - single_observations = self.config.single_observations() - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + window_length = self.config.resolve(window_length) + bkg_time_offset = self.config.resolve(background_time_offset) + observations = self.config.resolve(observations) + single_observations = self.config.resolve(single_observations) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) # Compute data assimilation window parameters window_begin = self.da_window_params.window_begin(window_length) @@ -54,12 +55,12 @@ def execute(self) -> None: # Observations self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index bb09b9cc1..a2d90443b 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -10,6 +10,7 @@ import os from ruamel.yaml import YAML +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -29,25 +30,25 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_type = self.config.window_type() - window_length = self.config.window_length() - forecast_length = self.config.forecast_length(window_length) - background_time_offset = self.config.background_time_offset() - number_of_iterations = self.config.number_of_iterations() - jedi_forecast_model = self.config.jedi_forecast_model(None) - generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - perhost = self.config.perhost(None) + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) + forecast_length = self.config.resolve(forecast_length, default=window_length) + background_time_offset = self.config.resolve(background_time_offset) + number_of_iterations = self.config.resolve(number_of_iterations) + jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + perhost = self.config.resolve(perhost, default=None) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) - gsibec_nlats = self.config.gsibec_nlats(None) - gsibec_nlons = self.config.gsibec_nlons(None) - gsibec_configuration = self.config.gsibec_configuration(None) - npx_proc = self.config.npx_proc(None) - npy_proc = self.config.npy_proc(None) - npx = self.config.npx(None) - npy = self.config.npy(None) + gsibec_nlats = self.config.resolve(gsibec_nlats, default=None) + gsibec_nlons = self.config.resolve(gsibec_nlons, default=None) + gsibec_configuration = self.config.resolve(gsibec_configuration, default=None) + npx_proc = self.config.resolve(npx_proc, default=None) + npy_proc = self.config.resolve(npy_proc, default=None) + npx = self.config.resolve(npx, default=None) + npy = self.config.resolve(npy, default=None) # Compute data assimilation window parameters # -------------------------------------------- @@ -66,41 +67,41 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) self.jedi_rendering.add_key('forecast_length', forecast_length) - self.jedi_rendering.add_key('minimizer', self.config.minimizer()) + self.jedi_rendering.add_key('minimizer', self.config.resolve(minimizer)) self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) - self.jedi_rendering.add_key('analysis_variables', self.config.analysis_variables()) - self.jedi_rendering.add_key('saber_central_block', self.config.saber_central_block(None)) - self.jedi_rendering.add_key('saber_outer_block', self.config.saber_outer_block(None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) + self.jedi_rendering.add_key('saber_central_block', self.config.resolve(saber_central_block, default=None)) + self.jedi_rendering.add_key('saber_outer_block', self.config.resolve(saber_outer_block, default=None)) self.jedi_rendering.add_key('gradient_norm_reduction', - self.config.gradient_norm_reduction()) - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.config.resolve(gradient_norm_reduction)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.horizontal_resolution()) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.vertical_resolution()) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) self.jedi_rendering.add_key('npx', npx) self.jedi_rendering.add_key('npy', npy) - self.jedi_rendering.add_key('total_processors', self.config.total_processors(None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.crtm_coeff_dir(None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.mock_experiment(False): + if self.config.resolve(mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') @@ -117,7 +118,7 @@ def execute(self) -> None: # Model # ----- if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.background_frequency()) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) # Jedi configuration file # ----------------------- diff --git a/src/swell/tasks/save_background.py b/src/swell/tasks/save_background.py index ba524d187..3a10acc7c 100644 --- a/src/swell/tasks/save_background.py +++ b/src/swell/tasks/save_background.py @@ -12,6 +12,7 @@ import os from r2d2 import store +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats from swell.utilities.r2d2 import load_r2d2_credentials @@ -53,7 +54,7 @@ def execute(self) -> None: # Load R2D2 credentials load_r2d2_credentials(self.logger, self.platform()) - dry_run = self.config.dry_run(True) + dry_run = self.config.resolve(dry_run, default=True) if dry_run: self.logger.info('DRY RUN MODE - No files will be stored') @@ -70,10 +71,10 @@ def execute(self) -> None: ) model = self.get_model() - source_template = self.config.background_source_path() - experiment = self.config.background_experiment('geos_cf_oper') - resolution = self.config.horizontal_resolution('c360') - store_as_symlink = self.config.store_as_symlink(True) + source_template = self.config.resolve(background_source_path) + experiment = self.config.resolve(background_experiment, default='geos_cf_oper') + resolution = self.config.resolve(horizontal_resolution, default='c360') + store_as_symlink = self.config.resolve(store_as_symlink, default=True) stored = 0 skipped = 0 diff --git a/src/swell/tasks/save_forecast_cf.py b/src/swell/tasks/save_forecast_cf.py index 9ec2f7b83..e18023856 100644 --- a/src/swell/tasks/save_forecast_cf.py +++ b/src/swell/tasks/save_forecast_cf.py @@ -14,6 +14,7 @@ from swell.configuration.jedi.interfaces.geos_cf.model.r2d2 import forecast_filename, r2d2 +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials @@ -39,10 +40,10 @@ def execute(self) -> None: # Parse config # ------------ - window_length = self.config.window_length() - forecast_length = self.config.forecast_length() - forecast_output_frequency = self.config.forecast_output_frequency() - expid = self.config.r2d2_experiment_id() + window_length = self.config.resolve(window_length) + forecast_length = self.config.resolve(forecast_length) + forecast_output_frequency = self.config.resolve(forecast_output_frequency) + expid = self.config.resolve(r2d2_experiment_id) fc_store = r2d2(self.jedi_rendering.__template_dict__)['store']['fc'][0] cycle_dir = self.cycle_dir() @@ -75,7 +76,7 @@ def execute(self) -> None: item='forecast', step=step, experiment=expid, - resolution=self.config.horizontal_resolution(), + resolution=self.config.resolve(horizontal_resolution), date=forecast_start.strftime('%Y%m%dT%H%M%S%z'), source_file=source_file, file_extension='nc', diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index eacf9ba58..0096ff4b6 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -8,6 +8,7 @@ # -------------------------------------------------------------------------------------------------- import r2d2 +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials from swell.utilities.run_jedi_executables import check_obs @@ -28,21 +29,21 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.r2d2_server(default=None), + r2d2_server=self.config.resolve(r2d2_server, default=None), ) - r2d2_datastore = self.config.r2d2_datastore(default=None) + r2d2_datastore = self.config.resolve(r2d2_datastore, default=None) # Parse config # ------------ - background_time_offset = self.config.background_time_offset() - crtm_coeff_dir = self.config.crtm_coeff_dir(None) - observations = self.config.observations() - window_length = self.config.window_length() + background_time_offset = self.config.resolve(background_time_offset) + crtm_coeff_dir = self.config.resolve(crtm_coeff_dir, default=None) + observations = self.config.resolve(observations) + window_length = self.config.resolve(window_length) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) - self.jedi_rendering.add_key('marine_models', self.config.marine_models(None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) # Get window beginning window_begin = self.da_window_params.window_begin(window_length) # dto @@ -86,7 +87,7 @@ def execute(self) -> None: try: store_kwargs = dict( item='feedback', - experiment=self.config.r2d2_experiment_id(), + experiment=self.config.resolve(r2d2_experiment_id), observation_type=name, file_extension=obs_path_file.split('.')[-1], window_length='PT6H', diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index 28f5b7254..680e2a40a 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -7,6 +7,7 @@ # -------------------------------------------------------------------------------------------------- +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/save_restart_cf.py b/src/swell/tasks/save_restart_cf.py index 9ee252f0e..2d3080d51 100644 --- a/src/swell/tasks/save_restart_cf.py +++ b/src/swell/tasks/save_restart_cf.py @@ -11,6 +11,7 @@ import os from r2d2 import store +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials @@ -35,9 +36,9 @@ def execute(self): # Parse config # ------------ - window_length = self.config.window_length() - expid = self.config.r2d2_experiment_id() - horizontal_resolution = self.config.horizontal_resolution() + window_length = self.config.resolve(window_length) + expid = self.config.resolve(r2d2_experiment_id) + horizontal_resolution = self.config.resolve(horizontal_resolution) cycle_dir = self.cycle_dir() scratch_dir = os.path.join(cycle_dir, 'scratch') @@ -51,8 +52,8 @@ def execute(self): next_window_begin = window_begin + isodate.parse_duration(window_length) checkpoint_time_str = next_window_begin.strftime('%Y%m%d_%H%Mz') - rst_file_types = self.config.rst_file_types() - rst_store_interval = self.config.rst_store_interval(None) + rst_file_types = self.config.resolve(rst_file_types) + rst_store_interval = self.config.resolve(rst_store_interval, default=None) # Determine whether to store as a symlink for this cycle # -------------------------------------------------------- diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 06847a72e..3c47bdafc 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -11,6 +11,7 @@ import os from swell.swell_path import get_swell_path +from swell.configuration.question_defaults import * from swell.tasks.base.task_base import taskBase from swell.utilities.filehandler import get_file_handler from swell.utilities.exceptions import SwellError @@ -32,10 +33,10 @@ def execute(self) -> None: """ # Extract potential template variables from config - horizontal_resolution = self.config.horizontal_resolution() + horizontal_resolution = self.config.resolve(horizontal_resolution) - swell_static_files_user = self.config.swell_static_files_user(None) - swell_static_files = self.config.swell_static_files() + swell_static_files_user = self.config.resolve(swell_static_files_user, default=None) + swell_static_files = self.config.resolve(swell_static_files) # Use static_files_user if present in config and contains files # ------------------------------------------------------------- @@ -45,13 +46,13 @@ def execute(self) -> None: self.logger.info(f'Using swell static files in {swell_static_files_user}') swell_static_files = swell_static_files_user - vertical_resolution = self.config.vertical_resolution() - npx_proc = self.config.npx_proc(None) - npy_proc = self.config.npy_proc(None) - gsibec_configuration = self.config.gsibec_configuration(None) - gsibec_nlats = self.config.gsibec_nlats(None) - gsibec_nlons = self.config.gsibec_nlons(None) - saber_central_block = self.config.saber_central_block(None) + vertical_resolution = self.config.resolve(vertical_resolution) + npx_proc = self.config.resolve(npx_proc, default=None) + npy_proc = self.config.resolve(npy_proc, default=None) + gsibec_configuration = self.config.resolve(gsibec_configuration, default=None) + gsibec_nlats = self.config.resolve(gsibec_nlats, default=None) + gsibec_nlons = self.config.resolve(gsibec_nlons, default=None) + saber_central_block = self.config.resolve(saber_central_block, default=None) # Add jedi interface template keys self.jedi_rendering.add_key('horizontal_resolution', horizontal_resolution) From 7980ff1c9b6592425087f83315f40d6aa995ce55 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 25 Aug 2026 15:34:05 -0400 Subject: [PATCH 07/15] Add to geos_marine tasks --- .../geos_marine/get_background_geos_marine.py | 15 ++++++++------- .../tasks/geos_marine/get_ensemble_geos_marine.py | 9 +++++---- .../run_compress_forecast_geos_marine.py | 11 ++++++----- .../geos_marine/save_forecast_geos_marine.py | 12 +++++++----- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/swell/tasks/geos_marine/get_background_geos_marine.py b/src/swell/tasks/geos_marine/get_background_geos_marine.py index b657cc529..fb02f5609 100644 --- a/src/swell/tasks/geos_marine/get_background_geos_marine.py +++ b/src/swell/tasks/geos_marine/get_background_geos_marine.py @@ -13,6 +13,7 @@ import r2d2 from swell.tasks.base.task_base import taskBase +from swell.configuration.question_defaults import * from swell.utilities.r2d2 import load_r2d2_credentials # -------------------------------------------------------------------------------------------------- @@ -54,15 +55,15 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.r2d2_server(default=None), + r2d2_server=self.config.resolve(r2d2_server, default=None) ) - r2d2_datastore = self.config.r2d2_datastore(default=None) - marine_models = self.config.marine_models(None) or [] - window_type = self.config.window_type() - window_length = self.config.window_length() - horizontal_resolution = self.config.horizontal_resolution() - background_experiment = self.config.background_experiment() + r2d2_datastore = self.config.resolve(r2d2_datastore, default=None) + marine_models = self.config.resolve(marine_models, default=None) or [] + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) + horizontal_resolution = self.config.resolve(horizontal_resolution) + background_experiment = self.config.resolve(background_experiment) local_background_time, local_background_time_dto = \ self.da_window_params.local_background_time(window_length, window_type, dto=True) diff --git a/src/swell/tasks/geos_marine/get_ensemble_geos_marine.py b/src/swell/tasks/geos_marine/get_ensemble_geos_marine.py index 38bf817d2..ad99bafc3 100644 --- a/src/swell/tasks/geos_marine/get_ensemble_geos_marine.py +++ b/src/swell/tasks/geos_marine/get_ensemble_geos_marine.py @@ -11,6 +11,7 @@ import tarfile from swell.tasks.base.task_base import taskBase +from swell.configuration.question_defaults import * from swell.utilities.datetime_util import datetime_formats # -------------------------------------------------------------------------------------------------- @@ -35,13 +36,13 @@ def execute(self) -> None: # Parse configuration # ------------------- - horizontal_resolution = self.config.horizontal_resolution() - vertical_resolution = self.config.vertical_resolution() - background_experiment = self.config.background_experiment() + horizontal_resolution = self.config.resolve(horizontal_resolution) + vertical_resolution = self.config.resolve(vertical_resolution) + background_experiment = self.config.resolve(background_experiment) # Get the path and pattern for the ensemble members # ------------------------------------------------- - ensemble_path = self.config.path_to_ensemble() + ensemble_path = self.config.resolve(path_to_ensemble) # For 3D window, analysis time is the cycle time # ------------------------------------------------- diff --git a/src/swell/tasks/geos_marine/run_compress_forecast_geos_marine.py b/src/swell/tasks/geos_marine/run_compress_forecast_geos_marine.py index 350fc486d..95921953c 100644 --- a/src/swell/tasks/geos_marine/run_compress_forecast_geos_marine.py +++ b/src/swell/tasks/geos_marine/run_compress_forecast_geos_marine.py @@ -13,6 +13,7 @@ import tarfile from swell.tasks.base.task_base import taskBase +from swell.configuration.question_defaults import * from swell.utilities.compress import compress_file # -------------------------------------------------------------------------------------------------- @@ -52,16 +53,16 @@ def execute(self) -> None: state files into a tar.gz archive. """ - marine_models = self.config.marine_models(None) or [] - window_type = self.config.window_type() - window_length = self.config.window_length() + marine_models = self.config.resolve(marine_models, default=None) or [] + window_type = self.config.resolve(window_type) + window_length = self.config.resolve(window_length) window_begin_iso = self.da_window_params.window_begin_iso(window_length) - horizontal_resolution = self.config.horizontal_resolution() + horizontal_resolution = self.config.resolve(horizontal_resolution) is_4d = window_type == '4D' or 'fgat' in self.suite_name() if is_4d: - background_frequency = self.config.background_frequency() + background_frequency = self.config.resolve(background_frequency) (local_background_time, local_background_time_dto) = \ self.da_window_params.local_background_time(window_length, window_type, dto=True) diff --git a/src/swell/tasks/geos_marine/save_forecast_geos_marine.py b/src/swell/tasks/geos_marine/save_forecast_geos_marine.py index ce12d8c1e..f49146a1c 100644 --- a/src/swell/tasks/geos_marine/save_forecast_geos_marine.py +++ b/src/swell/tasks/geos_marine/save_forecast_geos_marine.py @@ -12,6 +12,8 @@ from r2d2 import store from swell.tasks.base.task_base import taskBase +from swell.configuration.question_defaults import * + from swell.utilities.datetime_util import datetime_formats from swell.utilities.r2d2 import load_r2d2_credentials @@ -58,10 +60,10 @@ def execute(self) -> None: window type (and/or suite type) and length. """ - self.marine_models = self.config.marine_models(None) or [] - window_type = self.config.window_type() - self.window_length = self.config.window_length() - self.horizontal_resolution = self.config.horizontal_resolution() + self.marine_models = self.config.resolve(marine_models, default=None) or [] + window_type = self.config.resolve(window_type) + self.window_length = self.config.resolve(window_length) + self.horizontal_resolution = self.config.resolve(horizontal_resolution) load_r2d2_credentials(self.logger, self.platform()) @@ -100,7 +102,7 @@ def _store_forecast(self, model_name: str, file_type: str) -> None: store( item='forecast', model=model_name, - experiment=self.config.r2d2_experiment_id(), + experiment=self.config.resolve(r2d2_experiment_id), resolution=self.horizontal_resolution, date=self.local_background_time_dto.strftime('%Y-%m-%d %H:%M:%S'), source_file=archive_path, From bd482e57dbb1ffffba504d9fa01d37d03faf0834 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 25 Aug 2026 16:31:02 -0400 Subject: [PATCH 08/15] Fix clone_geos --- src/swell/tasks/clone_geos.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 4b9d48f1e..5d8507fac 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -35,15 +35,15 @@ def execute(self) -> None: # Choice to link to existing build or build GEOS # ---------------------------------------------- - if self.config.resolve(resolve, default=geos_build_method) == 'use_existing': + if self.config.resolve(geos_build_method) == 'use_existing': # Link the source code directory - link_path(self.config.resolve(resolve, default=existing_geos_gcm_source_path), geos_gcm_source_path) + link_path(self.config.resolve(existing_geos_gcm_source_path), geos_gcm_source_path) - elif self.config.resolve(resolve, default=geos_build_method) == 'create': + elif self.config.resolve(geos_build_method) == 'create': # Get tag to build - geos_gcm_tag = self.config.resolve(resolve, default=geos_gcm_tag) + geos_gcm_tag = self.config.resolve(geos_gcm_tag) # Make sure tag is prepended with 'v' if geos_gcm_tag[0] != 'v': From cd5d29280454dbcb11218f78ee6fa81efa6d4d97 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Tue, 25 Aug 2026 16:33:39 -0400 Subject: [PATCH 09/15] Fix clone_jedi --- src/swell/tasks/clone_jedi.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index a0dd1ce33..1df319321 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -38,17 +38,17 @@ def execute(self) -> None: # Choice to link to existing build or build JEDI using jedi_bundle # ---------------------------------------------------------------- - if self.config.resolve(resolve, default=jedi_build_method) == 'use_existing': + if self.config.resolve(jedi_build_method) == 'use_existing': # Link the source code directory - link_path(self.config.resolve(resolve, default=existing_jedi_source_directory), jedi_bundle_source_path) + link_path(self.config.resolve(existing_jedi_source_directory), jedi_bundle_source_path) - elif self.config.resolve(resolve, default=jedi_build_method) == 'use_pinned_existing': + elif self.config.resolve(jedi_build_method) == 'use_pinned_existing': # Check hashes before proceeding - check_hashes(self.config.resolve(resolve, default=existing_jedi_source_directory_pinned), self.logger) + check_hashes(self.config.resolve(existing_jedi_source_directory_pinned), self.logger) # Link the pinned source code directory - link_path(self.config.resolve(resolve, default=existing_jedi_source_directory_pinned), jedi_bundle_source_path) + link_path(self.config.resolve(existing_jedi_source_directory_pinned), jedi_bundle_source_path) - elif self.config.resolve(resolve, default=jedi_build_method) in ('create', 'pinned_create'): + elif self.config.resolve(jedi_build_method) in ('create', 'pinned_create'): # Determine which bundles need to be build model_components = self.get_model_components() if model_components is not None: @@ -65,11 +65,11 @@ def execute(self) -> None: # Determine whether to use pinned versions or not use_pinned = False - if self.config.resolve(resolve, default=jedi_build_method) == 'pinned_create': + if self.config.resolve(default=jedi_build_method) == 'pinned_create': use_pinned = True # Generate the build dictionary - jedi_bundle_dict = set_jedi_bundle_config(self.config.resolve(resolve, default=bundles, bundles), + jedi_bundle_dict = set_jedi_bundle_config(self.config.resolve(bundles, default=bundles), jedi_bundle_source_path, jedi_bundle_build_path, self.platform(), From aa2ba9e2276883d0817d4af82df53cde5bf72fdc Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 26 Aug 2026 11:14:38 -0400 Subject: [PATCH 10/15] Call from import --- src/swell/tasks/bufr_to_ioda.py | 2 +- src/swell/tasks/build_geos.py | 4 +- src/swell/tasks/build_geos_by_linking.py | 12 +-- src/swell/tasks/build_jedi.py | 10 +- src/swell/tasks/build_jedi_by_linking.py | 12 +-- src/swell/tasks/clean_cycle.py | 6 +- src/swell/tasks/clean_eda.py | 56 +++++------ src/swell/tasks/clone_geos.py | 12 +-- src/swell/tasks/clone_geos_mksi.py | 8 +- src/swell/tasks/clone_gmao_perllib.py | 6 +- src/swell/tasks/clone_jedi.py | 20 ++-- src/swell/tasks/convert_obs_to_ioda.py | 8 +- src/swell/tasks/download_obs.py | 8 +- src/swell/tasks/eva_comparison_increment.py | 4 +- src/swell/tasks/eva_comparison_jedi_log.py | 6 +- .../tasks/eva_comparison_observations.py | 8 +- src/swell/tasks/eva_increment.py | 10 +- src/swell/tasks/eva_jedi_log.py | 2 +- src/swell/tasks/eva_observations.py | 14 +-- src/swell/tasks/eva_timeseries.py | 16 ++-- src/swell/tasks/generate_b_climatology.py | 30 +++--- .../generate_b_climatology_by_linking.py | 18 ++-- .../generate_observing_system_records.py | 8 +- src/swell/tasks/get_background.py | 20 ++-- .../tasks/get_background_geos_experiment.py | 8 +- src/swell/tasks/get_bufr.py | 6 +- src/swell/tasks/get_coupled_geos_restart.py | 10 +- .../tasks/get_ensemble_geos_experiment.py | 8 +- src/swell/tasks/get_geos_adas_background.py | 4 +- src/swell/tasks/get_geovals.py | 16 ++-- src/swell/tasks/get_gsi_bc.py | 6 +- src/swell/tasks/get_gsi_ncdiag.py | 4 +- src/swell/tasks/get_ncdiags.py | 18 ++-- src/swell/tasks/get_obs_not_in_r2d2.py | 4 +- src/swell/tasks/get_observations.py | 30 +++--- src/swell/tasks/get_restart_cf.py | 12 +-- src/swell/tasks/gsi_bc_to_ioda.py | 12 +-- src/swell/tasks/gsi_ncdiag_to_ioda.py | 10 +- src/swell/tasks/ingest_obs.py | 14 +-- src/swell/tasks/jedi_log_comparison.py | 6 +- src/swell/tasks/jedi_oops_log_parser.py | 6 +- src/swell/tasks/link_coupled_geos_output.py | 10 +- src/swell/tasks/link_geos_output.py | 10 +- src/swell/tasks/move_da_restart.py | 10 +- src/swell/tasks/move_forecast_restart.py | 4 +- src/swell/tasks/prep_coupled_geos_run_dir.py | 16 ++-- src/swell/tasks/prep_forecast_cf.py | 22 ++--- src/swell/tasks/prepare_analysis.py | 12 +-- src/swell/tasks/publish_comparisons.py | 6 +- src/swell/tasks/render_jedi_observations.py | 18 ++-- ...jedi_convert_state_soca2cice_executable.py | 18 ++-- src/swell/tasks/run_jedi_diffstates.py | 26 ++--- src/swell/tasks/run_jedi_eda_executable.py | 62 ++++++------ .../tasks/run_jedi_ensemble_mean_variance.py | 30 +++--- src/swell/tasks/run_jedi_fgat_executable.py | 46 ++++----- .../run_jedi_hofx_ensemble_executable.py | 34 +++---- src/swell/tasks/run_jedi_hofx_executable.py | 44 ++++----- .../run_jedi_local_ensemble_da_executable.py | 96 +++++++++---------- .../tasks/run_jedi_obsfilters_executable.py | 32 +++---- .../tasks/run_jedi_ufo_tests_executable.py | 18 ++-- .../tasks/run_jedi_variational_executable.py | 58 +++++------ src/swell/tasks/save_background.py | 12 +-- src/swell/tasks/save_forecast_cf.py | 12 +-- src/swell/tasks/save_obs_diags.py | 20 ++-- src/swell/tasks/save_restart.py | 2 +- src/swell/tasks/save_restart_cf.py | 12 +-- src/swell/tasks/stage_jedi.py | 22 ++--- 67 files changed, 563 insertions(+), 563 deletions(-) diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index da2d54238..32ec279dd 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -13,7 +13,7 @@ import subprocess import shutil -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_geos.py b/src/swell/tasks/build_geos.py index 5ffdf93ef..d54ab494c 100644 --- a/src/swell/tasks/build_geos.py +++ b/src/swell/tasks/build_geos.py @@ -10,7 +10,7 @@ import os -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.build import build_and_source_dirs from swell.utilities.shell_commands import run_subprocess, create_executable_file @@ -35,7 +35,7 @@ def execute(self) -> None: # Check that the choice is to create build # ---------------------------------------- - if not self.config.resolve(geos_build_method) == 'create': + if not self.config.resolve(qd.geos_build_method) == 'create': self.logger.abort(f'Found \'{jedi_build_method}\' for jedi_build_method in the ' f'experiment dictionary. Must be \'create\'.') diff --git a/src/swell/tasks/build_geos_by_linking.py b/src/swell/tasks/build_geos_by_linking.py index 0e0077a40..379b39fdd 100644 --- a/src/swell/tasks/build_geos_by_linking.py +++ b/src/swell/tasks/build_geos_by_linking.py @@ -10,7 +10,7 @@ import os -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.build import build_and_source_dirs, link_path @@ -33,16 +33,16 @@ def execute(self) -> None: # Choice to link to existing build or build GEOS # ---------------------------------------------- - if not self.config.resolve(geos_build_method) == 'use_existing': - self.logger.abort(f'Found \'{self.config.resolve(geos_build_method)}\' for ' + + if not self.config.resolve(qd.geos_build_method) == 'use_existing': + self.logger.abort(f'Found \'{self.config.resolve(qd.geos_build_method)}\' for ' + f'geos_build_method in the experiment dictionary. Must be ' + f'\'use_existing\'.') # Assert that the existing build directory contains a bin directory - if not os.path.exists(os.path.join(self.config.resolve(existing_geos_gcm_build_path), 'bin')): + if not os.path.exists(os.path.join(self.config.resolve(qd.existing_geos_gcm_build_path), 'bin')): self.logger.abort(f'Existing GEOS build directory is provided but a bin ' + f'directory is not found in the path ' + - f'\'{self.config.resolve(existing_geos_gcm_build_path)}\'') + f'\'{self.config.resolve(qd.existing_geos_gcm_build_path)}\'') # Write warning to user self.logger.info('Suitable GEOS build found, linking build directory. Warning: ' + @@ -51,7 +51,7 @@ def execute(self) -> None: 'this experiment may not be reproducible if the build changes.') # Link the source code directory - link_path(self.config.resolve(existing_geos_gcm_build_path), geos_gcm_build_path) + link_path(self.config.resolve(qd.existing_geos_gcm_build_path), geos_gcm_build_path) # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/build_jedi.py b/src/swell/tasks/build_jedi.py index b9b53c8a9..abcde0d67 100644 --- a/src/swell/tasks/build_jedi.py +++ b/src/swell/tasks/build_jedi.py @@ -12,7 +12,7 @@ from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs @@ -34,8 +34,8 @@ def execute(self) -> None: # Choice to link to existing build or build JEDI using jedi_bundle # ---------------------------------------------------------------- - if self.config.resolve(jedi_build_method) not in ('create', 'pinned_create'): - self.logger.abort(f'Found \'{self.config.resolve(jedi_build_method)}\' for ' + + if self.config.resolve(qd.jedi_build_method) not in ('create', 'pinned_create'): + self.logger.abort(f'Found \'{self.config.resolve(qd.jedi_build_method)}\' for ' + f'jedi_build_method in the experiment dictionary. Must be ' + f'\'create\' or \'pinned_create\'.') @@ -54,11 +54,11 @@ def execute(self) -> None: bundles = get_bundles() use_pinned = False - if self.config.resolve(jedi_build_method) == 'pinned_create': + if self.config.resolve(qd.jedi_build_method) == 'pinned_create': use_pinned = True # Generate the build dictionary - jedi_bundle_dict = set_jedi_bundle_config(self.config.resolve(bundles, default=bundles), + jedi_bundle_dict = set_jedi_bundle_config(self.config.resolve(qd.bundles, default=bundles), jedi_bundle_source_path, jedi_bundle_build_path, self.platform(), use_pinned, 24) diff --git a/src/swell/tasks/build_jedi_by_linking.py b/src/swell/tasks/build_jedi_by_linking.py index 381cde5b4..0a209ce63 100644 --- a/src/swell/tasks/build_jedi_by_linking.py +++ b/src/swell/tasks/build_jedi_by_linking.py @@ -10,7 +10,7 @@ import os -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.build import build_and_source_dirs, link_path @@ -22,8 +22,8 @@ class BuildJediByLinking(taskBase): def execute(self) -> None: # Abort if jedi build method isn't use_existing or use_pinned_existing - if self.config.resolve(jedi_build_method) not in ('use_existing', 'use_pinned_existing'): - self.logger.abort(f'Found \'{self.config.resolve(jedi_build_method)}\' for ' + + if self.config.resolve(qd.jedi_build_method) not in ('use_existing', 'use_pinned_existing'): + self.logger.abort(f'Found \'{self.config.resolve(qd.jedi_build_method)}\' for ' + f'jedi_build_method in the experiment dictionary. Must be ' + f'\'use_existing\' or \'use_pinned_existing\'.') @@ -35,10 +35,10 @@ def execute(self) -> None: jedi_bundle_build_path, jedi_bundle_source_path = build_and_source_dirs(jedi_bundle_path) # Set existing jedi build directory based on jedi build method - if self.config.resolve(jedi_build_method) == 'use_existing': - existing_jedi_build_directory = self.config.resolve(existing_jedi_build_directory) + if self.config.resolve(qd.jedi_build_method) == 'use_existing': + existing_jedi_build_directory = self.config.resolve(qd.existing_jedi_build_directory) else: - existing_jedi_build_directory = self.config.resolve(existing_jedi_build_directory_pinned) + existing_jedi_build_directory = self.config.resolve(qd.existing_jedi_build_directory_pinned) # Assert that the existing build directory contains a bin directory if not os.path.exists(os.path.join(existing_jedi_build_directory, 'bin')): diff --git a/src/swell/tasks/clean_cycle.py b/src/swell/tasks/clean_cycle.py index a7ba54242..b3a7434f3 100644 --- a/src/swell/tasks/clean_cycle.py +++ b/src/swell/tasks/clean_cycle.py @@ -10,7 +10,7 @@ import isodate import os import shutil -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats from datetime import datetime as dt @@ -33,7 +33,7 @@ class CleanCycle(taskBase): def execute(self) -> None: # Parse config - clean_patterns = self.config.resolve(clean_patterns, default=None) + clean_patterns = self.config.resolve(qd.clean_patterns, default=None) # If no cleaning requested then exit if clean_patterns is None: @@ -92,7 +92,7 @@ def execute(self) -> None: # Clean the previous cycle's scratch directory now that this cycle has retrieved # any restart/checkpoint files it needed (which may have been symlinked from there) - window_length = self.config.resolve(window_length, default=None) + window_length = self.config.resolve(qd.window_length, default=None) if window_length is not None and self.cycle_time_dto() != self.start_cycle_point_dto(): prev_cycle_time = self.cycle_time_dto() - isodate.parse_duration(window_length) diff --git a/src/swell/tasks/clean_eda.py b/src/swell/tasks/clean_eda.py index bb8dec60b..4a7871acf 100644 --- a/src/swell/tasks/clean_eda.py +++ b/src/swell/tasks/clean_eda.py @@ -10,7 +10,7 @@ import os import glob import shutil -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase @@ -29,23 +29,23 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) - forecast_length = self.config.resolve(forecast_length, default=window_length) - background_time_offset = self.config.resolve(background_time_offset) - number_of_iterations = self.config.resolve(number_of_iterations) - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) + forecast_length = self.config.resolve(qd.forecast_length, default=window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) + number_of_iterations = self.config.resolve(qd.number_of_iterations) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) - gsibec_nlats = self.config.resolve(gsibec_nlats, default=None) - gsibec_nlons = self.config.resolve(gsibec_nlons, default=None) - gsibec_configuration = self.config.resolve(gsibec_configuration, default=None) - npx_proc = self.config.resolve(npx_proc, default=None) - npy_proc = self.config.resolve(npy_proc, default=None) - npx = self.config.resolve(npx, default=None) - npy = self.config.resolve(npy, default=None) + gsibec_nlats = self.config.resolve(qd.gsibec_nlats, default=None) + gsibec_nlons = self.config.resolve(qd.gsibec_nlons, default=None) + gsibec_configuration = self.config.resolve(qd.gsibec_configuration, default=None) + npx_proc = self.config.resolve(qd.npx_proc, default=None) + npy_proc = self.config.resolve(qd.npy_proc, default=None) + npx = self.config.resolve(qd.npx, default=None) + npy = self.config.resolve(qd.npy, default=None) # Compute data assimilation window parameters # -------------------------------------------- @@ -57,7 +57,7 @@ def execute(self) -> None: window_begin = self.da_window_params.window_begin(window_length) window_begin_iso = self.da_window_params.window_begin_iso(window_length) window_end_iso = self.da_window_params.window_end_iso(window_length) - nmember = self.config.resolve(ensemble_num_members) + nmember = self.config.resolve(qd.ensemble_num_members) # imember = self.get_ensemble_imember() # Populate jedi interface templates dictionary @@ -66,38 +66,38 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) self.jedi_rendering.add_key('forecast_length', forecast_length) - self.jedi_rendering.add_key('minimizer', self.config.resolve(minimizer)) + self.jedi_rendering.add_key('minimizer', self.config.resolve(qd.minimizer)) self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) - self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) - self.jedi_rendering.add_key('saber_central_block', self.config.resolve(saber_central_block, default=None)) - self.jedi_rendering.add_key('saber_outer_block', self.config.resolve(saber_outer_block, default=None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(qd.analysis_variables)) + self.jedi_rendering.add_key('saber_central_block', self.config.resolve(qd.saber_central_block, default=None)) + self.jedi_rendering.add_key('saber_outer_block', self.config.resolve(qd.saber_outer_block, default=None)) self.jedi_rendering.add_key('gradient_norm_reduction', - self.config.resolve(gradient_norm_reduction)) - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.config.resolve(qd.gradient_norm_reduction)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(qd.horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) - self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(ensemble_num_members)) + self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(qd.ensemble_num_members)) self.jedi_rendering.add_key('ensemble_imember', 1) # pass an integer to jinja2 # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(qd.vertical_resolution)) self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) self.jedi_rendering.add_key('npx', npx) self.jedi_rendering.add_key('npy', npy) - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Atmosphere background error model @@ -112,7 +112,7 @@ def execute(self) -> None: # Model # ----- if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(qd.background_frequency)) # Open the JEDI config file and fill initial templates # ---------------------------------------------------- diff --git a/src/swell/tasks/clone_geos.py b/src/swell/tasks/clone_geos.py index 5d8507fac..475fe4a80 100644 --- a/src/swell/tasks/clone_geos.py +++ b/src/swell/tasks/clone_geos.py @@ -10,8 +10,8 @@ import os -from swell.configuration.question_defaults import * -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.build import build_and_source_dirs, link_path from swell.utilities.git_utils import git_clone @@ -35,15 +35,15 @@ def execute(self) -> None: # Choice to link to existing build or build GEOS # ---------------------------------------------- - if self.config.resolve(geos_build_method) == 'use_existing': + if self.config.resolve(qd.geos_build_method) == 'use_existing': # Link the source code directory - link_path(self.config.resolve(existing_geos_gcm_source_path), geos_gcm_source_path) + link_path(self.config.resolve(qd.existing_geos_gcm_source_path), geos_gcm_source_path) - elif self.config.resolve(geos_build_method) == 'create': + elif self.config.resolve(qd.geos_build_method) == 'create': # Get tag to build - geos_gcm_tag = self.config.resolve(geos_gcm_tag) + geos_gcm_tag = self.config.resolve(qd.geos_gcm_tag) # Make sure tag is prepended with 'v' if geos_gcm_tag[0] != 'v': diff --git a/src/swell/tasks/clone_geos_mksi.py b/src/swell/tasks/clone_geos_mksi.py index ff7c15a34..f9756b64c 100644 --- a/src/swell/tasks/clone_geos_mksi.py +++ b/src/swell/tasks/clone_geos_mksi.py @@ -9,7 +9,7 @@ import os -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.build import link_path @@ -32,8 +32,8 @@ def execute(self) -> None: # Parse config # ------------ - path_to_geos_mksi = self.config.resolve(observing_system_records_mksi_path) - tag = self.config.resolve(observing_system_records_mksi_path_tag) + path_to_geos_mksi = self.config.resolve(qd.observing_system_records_mksi_path) + tag = self.config.resolve(qd.observing_system_records_mksi_path_tag) # If observing_system_records_mksi_path is None, clone GEOS_mksi repo to experiment # directory @@ -47,7 +47,7 @@ def execute(self) -> None: + os.path.join(self.experiment_path(), 'GEOS_mksi')) else: # Link the source code directory - link_path(self.config.resolve(observing_system_records_mksi_path), + link_path(self.config.resolve(qd.observing_system_records_mksi_path), os.path.join(self.experiment_path(), 'GEOS_mksi')) diff --git a/src/swell/tasks/clone_gmao_perllib.py b/src/swell/tasks/clone_gmao_perllib.py index bf3f6884b..085d63872 100644 --- a/src/swell/tasks/clone_gmao_perllib.py +++ b/src/swell/tasks/clone_gmao_perllib.py @@ -11,7 +11,7 @@ import os import subprocess -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -26,8 +26,8 @@ def execute(self) -> None: experiment_perllib_path = os.path.join(swell_exp_path, 'GMAO_perllib') # Get the existing location of GMAO_perllib - existing_perllib_path = self.config.resolve(gmao_perllib_path, default=None) - gmao_perllib_tag = self.config.resolve(gmao_perllib_tag, default=None) + existing_perllib_path = self.config.resolve(qd.gmao_perllib_path, default=None) + gmao_perllib_tag = self.config.resolve(qd.gmao_perllib_tag, default=None) # Set the default tag to g1.0.1 if not specified if gmao_perllib_tag is None: diff --git a/src/swell/tasks/clone_jedi.py b/src/swell/tasks/clone_jedi.py index 1df319321..3de1393ec 100644 --- a/src/swell/tasks/clone_jedi.py +++ b/src/swell/tasks/clone_jedi.py @@ -13,11 +13,11 @@ from jedi_bundle.bin.jedi_bundle import execute_tasks, get_bundles from swell.utilities.build import link_path -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.pinned_versions.check_hashes import check_hashes from swell.utilities.build import set_jedi_bundle_config, build_and_source_dirs -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- @@ -38,17 +38,17 @@ def execute(self) -> None: # Choice to link to existing build or build JEDI using jedi_bundle # ---------------------------------------------------------------- - if self.config.resolve(jedi_build_method) == 'use_existing': + if self.config.resolve(qd.jedi_build_method) == 'use_existing': # Link the source code directory - link_path(self.config.resolve(existing_jedi_source_directory), jedi_bundle_source_path) + link_path(self.config.resolve(qd.existing_jedi_source_directory), jedi_bundle_source_path) - elif self.config.resolve(jedi_build_method) == 'use_pinned_existing': + elif self.config.resolve(qd.jedi_build_method) == 'use_pinned_existing': # Check hashes before proceeding - check_hashes(self.config.resolve(existing_jedi_source_directory_pinned), self.logger) + check_hashes(self.config.resolve(qd.existing_jedi_source_directory_pinned), self.logger) # Link the pinned source code directory - link_path(self.config.resolve(existing_jedi_source_directory_pinned), jedi_bundle_source_path) + link_path(self.config.resolve(qd.existing_jedi_source_directory_pinned), jedi_bundle_source_path) - elif self.config.resolve(jedi_build_method) in ('create', 'pinned_create'): + elif self.config.resolve(qd.jedi_build_method) in ('create', 'pinned_create'): # Determine which bundles need to be build model_components = self.get_model_components() if model_components is not None: @@ -65,11 +65,11 @@ def execute(self) -> None: # Determine whether to use pinned versions or not use_pinned = False - if self.config.resolve(default=jedi_build_method) == 'pinned_create': + if self.config.resolve(qd.jedi_build_method) == 'pinned_create': use_pinned = True # Generate the build dictionary - jedi_bundle_dict = set_jedi_bundle_config(self.config.resolve(bundles, default=bundles), + jedi_bundle_dict = set_jedi_bundle_config(self.config.resolve(qd.bundles, default=bundles), jedi_bundle_source_path, jedi_bundle_build_path, self.platform(), diff --git a/src/swell/tasks/convert_obs_to_ioda.py b/src/swell/tasks/convert_obs_to_ioda.py index 97c23f281..a15df6548 100644 --- a/src/swell/tasks/convert_obs_to_ioda.py +++ b/src/swell/tasks/convert_obs_to_ioda.py @@ -20,7 +20,7 @@ import yaml from datetime import datetime -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase @@ -63,13 +63,13 @@ class ConvertObsToIoda(taskBase): def execute(self) -> None: - obs_to_convert = self.config.resolve(obs_to_download, default=[]) - dry_run = self.config.resolve(dry_run, default=True) + obs_to_convert = self.config.resolve(qd.obs_to_download, default=[]) + dry_run = self.config.resolve(qd.dry_run, default=True) if dry_run: self.logger.info('DRY RUN MODE - No converters will be run') - converter_path = self.config.resolve(converter_path, default='') + converter_path = self.config.resolve(qd.converter_path, default='') if converter_path: converter_bin = converter_path else: diff --git a/src/swell/tasks/download_obs.py b/src/swell/tasks/download_obs.py index 9d2b0ea76..36b26a861 100644 --- a/src/swell/tasks/download_obs.py +++ b/src/swell/tasks/download_obs.py @@ -35,7 +35,7 @@ import isodate import requests -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities import s3 as swell_s3 @@ -80,9 +80,9 @@ class DownloadObs(taskBase): def execute(self) -> None: - obs_to_download = self.config.resolve(obs_to_download, default=[]) - window_length = self.config.resolve(window_length) - dry_run = self.config.resolve(dry_run, default=True) + obs_to_download = self.config.resolve(qd.obs_to_download, default=[]) + window_length = self.config.resolve(qd.window_length) + dry_run = self.config.resolve(qd.dry_run, default=True) if dry_run: self.logger.info('DRY RUN MODE - No files will be downloaded') diff --git a/src/swell/tasks/eva_comparison_increment.py b/src/swell/tasks/eva_comparison_increment.py index 10c65aa30..64dd2820b 100644 --- a/src/swell/tasks/eva_comparison_increment.py +++ b/src/swell/tasks/eva_comparison_increment.py @@ -14,7 +14,7 @@ from eva.eva_driver import eva -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.data_assimilation_window_params import DataAssimilationWindowParams @@ -49,7 +49,7 @@ def execute(self) -> None: eva_str_template = eva_config_file_open.read() # Get the paths for the two experiments - experiment_paths = self.config.resolve(comparison_experiment_paths) + experiment_paths = self.config.resolve(qd.comparison_experiment_paths) print(experiment_paths) experiment_tag_paths = comparison_tags(experiment_paths, self.logger) diff --git a/src/swell/tasks/eva_comparison_jedi_log.py b/src/swell/tasks/eva_comparison_jedi_log.py index 46f5de944..f9b9da042 100644 --- a/src/swell/tasks/eva_comparison_jedi_log.py +++ b/src/swell/tasks/eva_comparison_jedi_log.py @@ -13,7 +13,7 @@ from eva.eva_driver import eva -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.jinja2 import template_string_jinja2 from swell.utilities.comparisons import comparison_tags @@ -32,7 +32,7 @@ def execute(self) -> None: # Get the log type # ---------------- - log_type = self.config.resolve(comparison_log_type) + log_type = self.config.resolve(qd.comparison_log_type) # Read Eva template file into dictionary # -------------------------------------- @@ -42,7 +42,7 @@ def execute(self) -> None: eva_str_template = eva_config_file_open.read() # Get the paths for the two experiments - experiment_paths = self.config.resolve(comparison_experiment_paths) + experiment_paths = self.config.resolve(qd.comparison_experiment_paths) experiment_tag_paths = comparison_tags(experiment_paths, self.logger) diff --git a/src/swell/tasks/eva_comparison_observations.py b/src/swell/tasks/eva_comparison_observations.py index 9716a2bc5..a6a748769 100644 --- a/src/swell/tasks/eva_comparison_observations.py +++ b/src/swell/tasks/eva_comparison_observations.py @@ -16,7 +16,7 @@ from swell.swell_path import get_swell_path from swell.deployment.platforms.platforms import login_or_compute -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 @@ -42,11 +42,11 @@ def execute(self) -> None: # Comparison log type # ------------------- - log_type = self.config.resolve(comparison_log_type) + log_type = self.config.resolve(qd.comparison_log_type) # Get the experiment paths # ------------------------ - experiment_paths = self.config.resolve(comparison_experiment_paths) + experiment_paths = self.config.resolve(qd.comparison_experiment_paths) experiment_tag_paths = comparison_tags(experiment_paths, self.logger) @@ -126,7 +126,7 @@ def execute(self) -> None: eva_dicts = [] # Empty list of dictionaries # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) for observation in observations: if self.get_model() == 'geos_atmosphere': diff --git a/src/swell/tasks/eva_increment.py b/src/swell/tasks/eva_increment.py index da2df4eca..a52f7613d 100644 --- a/src/swell/tasks/eva_increment.py +++ b/src/swell/tasks/eva_increment.py @@ -13,7 +13,7 @@ from eva.eva_driver import eva -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.jinja2 import template_string_jinja2 @@ -27,10 +27,10 @@ def execute(self) -> None: # Get the model and window type # ----------------------------- model = self.get_model() - window_type = self.config.resolve(window_type) + window_type = self.config.resolve(qd.window_type) if model == 'geos_marine': - marine_models = self.config.resolve(marine_models) + marine_models = self.config.resolve(qd.marine_models) # Read Eva template file into dictionary # -------------------------------------- @@ -47,12 +47,12 @@ def execute(self) -> None: # Create time strings for eva_override directory cycle_time_reformat = self.cycle_time_dto().strftime('%Y%m%d_%H%M%Sz') - window_begin_dto = self.da_window_params.window_begin(self.config.resolve(window_length), + window_begin_dto = self.da_window_params.window_begin(self.config.resolve(qd.window_length), dto=True) window_begin = window_begin_dto.strftime('%Y%m%d_%H%M%Sz') local_bkg_dir, local_bkg_dto = self.da_window_params.local_background_time( - self.config.window_length(), self.config.resolve(window_length), dto=True) + self.config.window_length(), self.config.resolve(qd.window_length), dto=True) local_bkg_time = local_bkg_dto.strftime('%Y%m%d_%H%M%Sz') # Define the increment filename and path diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index 3b2fc5465..eab6ae81c 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -13,7 +13,7 @@ from eva.eva_driver import eva -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.jinja2 import template_string_jinja2 diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 455f84db8..2ee68e9ed 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -15,7 +15,7 @@ from eva.eva_driver import eva from swell.deployment.platforms.platforms import login_or_compute -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary from swell.utilities.jinja2 import template_string_jinja2 @@ -37,24 +37,24 @@ class EvaObservations(taskBase): def execute(self) -> None: - window_length = self.config.resolve(window_length) + window_length = self.config.resolve(qd.window_length) # Compute window beginning time # ----------------------------- window_begin = self.da_window_params.window_begin(window_length) background_time = self.da_window_params.background_time( - self.config.resolve(background_time_offset)) + self.config.resolve(qd.background_time_offset)) # Create JEDI interface config templates dictionary # ------------------------------------------------- self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Get the model # ------------- model = self.get_model() - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) # Determine if running on login or compute node and set workers # ------------------------------------------------------------- @@ -95,11 +95,11 @@ def execute(self) -> None: eva_dicts = [] # Empty list of dictionaries # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) yaml = YAML(typ='safe') - for observation in self.config.resolve(observations): + for observation in self.config.resolve(qd.observations): # Load the observation dictionary observation_dict = self.jedi_rendering.render_interface_observations(observation) diff --git a/src/swell/tasks/eva_timeseries.py b/src/swell/tasks/eva_timeseries.py index 545e65152..225aa626c 100644 --- a/src/swell/tasks/eva_timeseries.py +++ b/src/swell/tasks/eva_timeseries.py @@ -17,7 +17,7 @@ from eva.eva_driver import eva from swell.deployment.platforms.platforms import login_or_compute -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats from swell.utilities.dictionary import remove_matching_keys, replace_string_in_dictionary @@ -39,15 +39,15 @@ class EvaTimeseries(taskBase): def execute(self) -> None: - window_length = self.config.resolve(window_length) + window_length = self.config.resolve(qd.window_length) # Compute window beginning time # ----------------------------- window_begin = self.da_window_params.window_begin(window_length) background_time = self.da_window_params.background_time( - self.config.resolve(background_time_offset)) + self.config.resolve(qd.background_time_offset)) - ncdiag_experiments = self.config.resolve(ncdiag_experiments) + ncdiag_experiments = self.config.resolve(qd.ncdiag_experiments) # Use built-in methods to get the start and end cycle points # ---------------------------------------------------------- @@ -71,14 +71,14 @@ def execute(self) -> None: # Create JEDI interface config templates dictionary # ------------------------------------------------- self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) self.jedi_rendering.add_key('window_length', window_length) # Get the model # ------------- model = self.get_model() - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) # Determine if running on login or compute node and set workers # ------------------------------------------------------------- @@ -117,11 +117,11 @@ def execute(self) -> None: eva_dicts = [] # Empty list of dictionaries # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) yaml = YAML(typ='safe') - for observation in self.config.resolve(observations): + for observation in self.config.resolve(qd.observations): # Load the observation dictionary observation_dict = self.jedi_rendering.render_interface_observations(observation) diff --git a/src/swell/tasks/generate_b_climatology.py b/src/swell/tasks/generate_b_climatology.py index 269871794..d4da78321 100644 --- a/src/swell/tasks/generate_b_climatology.py +++ b/src/swell/tasks/generate_b_climatology.py @@ -8,7 +8,7 @@ import os from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.shell_commands import run_track_log_subprocess from swell.utilities.file_system_operations import check_if_files_exist_in_path @@ -223,12 +223,12 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_length = self.config.resolve(window_length) - window_type = self.config.resolve(window_type) - background_error_model = self.config.resolve(background_error_model) + window_length = self.config.resolve(qd.window_length) + window_type = self.config.resolve(qd.window_type) + background_error_model = self.config.resolve(qd.background_error_model) - swell_static_files_user = self.config.resolve(swell_static_files_user, default=None) - self.swell_static_files = self.config.resolve(swell_static_files) + swell_static_files_user = self.config.resolve(qd.swell_static_files_user, default=None) + self.swell_static_files = self.config.resolve(qd.swell_static_files) # Use static_files_user if present in config and contains files # ------------------------------------------------------------- @@ -238,18 +238,18 @@ def execute(self) -> None: self.logger.info(f'Using swell static files in {swell_static_files_user}') self.swell_static_files = swell_static_files_user - self.horizontal_resolution = self.config.resolve(horizontal_resolution) - self.vertical_resolution = self.config.resolve(vertical_resolution) - self.generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + self.horizontal_resolution = self.config.resolve(qd.horizontal_resolution) + self.vertical_resolution = self.config.resolve(qd.vertical_resolution) + self.generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) # Get the JEDI interface for this model component # ----------------------------------------------- - self.jedi_rendering.add_key('npx_proc', self.config.resolve(npx_proc, default=None)) - self.jedi_rendering.add_key('npy_proc', self.config.resolve(npy_proc, default=None)) - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) - self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) - self.jedi_rendering.add_key('background_error_model', self.config.resolve(background_error_model)) - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.jedi_rendering.add_key('npx_proc', self.config.resolve(qd.npx_proc, default=None)) + self.jedi_rendering.add_key('npy_proc', self.config.resolve(qd.npy_proc, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(qd.analysis_variables)) + self.jedi_rendering.add_key('background_error_model', self.config.resolve(qd.background_error_model)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) # Compute data assimilation window parameters # ------------------------------------------- local_background_time = self.da_window_params.local_background_time(window_length, diff --git a/src/swell/tasks/generate_b_climatology_by_linking.py b/src/swell/tasks/generate_b_climatology_by_linking.py index 0fe4b53fb..e88e944f2 100644 --- a/src/swell/tasks/generate_b_climatology_by_linking.py +++ b/src/swell/tasks/generate_b_climatology_by_linking.py @@ -7,7 +7,7 @@ # -------------------------------------------------------------------------------------------------- import os -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import link_all_files_from_first_in_hierarchy_of_sources @@ -42,16 +42,16 @@ def execute(self) -> None: # Get the flavor of static background error model # ----------------------------------------------- - background_error_model = self.config.resolve(background_error_model) + background_error_model = self.config.resolve(qd.background_error_model) # Get the JEDI interface for this model component # ----------------------------------------------- - self.jedi_rendering.add_key('background_error_model', self.config.resolve(background_error_model)) + self.jedi_rendering.add_key('background_error_model', self.config.resolve(qd.background_error_model)) # Extract general parts of the config # ----------------------------------- - swell_static_files_main = self.config.resolve(swell_static_files) - swell_static_files_user = self.config.resolve(swell_static_files_user, default=None) + swell_static_files_main = self.config.resolve(qd.swell_static_files) + swell_static_files_user = self.config.resolve(qd.swell_static_files_user, default=None) # Set the destination directory # ------------------------------ @@ -65,8 +65,8 @@ def execute(self) -> None: # Compute data assimilation window parameters to obtain the local background time # ------------------------------------------------------------------------------- - window_length = self.config.resolve(window_length) - window_type = self.config.resolve(window_type) + window_length = self.config.resolve(qd.window_length) + window_type = self.config.resolve(qd.window_type) local_background_time = self.da_window_params.local_background_time(window_length, window_type) @@ -80,8 +80,8 @@ def execute(self) -> None: # long run. # ------------------------------------------------------------------------------ if background_error_model == 'explicit_diffusion': - horizontal_resolution = self.config.resolve(horizontal_resolution) - vertical_resolution = self.config.resolve(vertical_resolution) + horizontal_resolution = self.config.resolve(qd.horizontal_resolution) + vertical_resolution = self.config.resolve(qd.vertical_resolution) res_path = horizontal_resolution + 'x' + vertical_resolution correlation_files = 'vt.' + local_background_time + '.nc' diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index 6dc678496..5035a7035 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -10,7 +10,7 @@ import os -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.observing_system_records import ObservingSystemRecords @@ -33,21 +33,21 @@ def execute(self) -> None: # Parse GSI records and save yamls # -------------------------------- - observing_system_records_path = self.config.resolve(observing_system_records_path, default=None) + observing_system_records_path = self.config.resolve(qd.observing_system_records_path, default=None) if observing_system_records_path == 'None': cycle_dir = self.cycle_dir() observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') observing_system_records_path = os.path.expanduser(observing_system_records_path) - path_to_geos_mksi = os.path.expanduser(self.config.resolve(observing_system_records_mksi_path)) + path_to_geos_mksi = os.path.expanduser(self.config.resolve(qd.observing_system_records_mksi_path)) if path_to_geos_mksi == 'None': path_to_geos_mksi = os.path.join(self.experiment_path(), 'GEOS_mksi') # Parse sidb # ---------- record_type = 'channel' - observations = self.config.resolve(observations) + observations = self.config.resolve(qd.observations) path_to_gsi_records = os.path.join(path_to_geos_mksi, 'sidb') sat_records = ObservingSystemRecords(record_type) sat_records.parse_records(path_to_gsi_records) diff --git a/src/swell/tasks/get_background.py b/src/swell/tasks/get_background.py index b964cd91a..7e7e9f844 100644 --- a/src/swell/tasks/get_background.py +++ b/src/swell/tasks/get_background.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials, get_r2d2_model_name @@ -56,29 +56,29 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.resolve(r2d2_server, default=None), + r2d2_server=self.config.resolve(qd.r2d2_server, default=None), ) - r2d2_datastore = self.config.resolve(r2d2_datastore, default=None) + r2d2_datastore = self.config.resolve(qd.r2d2_datastore, default=None) # Get duration into forecast for first background file # ---------------------------------------------------- bkg_steps = [] # Parse config - background_frequency = self.config.resolve(background_frequency, default=None) - horizontal_resolution = self.config.resolve(horizontal_resolution) - window_length = self.config.resolve(window_length) - window_type = self.config.resolve(window_type) + background_frequency = self.config.resolve(qd.background_frequency, default=None) + horizontal_resolution = self.config.resolve(qd.horizontal_resolution) + window_length = self.config.resolve(qd.window_length) + window_type = self.config.resolve(qd.window_type) # For experiments with cycle in the suite name: # for the first cycle, use background_experiment in config # as the experiment id for fetching from r2d2 for cycles after # the first, use the current experiment id for fetching from r2d2 if self.cycle_time_dto() != self.start_cycle_point_dto() and 'cycle' in self.suite_name(): - background_experiment = self.config.resolve(r2d2_experiment_id) + background_experiment = self.config.resolve(qd.r2d2_experiment_id) else: - background_experiment = self.config.resolve(background_experiment) + background_experiment = self.config.resolve(qd.background_experiment) self.logger.info(f'Fetching background from experiment {background_experiment}') @@ -89,7 +89,7 @@ def execute(self) -> None: # Add to jedi config rendering dictionary self.jedi_rendering.add_key('local_background_time', local_background_time) - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) self.jedi_rendering.add_key('analysis_time_iso', analysis_time_iso) # Convert to datetime durations diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py index 2417e6494..f33f15df3 100644 --- a/src/swell/tasks/get_background_geos_experiment.py +++ b/src/swell/tasks/get_background_geos_experiment.py @@ -13,7 +13,7 @@ import os import tarfile -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats @@ -45,9 +45,9 @@ def execute(self): # Parse config # ------------ - background_experiment = self.config.resolve(background_experiment) - geos_x_background_directory = self.config.resolve(geos_x_background_directory) - background_time_offset = self.config.resolve(background_time_offset) + background_experiment = self.config.resolve(qd.background_experiment) + geos_x_background_directory = self.config.resolve(qd.geos_x_background_directory) + background_time_offset = self.config.resolve(qd.background_time_offset) # Since this is an optional task, check if the geos_x_background_directory is # set to /dev/null, if so fail the task diff --git a/src/swell/tasks/get_bufr.py b/src/swell/tasks/get_bufr.py index 1c406b143..f1d03a6e7 100644 --- a/src/swell/tasks/get_bufr.py +++ b/src/swell/tasks/get_bufr.py @@ -13,7 +13,7 @@ from datetime import datetime as dt from swell.utilities.datetime_util import datetime_formats -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -44,7 +44,7 @@ def execute(self) -> None: perl_executable_path = os.path.join(self.experiment_path(), 'GMAO_perllib') geos_mksi_obsysrc_path = os.path.join(self.experiment_path(), - self.config.resolve(obs_rc_path)) + self.config.resolve(qd.obs_rc_path)) # Environment variables for acquire_obsys env_dict = os.environ @@ -60,7 +60,7 @@ def execute(self) -> None: # Get BUFR obs classes from mksi and acquire them # ----------------------------------------------- - obsclasses = self.config.resolve(bufr_obs_classes) + obsclasses = self.config.resolve(qd.bufr_obs_classes) for obsclass in obsclasses: diff --git a/src/swell/tasks/get_coupled_geos_restart.py b/src/swell/tasks/get_coupled_geos_restart.py index 891c40360..aa5fbd295 100644 --- a/src/swell/tasks/get_coupled_geos_restart.py +++ b/src/swell/tasks/get_coupled_geos_restart.py @@ -10,7 +10,7 @@ import os import glob -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import copy_to_dst_dir @@ -48,7 +48,7 @@ def execute(self) -> None: swell_exp_path = self.experiment_path() # Obtain GEOS HOMDIR from user input - self.geos_homdir = self.config.resolve(geos_homdir) + self.geos_homdir = self.config.resolve(qd.geos_homdir) self.logger.info(f'GEOS HOME directory: {self.geos_homdir}') # Create GEOSgcm directory in the experiment folder if it doesn't exist yet @@ -70,8 +70,8 @@ def execute(self) -> None: geos_expdir_path = geos_homdir_path # If GEOS expdir is set to be different to homdir, create a link to expdir - if self.config.resolve(geos_expdir_different): - self.geos_expdir = self.config.resolve(geos_expdir) + if self.config.resolve(qd.geos_expdir_different): + self.geos_expdir = self.config.resolve(qd.geos_expdir) self.logger.info(f'GEOS EXPERIMENT directory: {self.geos_expdir}') if not os.path.exists(self.geos_expdir): @@ -109,7 +109,7 @@ def initial_restarts(self, geos_expdir_path: str) -> None: # 3) Hotstart, just use the existing restarts in the forecast directory (e.g., from a # previous run or manually placed there) # ---------------------------------------------------- - initial_restarts_method = self.config.resolve(initial_restarts_method, default='geos_expdir') + initial_restarts_method = self.config.resolve(qd.initial_restarts_method, default='geos_expdir') if initial_restarts_method == 'geos_expdir': self.initial_restarts_from_directory(geos_expdir_path) diff --git a/src/swell/tasks/get_ensemble_geos_experiment.py b/src/swell/tasks/get_ensemble_geos_experiment.py index ea234471b..abee4dded 100644 --- a/src/swell/tasks/get_ensemble_geos_experiment.py +++ b/src/swell/tasks/get_ensemble_geos_experiment.py @@ -12,7 +12,7 @@ import os import tarfile -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats @@ -44,9 +44,9 @@ def execute(self): # Parse config # ------------ - background_experiment = self.config.resolve(background_experiment) - geos_x_ensemble_directory = self.config.resolve(geos_x_ensemble_directory) - background_time_offset = self.config.resolve(ebkg_time_offset) + background_experiment = self.config.resolve(qd.background_experiment) + geos_x_ensemble_directory = self.config.resolve(qd.geos_x_ensemble_directory) + background_time_offset = self.config.resolve(qd.ebkg_time_offset) # Since this is an optional task, check if the geos_x_ensemble_directory is # set to /dev/null, if so fail the task diff --git a/src/swell/tasks/get_geos_adas_background.py b/src/swell/tasks/get_geos_adas_background.py index ec83cf31e..a5c1493a2 100644 --- a/src/swell/tasks/get_geos_adas_background.py +++ b/src/swell/tasks/get_geos_adas_background.py @@ -13,7 +13,7 @@ import os import re -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase @@ -26,7 +26,7 @@ def execute(self) -> None: # Get the path and pattern for the background files # ------------------------------------------------- - background_path = self.config.resolve(path_to_geos_adas_background) + background_path = self.config.resolve(qd.path_to_geos_adas_background) # Get list of ncdiags to test with # -------------------------------- diff --git a/src/swell/tasks/get_geovals.py b/src/swell/tasks/get_geovals.py index 257cd2526..73f4349fe 100644 --- a/src/swell/tasks/get_geovals.py +++ b/src/swell/tasks/get_geovals.py @@ -10,7 +10,7 @@ import os -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials from r2d2 import fetch @@ -27,17 +27,17 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.resolve(r2d2_server, default=None), + r2d2_server=self.config.resolve(qd.r2d2_server, default=None), ) # Parse config # ------------ - geovals_experiment = self.config.resolve(geovals_experiment) - geovals_provider = self.config.resolve(geovals_provider) - background_time_offset = self.config.resolve(background_time_offset) - observations = self.config.resolve(observations) - window_length = self.config.resolve(window_length) - crtm_coeff_dir = self.config.resolve(crtm_coeff_dir, default=None) + geovals_experiment = self.config.resolve(qd.geovals_experiment) + geovals_provider = self.config.resolve(qd.geovals_provider) + background_time_offset = self.config.resolve(qd.background_time_offset) + observations = self.config.resolve(qd.observations) + window_length = self.config.resolve(qd.window_length) + crtm_coeff_dir = self.config.resolve(qd.crtm_coeff_dir, default=None) # Get window begin time window_begin = self.da_window_params.window_begin(window_length) diff --git a/src/swell/tasks/get_gsi_bc.py b/src/swell/tasks/get_gsi_bc.py index 86050c2fe..d20e6b1f1 100644 --- a/src/swell/tasks/get_gsi_bc.py +++ b/src/swell/tasks/get_gsi_bc.py @@ -14,7 +14,7 @@ import shutil import tarfile -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase @@ -27,8 +27,8 @@ def execute(self) -> None: # Get the build method # -------------------- - gsi_bc_location = self.config.resolve(path_to_gsi_bc_coefficients) - window_length = self.config.resolve(window_length) + gsi_bc_location = self.config.resolve(qd.path_to_gsi_bc_coefficients) + window_length = self.config.resolve(qd.window_length) # Time of GSI analysis providing the bias correction coefficients # --------------------------------------------------------------- diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py index b58d4796f..0103d6ebf 100644 --- a/src/swell/tasks/get_gsi_ncdiag.py +++ b/src/swell/tasks/get_gsi_ncdiag.py @@ -11,7 +11,7 @@ import glob import os -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase @@ -24,7 +24,7 @@ def execute(self) -> None: # Get the build method # -------------------- - gsi_diag_path = self.config.resolve(path_to_gsi_nc_diags) + gsi_diag_path = self.config.resolve(qd.path_to_gsi_nc_diags) # Replace gsi_diag_path datetime string with the actual datetime # -------------------------------------------------------------- diff --git a/src/swell/tasks/get_ncdiags.py b/src/swell/tasks/get_ncdiags.py index 97a275592..561ac747d 100644 --- a/src/swell/tasks/get_ncdiags.py +++ b/src/swell/tasks/get_ncdiags.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- import os -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from r2d2 import fetch from swell.utilities.r2d2 import load_r2d2_credentials @@ -29,15 +29,15 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.resolve(r2d2_server, default=None), + r2d2_server=self.config.resolve(qd.r2d2_server, default=None), ) # Parse config # ------------ - ncdiag_experiments = self.config.resolve(ncdiag_experiments) - observations = self.config.resolve(observations) - window_length = self.config.resolve(window_length) - background_time_offset = self.config.resolve(background_time_offset) + ncdiag_experiments = self.config.resolve(qd.ncdiag_experiments) + observations = self.config.resolve(qd.observations) + window_length = self.config.resolve(qd.window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) # Compute data assimilation window parameters # -------------------------------------------- @@ -47,11 +47,11 @@ def execute(self) -> None: self.jedi_rendering.add_key('background_time', background_time) # Set the JEDI rendering parameters. Model specific ones have None as default - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) # Loop over ncdiag experiments # ------------------------------- diff --git a/src/swell/tasks/get_obs_not_in_r2d2.py b/src/swell/tasks/get_obs_not_in_r2d2.py index e09c139a6..c5fefeeb6 100644 --- a/src/swell/tasks/get_obs_not_in_r2d2.py +++ b/src/swell/tasks/get_obs_not_in_r2d2.py @@ -12,7 +12,7 @@ import os import subprocess -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase @@ -29,7 +29,7 @@ def execute(self) -> None: # Get the path and pattern for the observation files # ------------------------------------------------- - existing_path = self.config.resolve(ioda_locations_not_in_r2d2) + existing_path = self.config.resolve(qd.ioda_locations_not_in_r2d2) # Point to the model directory # ---------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 0f7161701..b9b2cc13e 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -18,7 +18,7 @@ from concurrent.futures import ThreadPoolExecutor from datetime import timedelta, datetime as dt -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities import s3 as swell_s3 from swell.utilities.r2d2 import load_r2d2_credentials, get_r2d2_model_name @@ -211,32 +211,32 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.resolve(r2d2_server, default=None), + r2d2_server=self.config.resolve(qd.r2d2_server, default=None), ) - r2d2_datastore = self.config.resolve(r2d2_datastore, default=None) + r2d2_datastore = self.config.resolve(qd.r2d2_datastore, default=None) # Parse config # ------------ - obs_experiment = self.config.resolve(obs_experiment) - background_time_offset = self.config.resolve(background_time_offset) - observations = self.config.resolve(observations) - observation_providers = self.config.resolve(observation_providers, default={}) - window_length = self.config.resolve(window_length) - crtm_coeff_dir = self.config.resolve(crtm_coeff_dir, default=None) - window_length = self.config.resolve(window_length) - cycling_varbc = self.config.resolve(cycling_varbc, default=None) - cache_fetch = self.config.resolve(cache_fetch, default=True) + obs_experiment = self.config.resolve(qd.obs_experiment) + background_time_offset = self.config.resolve(qd.background_time_offset) + observations = self.config.resolve(qd.observations) + observation_providers = self.config.resolve(qd.observation_providers, default={}) + window_length = self.config.resolve(qd.window_length) + crtm_coeff_dir = self.config.resolve(qd.crtm_coeff_dir, default=None) + window_length = self.config.resolve(qd.window_length) + cycling_varbc = self.config.resolve(qd.cycling_varbc, default=None) + cache_fetch = self.config.resolve(qd.cache_fetch, default=True) # When True, observations that have a fetch_observations_s3 # registry entry are pulled directly from a public S3 bucket instead of # R2D2. When False (default) every observation goes through R2D2 as before. - fetch_obs_from_public_s3 = self.config.resolve(fetch_obs_from_public_s3, default=False) + fetch_obs_from_public_s3 = self.config.resolve(qd.fetch_obs_from_public_s3, default=False) # Get model component and translate to R2D2 model name model_component = self.get_model() r2d2_model = get_r2d2_model_name(model_component) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) # Get window begin time window_begin = self.da_window_params.window_begin(window_length) @@ -260,7 +260,7 @@ def execute(self) -> None: self.jedi_rendering.add_key('background_time', background_time) self.jedi_rendering.add_key('crtm_coeff_dir', crtm_coeff_dir) self.jedi_rendering.add_key('window_begin', window_begin) - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) # Read observation ioda names ioda_names_list = get_ioda_names_list() diff --git a/src/swell/tasks/get_restart_cf.py b/src/swell/tasks/get_restart_cf.py index 4caa701da..50f0dd86c 100644 --- a/src/swell/tasks/get_restart_cf.py +++ b/src/swell/tasks/get_restart_cf.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials @@ -30,15 +30,15 @@ def execute(self) -> None: # --------------------- load_r2d2_credentials(self.logger, self.platform()) - rst_file_types = self.config.resolve(rst_file_types) + rst_file_types = self.config.resolve(qd.rst_file_types) cycle_dir = self.cycle_dir() scratch_dir = os.path.join(cycle_dir, 'scratch') model = self.__model__ - horizontal_resolution = self.config.resolve(horizontal_resolution) + horizontal_resolution = self.config.resolve(qd.horizontal_resolution) - window_length = self.config.resolve(window_length) + window_length = self.config.resolve(qd.window_length) window_begin = self.da_window_params.window_begin(window_length, dto=True) window_begin_prev = window_begin - isodate.parse_duration(window_length) @@ -46,9 +46,9 @@ def execute(self) -> None: # Use rst_experiment for first cycle if self.cycle_time_dto() == self.start_cycle_point_dto(): - rst_exp = self.config.resolve(rst_experiment) + rst_exp = self.config.resolve(qd.rst_experiment) else: - rst_exp = self.config.resolve(r2d2_experiment_id) + rst_exp = self.config.resolve(qd.r2d2_experiment_id) self.logger.info(f'Fetching rst files from experiment {rst_exp}') diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 859a69785..966433c3f 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -12,7 +12,7 @@ import os -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.dictionary import write_dict_to_yaml from swell.utilities.shell_commands import run_track_log_subprocess @@ -27,10 +27,10 @@ def execute(self) -> None: # Parse configuration # ------------------- - observations = self.config.resolve(observations) - background_time_offset = self.config.resolve(background_time_offset) - window_length = self.config.resolve(window_length) - crtm_coeff_dir = self.config.resolve(crtm_coeff_dir, default=None) + observations = self.config.resolve(qd.observations) + background_time_offset = self.config.resolve(qd.background_time_offset) + window_length = self.config.resolve(qd.window_length) + crtm_coeff_dir = self.config.resolve(qd.crtm_coeff_dir, default=None) # Get window beginning time window_begin = self.da_window_params.window_begin(window_length) @@ -48,7 +48,7 @@ def execute(self) -> None: sensors_tlapse = [] # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) for observation in observations: diff --git a/src/swell/tasks/gsi_ncdiag_to_ioda.py b/src/swell/tasks/gsi_ncdiag_to_ioda.py index 9cda64983..3dc31f2e4 100644 --- a/src/swell/tasks/gsi_ncdiag_to_ioda.py +++ b/src/swell/tasks/gsi_ncdiag_to_ioda.py @@ -18,7 +18,7 @@ import pyiodaconv.gsi_ncdiag as gsid from pyiodaconv.combine_obsspace import combine_obsspace -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats from swell.utilities.shell_commands import run_subprocess, create_executable_file @@ -33,10 +33,10 @@ def execute(self) -> None: # Parse configuration # ------------------- - observations = self.config.resolve(observations) - single_observations = self.config.resolve(single_observations) - produce_geovals = self.config.resolve(produce_geovals) - window_length = self.config.resolve(window_length) + observations = self.config.resolve(qd.observations) + single_observations = self.config.resolve(qd.single_observations) + produce_geovals = self.config.resolve(qd.produce_geovals) + window_length = self.config.resolve(qd.window_length) # Get window beginning time window_begin = self.da_window_params.window_begin(window_length) diff --git a/src/swell/tasks/ingest_obs.py b/src/swell/tasks/ingest_obs.py index 697b5f1bb..995697170 100644 --- a/src/swell/tasks/ingest_obs.py +++ b/src/swell/tasks/ingest_obs.py @@ -17,7 +17,7 @@ import requests -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials from swell.utilities.observations import get_ioda_names_list, get_provider_for_observation @@ -73,19 +73,19 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.resolve(r2d2_server, default=None), + r2d2_server=self.config.resolve(qd.r2d2_server, default=None), ) - r2d2_datastore = self.config.resolve(r2d2_datastore, default=None) + r2d2_datastore = self.config.resolve(qd.r2d2_datastore, default=None) # Get list of observations to ingest (strings) - obs_to_ingest = self.config.resolve(obs_to_ingest, default=[]) + obs_to_ingest = self.config.resolve(qd.obs_to_ingest, default=[]) # Read observation ioda names (for provider lookup) self.ioda_names_list = get_ioda_names_list() # Get window parameters - window_length = self.config.resolve(window_length) + window_length = self.config.resolve(qd.window_length) # Beginning of the DA window window_start = self.da_window_params.window_begin_iso(window_length) @@ -94,13 +94,13 @@ def execute(self) -> None: cycle_time = self.cycle_time() # Check for dry-run mode (default True for safety) - dry_run = self.config.resolve(dry_run, default=True) + dry_run = self.config.resolve(qd.dry_run, default=True) if dry_run: self.logger.info( "DRY RUN MODE - No files will be ingested to R2D2") - store_as_symlink = self.config.resolve(store_as_symlink, default=False) + store_as_symlink = self.config.resolve(qd.store_as_symlink, default=False) total_ingested = 0 total_failed = 0 diff --git a/src/swell/tasks/jedi_log_comparison.py b/src/swell/tasks/jedi_log_comparison.py index 11324937b..73f00d95a 100644 --- a/src/swell/tasks/jedi_log_comparison.py +++ b/src/swell/tasks/jedi_log_comparison.py @@ -13,7 +13,7 @@ import re import numpy as np -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.comparisons import comparison_tags @@ -28,7 +28,7 @@ class JediLogComparison(taskBase): def execute(self): - experiment_paths = self.config.resolve(comparison_experiment_paths) + experiment_paths = self.config.resolve(qd.comparison_experiment_paths) experiment_tag_paths = comparison_tags(experiment_paths, self.logger) @@ -60,7 +60,7 @@ def execute(self): # Boolean for whether fields fall within tolerances passed = True - log_type = self.config.resolve(comparison_log_type) + log_type = self.config.resolve(qd.comparison_log_type) for exp_tag, experiment_path in experiment_tag_paths.items(): diff --git a/src/swell/tasks/jedi_oops_log_parser.py b/src/swell/tasks/jedi_oops_log_parser.py index abd75dd5e..3fbdd7a36 100644 --- a/src/swell/tasks/jedi_oops_log_parser.py +++ b/src/swell/tasks/jedi_oops_log_parser.py @@ -11,7 +11,7 @@ import os import subprocess -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -26,7 +26,7 @@ def fgrep_residual_norm(self, output_file): cycle_time = self.__datetime__.string_directory() model = self.get_model() - log_type = self.config.resolve(comparison_log_type) + log_type = self.config.resolve(qd.comparison_log_type) # Build the command command = ['fgrep', '"Residual norm"'] + [ @@ -49,7 +49,7 @@ def execute(self) -> None: output_file = os.path.join(self.cycle_dir(), 'jedi_log_analysis.txt') - for parser_option in self.config.resolve(parser_options, default=['fgrep_residual_norm']): + for parser_option in self.config.resolve(qd.parser_options, default=['fgrep_residual_norm']): if parser_option == 'fgrep_residual_norm': self.fgrep_residual_norm(output_file) diff --git a/src/swell/tasks/link_coupled_geos_output.py b/src/swell/tasks/link_coupled_geos_output.py index f0847c576..b1dca9185 100644 --- a/src/swell/tasks/link_coupled_geos_output.py +++ b/src/swell/tasks/link_coupled_geos_output.py @@ -16,7 +16,7 @@ from typing import Tuple from swell.utilities.datetime_util import datetime_formats -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -37,13 +37,13 @@ def execute(self) -> None: # Parse configuration # ------------------- - self.marine_models = self.config.resolve(marine_models, default=None) or [] - self.window_type = self.config.resolve(window_type) - self.window_length = self.config.resolve(window_length) + self.marine_models = self.config.resolve(qd.marine_models, default=None) or [] + self.window_type = self.config.resolve(qd.window_type) + self.window_length = self.config.resolve(qd.window_length) self.window_begin_iso = self.da_window_params.window_begin_iso(self.window_length) if self.window_type == '4D' or 'fgat' in self.suite_name(): - self.background_frequency = self.config.resolve(background_frequency) + self.background_frequency = self.config.resolve(qd.background_frequency) self.bkgr_time_iso, self.bkgr_time_dto = self.da_window_params.local_background_time( self.window_length, diff --git a/src/swell/tasks/link_geos_output.py b/src/swell/tasks/link_geos_output.py index fca1bb95c..5e87eb078 100644 --- a/src/swell/tasks/link_geos_output.py +++ b/src/swell/tasks/link_geos_output.py @@ -16,7 +16,7 @@ from typing import Tuple from swell.utilities.datetime_util import datetime_formats -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -36,13 +36,13 @@ def execute(self) -> None: # Parse configuration # ------------------- - self.marine_models = self.config.resolve(marine_models, default=None) or [] - self.window_type = self.config.resolve(window_type) - self.window_length = self.config.resolve(window_length) + self.marine_models = self.config.resolve(qd.marine_models, default=None) or [] + self.window_type = self.config.resolve(qd.window_type) + self.window_length = self.config.resolve(qd.window_length) self.window_begin_iso = self.da_window_params.window_begin_iso(self.window_length) if self.window_type == '4D' or 'fgat' in self.suite_name(): - self.background_frequency = self.config.resolve(background_frequency) + self.background_frequency = self.config.resolve(qd.background_frequency) self.bkgr_time_iso, self.bkgr_time_dto = self.da_window_params.local_background_time( self.window_length, diff --git a/src/swell/tasks/move_da_restart.py b/src/swell/tasks/move_da_restart.py index 2ac88a570..06c7d0ced 100644 --- a/src/swell/tasks/move_da_restart.py +++ b/src/swell/tasks/move_da_restart.py @@ -12,7 +12,7 @@ import os import re -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import move_files @@ -38,8 +38,8 @@ def execute(self) -> None: # Obtain MOM6 IAU bool # ---------------------- - self.mom6_iau = self.config.resolve(mom6_iau) - self.jedi_rendering.add_key('mom6_iau', self.config.resolve(mom6_iau, default=False)) + self.mom6_iau = self.config.resolve(qd.mom6_iau) + self.jedi_rendering.add_key('mom6_iau', self.config.resolve(qd.mom6_iau, default=False)) # Current and restart time objects # -------------------------------- @@ -77,7 +77,7 @@ def move_restarts(self) -> None: if int(agcm_dict.get('RECORD_FREQUENCY', '0')) > 0: self.logger.info('RECORD_FREQUENCY is turned on in AGCM.rc. Using _checkpoint files ' 'with timestamps.') - window_length = self.config.resolve(window_length) + window_length = self.config.resolve(qd.window_length) an_fcst_offset = self.da_window_params.analysis_forecast_window_offset(window_length) rst_dto = self.cc_dto + isodate.parse_duration(an_fcst_offset) @@ -119,7 +119,7 @@ def move_marine_restarts(self) -> None: if int(agcm_dict.get('RECORD_FREQUENCY', '0')) > 0: - window_length = self.config.resolve(window_length) + window_length = self.config.resolve(qd.window_length) an_fcst_offset = self.da_window_params.analysis_forecast_window_offset(window_length) rst_dto = self.cc_dto + isodate.parse_duration(an_fcst_offset) diff --git a/src/swell/tasks/move_forecast_restart.py b/src/swell/tasks/move_forecast_restart.py index e5a32da82..92037e958 100644 --- a/src/swell/tasks/move_forecast_restart.py +++ b/src/swell/tasks/move_forecast_restart.py @@ -10,7 +10,7 @@ import os import glob -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import move_files @@ -33,7 +33,7 @@ def execute(self) -> None: # Next cycle folder name # ----------------------- - self.forecast_duration = self.config.resolve(forecast_duration) + self.forecast_duration = self.config.resolve(qd.forecast_duration) # Create cycle_dir and RESTART # ---------------------------- diff --git a/src/swell/tasks/prep_coupled_geos_run_dir.py b/src/swell/tasks/prep_coupled_geos_run_dir.py index 7ce19e8f4..0d5e2269b 100644 --- a/src/swell/tasks/prep_coupled_geos_run_dir.py +++ b/src/swell/tasks/prep_coupled_geos_run_dir.py @@ -11,7 +11,7 @@ import isodate import re -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.file_system_operations import copy_to_dst_dir @@ -44,12 +44,12 @@ def execute(self) -> None: # config.s to the cycle forecast directory self.geos_homdir = os.path.join(self.experiment_path(), 'GEOSgcm', 'GEOS_homdir') - if self.config.resolve(geos_expdir_different): + if self.config.resolve(qd.geos_expdir_different): self.geos_expdir = os.path.join(self.experiment_path(), 'GEOSgcm', 'GEOS_expdir') else: self.geos_expdir = self.geos_homdir - self.geos_build = self.config.resolve(existing_geos_gcm_build_path) + self.geos_build = self.config.resolve(qd.existing_geos_gcm_build_path) self.logger.info('Preparing GEOS Forecast directory......') self.logger.info('Some steps involve modifying input files and replacing file contents.') @@ -60,7 +60,7 @@ def execute(self) -> None: # context. So forecast start will be assigned as cycle_date - forecast_duration * 3/4 # Need to convert forecast_duration to a datetime object first # ------------------------------------------------------------- - self.forecast_duration = self.config.resolve(forecast_duration) + self.forecast_duration = self.config.resolve(qd.forecast_duration) self.fc_dto = self.cycle_time_dto() - isodate.parse_duration(self.forecast_duration) * 3 / 4 # Get static files @@ -197,7 +197,7 @@ def mom6_iau(self) -> None: # file is located inside the INPUT directory. At the first cycle, mom6_increment.nc may not # be present in the INPUT directory, so this step is skipped. # -------------------------------------------------------------------------- - if self.config.resolve(get_key_for_model, default='mom6_iau', 'geos_marine', False): + if self.config.resolve(qd.get_key_for_model, default='mom6_iau', 'geos_marine', False): if os.path.exists(self.forecast_dir('RESTART/mom6_increment.nc')): self.logger.info('MOM6 Increment file found in RESTART directory') @@ -207,8 +207,8 @@ def mom6_iau(self) -> None: mom_oda_incupd = self.forecast_dir('MOM_oda_incupd') mom6_config = self.geos.parse_mom6_input(mom_oda_incupd) # P50D is just a random input for get_key_for_model to function -self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)6self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)hself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)=self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)lself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',).self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)cself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',).self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)tself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)kself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)yself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)dself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)lself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)(self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)6self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)hself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',),self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',),self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) -self.config.resolve(get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) 'PT50D') +self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)6self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)hself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)=self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)lself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',).self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)cself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',).self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)tself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)kself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)yself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)dself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)lself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)(self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)6self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)hself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',),self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',),self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) +self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) 'PT50D') # convert ISO to 3.0 duration = isodate.parse_duration(mom6_iau_nhours) @@ -262,7 +262,7 @@ def rewrite_cap(self, rcdict: dict, rcfile: str) -> dict: # This method returns rcdict with the bool fix # --------------------------------------------- self.logger.info('Modifying CAP.rc') - [time_string, days, _] = self.geos.iso_to_time_str(self.config.resolve(forecast_duration)) + [time_string, days, _] = self.geos.iso_to_time_str(self.config.resolve(qd.forecast_duration)) # Prepend day information # ----------------------- diff --git a/src/swell/tasks/prep_forecast_cf.py b/src/swell/tasks/prep_forecast_cf.py index 7da2291af..9c028735a 100644 --- a/src/swell/tasks/prep_forecast_cf.py +++ b/src/swell/tasks/prep_forecast_cf.py @@ -15,7 +15,7 @@ import xarray as xr from swell.configuration.jedi.interfaces.geos_cf.model.r2d2 import forecast_history -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.shell_commands import run_subprocess @@ -48,20 +48,20 @@ def execute(self) -> None: # Gather config values # -------------------- self.expid = self.experiment_id() - self.window_length = self.config.resolve(window_length) - self.forecast_length = self.config.resolve(forecast_length) - self.forecast_output_frequency = self.config.resolve(forecast_output_frequency) - self.resolution = self.config.resolve(horizontal_resolution) - self.an_vars_long = self.config.resolve(analysis_variables) + self.window_length = self.config.resolve(qd.window_length) + self.forecast_length = self.config.resolve(qd.forecast_length) + self.forecast_output_frequency = self.config.resolve(qd.forecast_output_frequency) + self.resolution = self.config.resolve(qd.horizontal_resolution) + self.an_vars_long = self.config.resolve(qd.analysis_variables) - self.geos_cf_run_dir = self.config.resolve(geos_cf_run_dir) - self.geos_cf_install_dir = self.config.resolve(geos_cf_install_dir) + self.geos_cf_run_dir = self.config.resolve(qd.geos_cf_run_dir) + self.geos_cf_install_dir = self.config.resolve(qd.geos_cf_install_dir) self.namelists_dir = os.path.join(self.experiment_config_path(), 'jedi', 'interfaces', 'geos_cf', 'namelists') - self.fp_exp = self.config.resolve(geosfp_exp) - self.fp_loc = self.config.resolve(geosfp_path) + self.fp_exp = self.config.resolve(qd.geosfp_exp) + self.fp_loc = self.config.resolve(qd.geosfp_path) # Derive window times # ------------------- @@ -111,7 +111,7 @@ def replace_string(self, filename: str, string1: str, string2: str) -> None: def create_geos_cf_increments(self) -> None: """Convert JEDI increment files to GEOS-CF format using the increment template.""" - inc_template = self.config.resolve(inc_template) + inc_template = self.config.resolve(qd.inc_template) tstring_date = (self.parse_andate.strftime('%Y-%m-%d') + ' ' + self.parse_andate.strftime('%H:%M:%S')) diff --git a/src/swell/tasks/prepare_analysis.py b/src/swell/tasks/prepare_analysis.py index eb032411d..f5ae31c38 100644 --- a/src/swell/tasks/prepare_analysis.py +++ b/src/swell/tasks/prepare_analysis.py @@ -15,7 +15,7 @@ from typing import Union from swell.utilities.shell_commands import run_subprocess -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -41,8 +41,8 @@ def execute(self) -> None: # This will change with different model types # -------------------------------- - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) - self.jedi_rendering.add_key('mom6_iau', self.config.resolve(mom6_iau, default=False)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) + self.jedi_rendering.add_key('mom6_iau', self.config.resolve(qd.mom6_iau, default=False)) model_component_meta = self.jedi_rendering.render_interface_meta() @@ -50,7 +50,7 @@ def execute(self) -> None: # -------------------------------- self.cc_dto = self.cycle_time_dto() - window_length = self.config.resolve(window_length) + window_length = self.config.resolve(qd.window_length) # GEOS restarts have seconds in their filename # We want to use rst_dto at the beginning of the DA window (window offset is negative) @@ -76,7 +76,7 @@ def execute(self) -> None: # Obtain MOM6 IAU bool # ---------------------- - mom6_iau = self.config.resolve(mom6_iau) + mom6_iau = self.config.resolve(qd.mom6_iau) # Generic rst file format # ------------------------ @@ -91,7 +91,7 @@ def execute(self) -> None: f_rst = self.forecast_dir(['scratch', 'RESTART', rst_dto.strftime('MOM.res_Y%Y_D%j_S') + seconds + '.nc']) - self.soca_ana = self.config.resolve(analysis_variables) + self.soca_ana = self.config.resolve(qd.analysis_variables) if mom6_iau: self.mom6_increment(f_rst, ana_path, incr_path) diff --git a/src/swell/tasks/publish_comparisons.py b/src/swell/tasks/publish_comparisons.py index d6663fba4..029e22422 100644 --- a/src/swell/tasks/publish_comparisons.py +++ b/src/swell/tasks/publish_comparisons.py @@ -11,7 +11,7 @@ import shutil from pathlib import Path -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- @@ -27,7 +27,7 @@ class PublishComparisons(taskBase): def execute(self) -> None: # Output path base to copy files to - publish_directory = self.config.resolve(publish_directory, default=None) + publish_directory = self.config.resolve(qd.publish_directory, default=None) # Skip this task if there is no publish directory if publish_directory is None: @@ -41,7 +41,7 @@ def execute(self) -> None: experiment_id = github_run_id / experiment_id # Name the location after the experiment ID - publish_location = Path(self.config.resolve(publish_directory)) / experiment_id + publish_location = Path(self.config.resolve(qd.publish_directory)) / experiment_id self.logger.info(f'Copying comparison test results to {publish_location}') diff --git a/src/swell/tasks/render_jedi_observations.py b/src/swell/tasks/render_jedi_observations.py index 2602886c4..99139e543 100644 --- a/src/swell/tasks/render_jedi_observations.py +++ b/src/swell/tasks/render_jedi_observations.py @@ -11,7 +11,7 @@ import os from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import check_obs @@ -23,26 +23,26 @@ class RenderJediObservations(taskBase): def execute(self) -> None: # List of observations - obs_list = self.config.resolve(observations) + obs_list = self.config.resolve(qd.observations) # Whether to run get_channels for obs - check_for_obs = self.config.resolve(check_for_obs, default=True) + check_for_obs = self.config.resolve(qd.check_for_obs, default=True) # Observing system records paths - observing_system_records_path = self.config.resolve(observing_system_records_path, default=None) + observing_system_records_path = self.config.resolve(qd.observing_system_records_path, default=None) self.jedi_rendering.set_obs_records_path(observing_system_records_path) # Marine models - marine_models = self.config.resolve(marine_models, default=None) + marine_models = self.config.resolve(qd.marine_models, default=None) # Window parameters - window_length = self.config.resolve(window_length) - background_time_offset = self.config.resolve(background_time_offset) + window_length = self.config.resolve(qd.window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) # Window parameters for observations window_begin = self.da_window_params.window_begin(window_length) background_time = self.da_window_params.background_time(background_time_offset) - crtm_coeff_dir = self.config.resolve(crtm_coeff_dir, default=None) + crtm_coeff_dir = self.config.resolve(qd.crtm_coeff_dir, default=None) # Set fields for obs files self.jedi_rendering.add_key('window_begin', window_begin) @@ -52,7 +52,7 @@ def execute(self) -> None: cwd = os.getcwd() - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('experiment_root', 'experiment_root') diff --git a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py index 400ee6151..3d4081af6 100644 --- a/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py +++ b/src/swell/tasks/run_jedi_convert_state_soca2cice_executable.py @@ -11,7 +11,7 @@ import os from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -29,10 +29,10 @@ def execute(self) -> None: # --------------------- jedi_application = 'convert_state_soca2cice' - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) # Compute data assimilation window parameters # -------------------------------------------- @@ -45,8 +45,8 @@ def execute(self) -> None: # Populate jedi interface templates dictionary # -------------------------------------------- - self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(qd.analysis_variables)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) # Background and analysis times # ----------------------------- @@ -57,14 +57,14 @@ def execute(self) -> None: # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') # Geometry # -------- - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) # Jedi configuration file # ----------------------- diff --git a/src/swell/tasks/run_jedi_diffstates.py b/src/swell/tasks/run_jedi_diffstates.py index fb1424388..4dbef0919 100644 --- a/src/swell/tasks/run_jedi_diffstates.py +++ b/src/swell/tasks/run_jedi_diffstates.py @@ -11,7 +11,7 @@ import os from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -31,14 +31,14 @@ def execute(self) -> None: # Parse configuration # ------------------- - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) - npx_proc = self.config.resolve(npx_proc, default=None) - npy_proc = self.config.resolve(npy_proc, default=None) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) + npx_proc = self.config.resolve(qd.npx_proc, default=None) + npy_proc = self.config.resolve(qd.npy_proc, default=None) # Compute data assimilation window parameters - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) local_background_time = self.da_window_params.local_background_time(window_length, window_type) local_background_time_iso = self.da_window_params.local_background_time_iso(window_length, @@ -47,33 +47,33 @@ def execute(self) -> None: window_end_iso = self.da_window_params.window_end_iso(window_length) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) # Populate jedi interface templates dictionary # -------------------------------------------- self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) - self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(qd.analysis_variables)) # Background - self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(qd.horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry - self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(qd.vertical_resolution)) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) # Diffstates - spec_dict = self.config.resolve(diffstates_spec) + spec_dict = self.config.resolve(qd.diffstates_spec) self.jedi_rendering.add_key('diffstates_spec', spec_dict) self.logger.info(f'diffstates = {spec_dict}') # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/run_jedi_eda_executable.py b/src/swell/tasks/run_jedi_eda_executable.py index 5c5c48ae0..e89c5b3f2 100644 --- a/src/swell/tasks/run_jedi_eda_executable.py +++ b/src/swell/tasks/run_jedi_eda_executable.py @@ -12,7 +12,7 @@ import shutil from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable from swell.utilities.yaml_utils import replace_string_value @@ -32,25 +32,25 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) - forecast_length = self.config.resolve(forecast_length, default=window_length) - background_time_offset = self.config.resolve(background_time_offset) - number_of_iterations = self.config.resolve(number_of_iterations) - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) - perhost = self.config.resolve(perhost, default=None) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) + forecast_length = self.config.resolve(qd.forecast_length, default=window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) + number_of_iterations = self.config.resolve(qd.number_of_iterations) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) + perhost = self.config.resolve(qd.perhost, default=None) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) - gsibec_nlats = self.config.resolve(gsibec_nlats, default=None) - gsibec_nlons = self.config.resolve(gsibec_nlons, default=None) - gsibec_configuration = self.config.resolve(gsibec_configuration, default=None) - npx_proc = self.config.resolve(npx_proc, default=None) - npy_proc = self.config.resolve(npy_proc, default=None) - npx = self.config.resolve(npx, default=None) - npy = self.config.resolve(npy, default=None) + gsibec_nlats = self.config.resolve(qd.gsibec_nlats, default=None) + gsibec_nlons = self.config.resolve(qd.gsibec_nlons, default=None) + gsibec_configuration = self.config.resolve(qd.gsibec_configuration, default=None) + npx_proc = self.config.resolve(qd.npx_proc, default=None) + npy_proc = self.config.resolve(qd.npy_proc, default=None) + npx = self.config.resolve(qd.npx, default=None) + npy = self.config.resolve(qd.npy, default=None) # Compute data assimilation window parameters # -------------------------------------------- @@ -62,8 +62,8 @@ def execute(self) -> None: window_begin = self.da_window_params.window_begin(window_length) window_begin_iso = self.da_window_params.window_begin_iso(window_length) window_end_iso = self.da_window_params.window_end_iso(window_length) - obs_pert_amplitude = self.config.resolve(obs_pert_amplitude) - nmember = self.config.resolve(ensemble_num_members) + obs_pert_amplitude = self.config.resolve(qd.obs_pert_amplitude) + nmember = self.config.resolve(qd.ensemble_num_members) imember = self.get_ensemble_imember() # Populate jedi interface templates dictionary @@ -72,38 +72,38 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) self.jedi_rendering.add_key('forecast_length', forecast_length) - self.jedi_rendering.add_key('minimizer', self.config.resolve(minimizer)) + self.jedi_rendering.add_key('minimizer', self.config.resolve(qd.minimizer)) self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) - self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) - self.jedi_rendering.add_key('saber_central_block', self.config.resolve(saber_central_block, default=None)) - self.jedi_rendering.add_key('saber_outer_block', self.config.resolve(saber_outer_block, default=None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(qd.analysis_variables)) + self.jedi_rendering.add_key('saber_central_block', self.config.resolve(qd.saber_central_block, default=None)) + self.jedi_rendering.add_key('saber_outer_block', self.config.resolve(qd.saber_outer_block, default=None)) self.jedi_rendering.add_key('gradient_norm_reduction', - self.config.resolve(gradient_norm_reduction)) - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.config.resolve(qd.gradient_norm_reduction)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(qd.horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) - self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(ensemble_num_members)) + self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(qd.ensemble_num_members)) self.jedi_rendering.add_key('ensemble_imember', imember) # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(qd.vertical_resolution)) self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) self.jedi_rendering.add_key('npx', npx) self.jedi_rendering.add_key('npy', npy) - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Atmosphere background error model @@ -118,7 +118,7 @@ def execute(self) -> None: # Model # ----- if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(qd.background_frequency)) # Jedi configuration file # ----------------------- diff --git a/src/swell/tasks/run_jedi_ensemble_mean_variance.py b/src/swell/tasks/run_jedi_ensemble_mean_variance.py index 2e6ec265d..ad8f6ff80 100644 --- a/src/swell/tasks/run_jedi_ensemble_mean_variance.py +++ b/src/swell/tasks/run_jedi_ensemble_mean_variance.py @@ -11,7 +11,7 @@ import os from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -31,14 +31,14 @@ def execute(self) -> None: # Parse configuration # ------------------- - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) - npx_proc = self.config.resolve(npx_proc, default=None) - npy_proc = self.config.resolve(npy_proc, default=None) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) + npx_proc = self.config.resolve(qd.npx_proc, default=None) + npy_proc = self.config.resolve(qd.npy_proc, default=None) # Compute data assimilation window parameters - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) local_background_time = self.da_window_params.local_background_time(window_length, window_type) local_background_time_iso = self.da_window_params.local_background_time_iso(window_length, @@ -47,7 +47,7 @@ def execute(self) -> None: window_end_iso = self.da_window_params.window_end_iso(window_length) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) # Ensemble # ------------------------ @@ -57,27 +57,27 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) - self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(qd.analysis_variables)) # Background - self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(qd.horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry - self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(qd.vertical_resolution)) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) # Ensemble - self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(ensemble_num_members, default=None)) + self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(qd.ensemble_num_members, default=None)) - self.logger.info(f'ensmeanvariance_spec = {self.config.resolve(ensmeanvariance_spec)}') - meanvar_spec_dict = self.config.resolve(ensmeanvariance_spec) + self.logger.info(f'ensmeanvariance_spec = {self.config.resolve(qd.ensmeanvariance_spec)}') + meanvar_spec_dict = self.config.resolve(qd.ensmeanvariance_spec) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/run_jedi_fgat_executable.py b/src/swell/tasks/run_jedi_fgat_executable.py index 6c6462c2f..e9fa28f93 100644 --- a/src/swell/tasks/run_jedi_fgat_executable.py +++ b/src/swell/tasks/run_jedi_fgat_executable.py @@ -10,7 +10,7 @@ import os from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -30,23 +30,23 @@ def execute(self) -> None: # Parse configuration # ------------------- - marine_models = self.config.resolve(marine_models) - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) - background_time_offset = self.config.resolve(background_time_offset) - number_of_iterations = self.config.resolve(number_of_iterations) - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + marine_models = self.config.resolve(qd.marine_models) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) + number_of_iterations = self.config.resolve(qd.number_of_iterations) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) # Atmosphere specific settings # ---------------------------- # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) - gsibec_nlats = self.config.resolve(gsibec_nlats, default=None) - gsibec_nlons = self.config.resolve(gsibec_nlons, default=None) - npx_proc = self.config.resolve(npx_proc, default=None) - npy_proc = self.config.resolve(npy_proc, default=None) + gsibec_nlats = self.config.resolve(qd.gsibec_nlats, default=None) + gsibec_nlons = self.config.resolve(qd.gsibec_nlons, default=None) + npx_proc = self.config.resolve(qd.npx_proc, default=None) + npy_proc = self.config.resolve(qd.npy_proc, default=None) # Compute data assimilation window parameters # -------------------------------------------- @@ -65,37 +65,37 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) - self.jedi_rendering.add_key('minimizer', self.config.resolve(minimizer)) + self.jedi_rendering.add_key('minimizer', self.config.resolve(qd.minimizer)) self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) - self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(qd.analysis_variables)) self.jedi_rendering.add_key('gradient_norm_reduction', - self.config.resolve(gradient_norm_reduction)) + self.config.resolve(qd.gradient_norm_reduction)) # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(qd.horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(qd.vertical_resolution)) self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Atmosphere background error model # --------------------------------- if npx_proc is not None and npy_proc is not None: - self.jedi_rendering.add_key('gsibec_configuration', self.config.resolve(gsibec_configuration)) + self.jedi_rendering.add_key('gsibec_configuration', self.config.resolve(qd.gsibec_configuration)) self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) self.jedi_rendering.add_key('gsibec_npx_proc', npx_proc) @@ -103,12 +103,12 @@ def execute(self) -> None: # Background frequency is required for FGAT irrespective of the model type # ------------------------------------------------------------------------ - background_frequency = self.config.resolve(background_frequency) + background_frequency = self.config.resolve(qd.background_frequency) self.jedi_rendering.add_key('background_frequency', background_frequency) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py index 5b3cef852..57cc993f1 100644 --- a/src/swell/tasks/run_jedi_hofx_ensemble_executable.py +++ b/src/swell/tasks/run_jedi_hofx_ensemble_executable.py @@ -11,7 +11,7 @@ import os from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable from swell.tasks.run_jedi_hofx_executable import RunJediHofxExecutable @@ -36,11 +36,11 @@ def execute(self) -> None: # Parse configuration ... despite same block in RunJediHofxExecutable # ------------------------------------------------------------------- - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) - background_time_offset = self.config.resolve(background_time_offset) - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) # Compute data assimilation window parameters background_time = self.da_window_params.background_time(background_time_offset) @@ -53,9 +53,9 @@ def execute(self) -> None: # Ensemble hofx components # ------------------------ - ensemble_hofx_packets = self.config.resolve(ensemble_hofx_packets) - ensemble_hofx_strategy = self.config.resolve(ensemble_hofx_strategy) - ensemble_num_members = self.config.resolve(ensemble_num_members) + ensemble_hofx_packets = self.config.resolve(qd.ensemble_hofx_packets) + ensemble_hofx_strategy = self.config.resolve(qd.ensemble_hofx_strategy) + ensemble_num_members = self.config.resolve(qd.ensemble_num_members) # Force packets of equal size (i.e., members handled) # --------------------------------------------------- @@ -84,24 +84,24 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_end_iso', window_end_iso) # Background - self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(qd.horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry - self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) - self.jedi_rendering.add_key('npx_proc', self.config.resolve(npx_proc, default=None)) - self.jedi_rendering.add_key('npy_proc', self.config.resolve(npy_proc, default=None)) - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(qd.vertical_resolution)) + self.jedi_rendering.add_key('npx_proc', self.config.resolve(qd.npx_proc, default=None)) + self.jedi_rendering.add_key('npy_proc', self.config.resolve(qd.npy_proc, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) # Observations self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Model if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(qd.background_frequency)) # Populate remaining entries of jedi interface templates dictionary # ----------------------------------------------------------------- @@ -110,7 +110,7 @@ def execute(self) -> None: # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 0fd2f16d0..1f5242202 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -13,7 +13,7 @@ from ruamel.yaml import YAML from typing import Optional -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.netcdf_files import combine_files_without_groups from swell.utilities.run_jedi_executables import run_executable @@ -34,17 +34,17 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Parse configuration # ------------------- - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) - forecast_length = self.config.resolve(forecast_length, default=window_length) - background_time_offset = self.config.resolve(background_time_offset) - observations = self.config.resolve(observations) - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) - save_geovals = self.config.resolve(save_geovals, default=False) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) + forecast_length = self.config.resolve(qd.forecast_length, default=window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) + observations = self.config.resolve(qd.observations) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) + save_geovals = self.config.resolve(qd.save_geovals, default=False) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) # Compute data assimilation window parameters # -------------------------------------------- @@ -66,28 +66,28 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(qd.horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) - self.jedi_rendering.add_key('npx_proc', self.config.resolve(npx_proc, default=None)) - self.jedi_rendering.add_key('npy_proc', self.config.resolve(npy_proc, default=None)) - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) - self.jedi_rendering.add_key('npx', self.config.resolve(npx, default=None)) - self.jedi_rendering.add_key('npy', self.config.resolve(npy, default=None)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(qd.vertical_resolution)) + self.jedi_rendering.add_key('npx_proc', self.config.resolve(qd.npx_proc, default=None)) + self.jedi_rendering.add_key('npy_proc', self.config.resolve(qd.npy_proc, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) + self.jedi_rendering.add_key('npx', self.config.resolve(qd.npx, default=None)) + self.jedi_rendering.add_key('npy', self.config.resolve(qd.npy, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') @@ -95,7 +95,7 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Model # ----- if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(qd.background_frequency)) # Get the JEDI interface metadata # ------------------------------- @@ -133,7 +133,7 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: 'time interpolation': 'linear' } self.jedi_rendering.add_key('forecast_length', - self.config.resolve(forecast_length, default=window_length)) + self.config.resolve(qd.forecast_length, default=window_length)) # Update config filters to save the GeoVaLs from the model interface. # Add GOMsaver to either obs filters OR obs prior filters, if neither @@ -271,7 +271,7 @@ def append_gomsaver( # Add mem to the filename if it is not None mem_str = f'_mem{mem}' if mem is not None else '' - if not self.config.resolve(mock_experiment, default=False): + if not self.config.resolve(qd.mock_experiment, default=False): cycle_dir = self.cycle_dir() else: cycle_dir = 'cycle_dir' diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 7211e5d19..34a7c59d9 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -12,7 +12,7 @@ from importlib import resources from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable from swell.utilities.yaml_utils import replace_key @@ -33,18 +33,18 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) - background_time_offset = self.config.resolve(background_time_offset) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) - ensmean_only = self.config.resolve(ensmean_only) - ensmeanvariance_only = self.config.resolve(ensmeanvariance_only) - perhost = self.config.resolve(perhost, default=None) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) + ensmean_only = self.config.resolve(qd.ensmean_only) + ensmeanvariance_only = self.config.resolve(qd.ensmeanvariance_only) + perhost = self.config.resolve(qd.perhost, default=None) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) # Compute data assimilation window parameters background_time = self.da_window_params.background_time(background_time_offset) @@ -61,74 +61,74 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_begin_iso', window_begin_iso) self.jedi_rendering.add_key('window_length', window_length) self.jedi_rendering.add_key('window_end_iso', window_end_iso) - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) - self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(qd.analysis_variables)) # Background - self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(qd.horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) - self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(ensemble_num_members)) + self.jedi_rendering.add_key('ensemble_num_members', self.config.resolve(qd.ensemble_num_members)) # Geometry - self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) - self.jedi_rendering.add_key('npx_proc', self.config.resolve(npx_proc, default=None)) - self.jedi_rendering.add_key('npy_proc', self.config.resolve(npy_proc, default=None)) - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(qd.vertical_resolution)) + self.jedi_rendering.add_key('npx_proc', self.config.resolve(qd.npx_proc, default=None)) + self.jedi_rendering.add_key('npy_proc', self.config.resolve(qd.npy_proc, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) # Observations self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Ensemble hofx components - self.jedi_rendering.add_key('ensemble_hofx_strategy', self.config.resolve(ensemble_hofx_strategy)) - self.jedi_rendering.add_key('ensemble_hofx_packets', self.config.resolve(ensemble_hofx_packets)) + self.jedi_rendering.add_key('ensemble_hofx_strategy', self.config.resolve(qd.ensemble_hofx_strategy)) + self.jedi_rendering.add_key('ensemble_hofx_packets', self.config.resolve(qd.ensemble_hofx_packets)) # Ensemble Localizations # ------------------------------ if self.get_model() == 'geos_atmosphere': self.jedi_rendering.add_key('vertical_localization_method', - self.config.resolve(vertical_localization_method)) + self.config.resolve(qd.vertical_localization_method)) self.jedi_rendering.add_key('vertical_localization_apply_log_transform', - self.config.resolve(vertical_localization_apply_log_transform)) + self.config.resolve(qd.vertical_localization_apply_log_transform)) self.jedi_rendering.add_key('vertical_localization_lengthscale', - self.config.resolve(vertical_localization_lengthscale)) + self.config.resolve(qd.vertical_localization_lengthscale)) self.jedi_rendering.add_key('vertical_localization_ioda_vertical_coord', - self.config.resolve(vertical_localization_ioda_vertical_coord)) + self.config.resolve(qd.vertical_localization_ioda_vertical_coord)) self.jedi_rendering.add_key( 'vertical_localization_ioda_vertical_coord_group', - self.config.resolve(vertical_localization_ioda_vertical_coord_group)) + self.config.resolve(qd.vertical_localization_ioda_vertical_coord_group)) self.jedi_rendering.add_key('vertical_localization_function', - self.config.resolve(vertical_localization_function)) + self.config.resolve(qd.vertical_localization_function)) # Driver - self.jedi_rendering.add_key('local_ensemble_solver', self.config.resolve(local_ensemble_solver)) + self.jedi_rendering.add_key('local_ensemble_solver', self.config.resolve(qd.local_ensemble_solver)) self.jedi_rendering.add_key('local_ensemble_inflation_rtps', - self.config.resolve(local_ensemble_inflation_rtps)) + self.config.resolve(qd.local_ensemble_inflation_rtps)) self.jedi_rendering.add_key('local_ensemble_inflation_rtpp', - self.config.resolve(local_ensemble_inflation_rtpp)) + self.config.resolve(qd.local_ensemble_inflation_rtpp)) self.jedi_rendering.add_key('local_ensemble_inflation_mult', - self.config.resolve(local_ensemble_inflation_mult)) + self.config.resolve(qd.local_ensemble_inflation_mult)) self.jedi_rendering.add_key('local_ensemble_save_posterior_mean', - self.config.resolve(local_ensemble_save_posterior_mean)) + self.config.resolve(qd.local_ensemble_save_posterior_mean)) self.jedi_rendering.add_key('local_ensemble_save_posterior_ensemble', - self.config.resolve(local_ensemble_save_posterior_ensemble)) + self.config.resolve(qd.local_ensemble_save_posterior_ensemble)) self.jedi_rendering.add_key('local_ensemble_save_posterior_mean_increment', - self.config.resolve(local_ensemble_save_posterior_mean_increment)) + self.config.resolve(qd.local_ensemble_save_posterior_mean_increment)) self.jedi_rendering.add_key('local_ensemble_save_posterior_ensemble_increments', - self.config.resolve(local_ensemble_save_posterior_ensemble_increments)) + self.config.resolve(qd.local_ensemble_save_posterior_ensemble_increments)) self.jedi_rendering.add_key('ensmean_only', - self.config.resolve(ensmean_only)) + self.config.resolve(qd.ensmean_only)) self.jedi_rendering.add_key('ensmeanvariance_only', - self.config.resolve(ensmeanvariance_only)) + self.config.resolve(qd.ensmeanvariance_only)) self.jedi_rendering.add_key('local_ensemble_use_linear_observer', - self.config.resolve(local_ensemble_use_linear_observer)) - self.jedi_rendering.add_key('skip_ensemble_hofx', self.config.resolve(skip_ensemble_hofx)) + self.config.resolve(qd.local_ensemble_use_linear_observer)) + self.jedi_rendering.add_key('skip_ensemble_hofx', self.config.resolve(qd.skip_ensemble_hofx)) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') @@ -136,8 +136,8 @@ def execute(self) -> None: # Prevent both 'local_ensemble_save_posterior_mean' and # 'local_ensemble_save_posterior_ensemble' from being true # -------------------------------------------------------- - if self.config.resolve(local_ensemble_save_posterior_mean) and \ - self.config.resolve(local_ensemble_save_posterior_ensemble): + if self.config.resolve(qd.local_ensemble_save_posterior_mean) and \ + self.config.resolve(qd.local_ensemble_save_posterior_ensemble): raise ValueError("'local_ensemble_save_posterior_mean' and\ 'local_ensemble_save_posterior_ensemble' cannot be both true!") @@ -158,15 +158,15 @@ def execute(self) -> None: # Assemble localizations # ---------------------- # # Vertical localizations have bug(s) - Commented out for now... - # vertLoc = {'localization method': self.config.resolve(vertical_localization_method), + # vertLoc = {'localization method': self.config.resolve(qd.vertical_localization_method), # 'apply log transformation': - # self.config.resolve(vertical_localization_apply_log_transform), - # 'vertical lengthscale': self.config.resolve(vertical_localization_lengthscale), + # self.config.resolve(qd.vertical_localization_apply_log_transform), + # 'vertical lengthscale': self.config.resolve(qd.vertical_localization_lengthscale), # 'ioda vertical coordinate': - # self.config.resolve(vertical_localization_ioda_vertical_coord), + # self.config.resolve(qd.vertical_localization_ioda_vertical_coord), # 'ioda vertical coordinate group': - # self.config.resolve(vertical_localization_ioda_vertical_coord_group), - # 'localization function': self.config.resolve(vertical_localization_function)} + # self.config.resolve(qd.vertical_localization_ioda_vertical_coord_group), + # 'localization function': self.config.resolve(qd.vertical_localization_function)} # localizations = [horizLoc, vertLoc] if len(vertLoc) != 0 else [horizLoc] # Include ensemble localizations and halo types with each observation diff --git a/src/swell/tasks/run_jedi_obsfilters_executable.py b/src/swell/tasks/run_jedi_obsfilters_executable.py index 27772e0ee..021df2f90 100644 --- a/src/swell/tasks/run_jedi_obsfilters_executable.py +++ b/src/swell/tasks/run_jedi_obsfilters_executable.py @@ -12,7 +12,7 @@ from ruamel.yaml import YAML from typing import Optional import random -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -36,15 +36,15 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Parse configuration # ------------------- - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) - background_time_offset = self.config.resolve(background_time_offset) - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) - obs_thinning_rej_fraction = self.config.resolve(obs_thinning_rej_fraction) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) + obs_thinning_rej_fraction = self.config.resolve(qd.obs_thinning_rej_fraction) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) # Compute data assimilation window parameters # -------------------------------------------- @@ -65,26 +65,26 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(qd.horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) - self.jedi_rendering.add_key('npx_proc', self.config.resolve(npx_proc, default=None)) - self.jedi_rendering.add_key('npy_proc', self.config.resolve(npy_proc, default=None)) - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(qd.vertical_resolution)) + self.jedi_rendering.add_key('npx_proc', self.config.resolve(qd.npx_proc, default=None)) + self.jedi_rendering.add_key('npy_proc', self.config.resolve(qd.npy_proc, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') @@ -92,7 +92,7 @@ def execute(self, ensemble_members: Optional[list] = None) -> None: # Model # ----- if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(qd.background_frequency)) # Get the JEDI interface metadata # ------------------------------- diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 62842e037..ed001a03f 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -12,7 +12,7 @@ import os from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.dictionary import update_dict from swell.utilities.run_jedi_executables import run_executable @@ -33,14 +33,14 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_length = self.config.resolve(window_length) - bkg_time_offset = self.config.resolve(background_time_offset) - observations = self.config.resolve(observations) - single_observations = self.config.resolve(single_observations) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) + window_length = self.config.resolve(qd.window_length) + bkg_time_offset = self.config.resolve(qd.background_time_offset) + observations = self.config.resolve(qd.observations) + single_observations = self.config.resolve(qd.single_observations) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) # Compute data assimilation window parameters window_begin = self.da_window_params.window_begin(window_length) @@ -55,12 +55,12 @@ def execute(self) -> None: # Observations self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index a2d90443b..6aa92f102 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -10,7 +10,7 @@ import os from ruamel.yaml import YAML -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.run_jedi_executables import run_executable @@ -30,25 +30,25 @@ def execute(self) -> None: # Parse configuration # ------------------- - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) - forecast_length = self.config.resolve(forecast_length, default=window_length) - background_time_offset = self.config.resolve(background_time_offset) - number_of_iterations = self.config.resolve(number_of_iterations) - jedi_forecast_model = self.config.resolve(jedi_forecast_model, default=None) - generate_yaml_and_exit = self.config.resolve(generate_yaml_and_exit, default=False) - perhost = self.config.resolve(perhost, default=None) + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) + forecast_length = self.config.resolve(qd.forecast_length, default=window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) + number_of_iterations = self.config.resolve(qd.number_of_iterations) + jedi_forecast_model = self.config.resolve(qd.jedi_forecast_model, default=None) + generate_yaml_and_exit = self.config.resolve(qd.generate_yaml_and_exit, default=False) + perhost = self.config.resolve(qd.perhost, default=None) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) - gsibec_nlats = self.config.resolve(gsibec_nlats, default=None) - gsibec_nlons = self.config.resolve(gsibec_nlons, default=None) - gsibec_configuration = self.config.resolve(gsibec_configuration, default=None) - npx_proc = self.config.resolve(npx_proc, default=None) - npy_proc = self.config.resolve(npy_proc, default=None) - npx = self.config.resolve(npx, default=None) - npy = self.config.resolve(npy, default=None) + gsibec_nlats = self.config.resolve(qd.gsibec_nlats, default=None) + gsibec_nlons = self.config.resolve(qd.gsibec_nlons, default=None) + gsibec_configuration = self.config.resolve(qd.gsibec_configuration, default=None) + npx_proc = self.config.resolve(qd.npx_proc, default=None) + npy_proc = self.config.resolve(qd.npy_proc, default=None) + npx = self.config.resolve(qd.npx, default=None) + npy = self.config.resolve(qd.npy, default=None) # Compute data assimilation window parameters # -------------------------------------------- @@ -67,41 +67,41 @@ def execute(self) -> None: self.jedi_rendering.add_key('window_end_iso', window_end_iso) self.jedi_rendering.add_key('window_length', window_length) self.jedi_rendering.add_key('forecast_length', forecast_length) - self.jedi_rendering.add_key('minimizer', self.config.resolve(minimizer)) + self.jedi_rendering.add_key('minimizer', self.config.resolve(qd.minimizer)) self.jedi_rendering.add_key('number_of_iterations', number_of_iterations[0]) - self.jedi_rendering.add_key('analysis_variables', self.config.resolve(analysis_variables)) - self.jedi_rendering.add_key('saber_central_block', self.config.resolve(saber_central_block, default=None)) - self.jedi_rendering.add_key('saber_outer_block', self.config.resolve(saber_outer_block, default=None)) + self.jedi_rendering.add_key('analysis_variables', self.config.resolve(qd.analysis_variables)) + self.jedi_rendering.add_key('saber_central_block', self.config.resolve(qd.saber_central_block, default=None)) + self.jedi_rendering.add_key('saber_outer_block', self.config.resolve(qd.saber_outer_block, default=None)) self.jedi_rendering.add_key('gradient_norm_reduction', - self.config.resolve(gradient_norm_reduction)) - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.config.resolve(qd.gradient_norm_reduction)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) # Background # ---------- - self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(horizontal_resolution)) + self.jedi_rendering.add_key('horizontal_resolution', self.config.resolve(qd.horizontal_resolution)) self.jedi_rendering.add_key('local_background_time', local_background_time) self.jedi_rendering.add_key('local_background_time_iso', local_background_time_iso) # Geometry # -------- - self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(vertical_resolution)) + self.jedi_rendering.add_key('vertical_resolution', self.config.resolve(qd.vertical_resolution)) self.jedi_rendering.add_key('gsibec_nlats', gsibec_nlats) self.jedi_rendering.add_key('gsibec_nlons', gsibec_nlons) self.jedi_rendering.add_key('npx_proc', npx_proc) self.jedi_rendering.add_key('npy_proc', npy_proc) self.jedi_rendering.add_key('npx', npx) self.jedi_rendering.add_key('npy', npy) - self.jedi_rendering.add_key('total_processors', self.config.resolve(total_processors, default=None)) + self.jedi_rendering.add_key('total_processors', self.config.resolve(qd.total_processors, default=None)) # Observations # ------------ self.jedi_rendering.add_key('background_time', background_time) - self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(crtm_coeff_dir, default=None)) + self.jedi_rendering.add_key('crtm_coeff_dir', self.config.resolve(qd.crtm_coeff_dir, default=None)) self.jedi_rendering.add_key('window_begin', window_begin) # Add placeholder names if mock experiment # ---------------------------------------- - if self.config.resolve(mock_experiment, default=False): + if self.config.resolve(qd.mock_experiment, default=False): self.jedi_rendering.add_key('experiment_root', 'experiment_root') self.jedi_rendering.add_key('experiment_id', 'experiment_id') self.jedi_rendering.add_key('cycle_dir', 'cycle_dir') @@ -118,7 +118,7 @@ def execute(self) -> None: # Model # ----- if window_type == '4D': - self.jedi_rendering.add_key('background_frequency', self.config.resolve(background_frequency)) + self.jedi_rendering.add_key('background_frequency', self.config.resolve(qd.background_frequency)) # Jedi configuration file # ----------------------- diff --git a/src/swell/tasks/save_background.py b/src/swell/tasks/save_background.py index 3a10acc7c..1a2d8aafb 100644 --- a/src/swell/tasks/save_background.py +++ b/src/swell/tasks/save_background.py @@ -12,7 +12,7 @@ import os from r2d2 import store -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.datetime_util import datetime_formats from swell.utilities.r2d2 import load_r2d2_credentials @@ -54,7 +54,7 @@ def execute(self) -> None: # Load R2D2 credentials load_r2d2_credentials(self.logger, self.platform()) - dry_run = self.config.resolve(dry_run, default=True) + dry_run = self.config.resolve(qd.dry_run, default=True) if dry_run: self.logger.info('DRY RUN MODE - No files will be stored') @@ -71,10 +71,10 @@ def execute(self) -> None: ) model = self.get_model() - source_template = self.config.resolve(background_source_path) - experiment = self.config.resolve(background_experiment, default='geos_cf_oper') - resolution = self.config.resolve(horizontal_resolution, default='c360') - store_as_symlink = self.config.resolve(store_as_symlink, default=True) + source_template = self.config.resolve(qd.background_source_path) + experiment = self.config.resolve(qd.background_experiment, default='geos_cf_oper') + resolution = self.config.resolve(qd.horizontal_resolution, default='c360') + store_as_symlink = self.config.resolve(qd.store_as_symlink, default=True) stored = 0 skipped = 0 diff --git a/src/swell/tasks/save_forecast_cf.py b/src/swell/tasks/save_forecast_cf.py index e18023856..fbd7172de 100644 --- a/src/swell/tasks/save_forecast_cf.py +++ b/src/swell/tasks/save_forecast_cf.py @@ -14,7 +14,7 @@ from swell.configuration.jedi.interfaces.geos_cf.model.r2d2 import forecast_filename, r2d2 -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials @@ -40,10 +40,10 @@ def execute(self) -> None: # Parse config # ------------ - window_length = self.config.resolve(window_length) - forecast_length = self.config.resolve(forecast_length) - forecast_output_frequency = self.config.resolve(forecast_output_frequency) - expid = self.config.resolve(r2d2_experiment_id) + window_length = self.config.resolve(qd.window_length) + forecast_length = self.config.resolve(qd.forecast_length) + forecast_output_frequency = self.config.resolve(qd.forecast_output_frequency) + expid = self.config.resolve(qd.r2d2_experiment_id) fc_store = r2d2(self.jedi_rendering.__template_dict__)['store']['fc'][0] cycle_dir = self.cycle_dir() @@ -76,7 +76,7 @@ def execute(self) -> None: item='forecast', step=step, experiment=expid, - resolution=self.config.resolve(horizontal_resolution), + resolution=self.config.resolve(qd.horizontal_resolution), date=forecast_start.strftime('%Y%m%dT%H%M%S%z'), source_file=source_file, file_extension='nc', diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 0096ff4b6..205599d1f 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -8,7 +8,7 @@ # -------------------------------------------------------------------------------------------------- import r2d2 -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials from swell.utilities.run_jedi_executables import check_obs @@ -29,21 +29,21 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.resolve(r2d2_server, default=None), + r2d2_server=self.config.resolve(qd.r2d2_server, default=None), ) - r2d2_datastore = self.config.resolve(r2d2_datastore, default=None) + r2d2_datastore = self.config.resolve(qd.r2d2_datastore, default=None) # Parse config # ------------ - background_time_offset = self.config.resolve(background_time_offset) - crtm_coeff_dir = self.config.resolve(crtm_coeff_dir, default=None) - observations = self.config.resolve(observations) - window_length = self.config.resolve(window_length) + background_time_offset = self.config.resolve(qd.background_time_offset) + crtm_coeff_dir = self.config.resolve(qd.crtm_coeff_dir, default=None) + observations = self.config.resolve(qd.observations) + window_length = self.config.resolve(qd.window_length) # Set the observing system records path - self.jedi_rendering.set_obs_records_path(self.config.resolve(observing_system_records_path, default=None)) - self.jedi_rendering.add_key('marine_models', self.config.resolve(marine_models, default=None)) + self.jedi_rendering.set_obs_records_path(self.config.resolve(qd.observing_system_records_path, default=None)) + self.jedi_rendering.add_key('marine_models', self.config.resolve(qd.marine_models, default=None)) # Get window beginning window_begin = self.da_window_params.window_begin(window_length) # dto @@ -87,7 +87,7 @@ def execute(self) -> None: try: store_kwargs = dict( item='feedback', - experiment=self.config.resolve(r2d2_experiment_id), + experiment=self.config.resolve(qd.r2d2_experiment_id), observation_type=name, file_extension=obs_path_file.split('.')[-1], window_length='PT6H', diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index 680e2a40a..d46116a56 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -7,7 +7,7 @@ # -------------------------------------------------------------------------------------------------- -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/save_restart_cf.py b/src/swell/tasks/save_restart_cf.py index 2d3080d51..947dae5c3 100644 --- a/src/swell/tasks/save_restart_cf.py +++ b/src/swell/tasks/save_restart_cf.py @@ -11,7 +11,7 @@ import os from r2d2 import store -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.r2d2 import load_r2d2_credentials @@ -36,9 +36,9 @@ def execute(self): # Parse config # ------------ - window_length = self.config.resolve(window_length) - expid = self.config.resolve(r2d2_experiment_id) - horizontal_resolution = self.config.resolve(horizontal_resolution) + window_length = self.config.resolve(qd.window_length) + expid = self.config.resolve(qd.r2d2_experiment_id) + horizontal_resolution = self.config.resolve(qd.horizontal_resolution) cycle_dir = self.cycle_dir() scratch_dir = os.path.join(cycle_dir, 'scratch') @@ -52,8 +52,8 @@ def execute(self): next_window_begin = window_begin + isodate.parse_duration(window_length) checkpoint_time_str = next_window_begin.strftime('%Y%m%d_%H%Mz') - rst_file_types = self.config.resolve(rst_file_types) - rst_store_interval = self.config.resolve(rst_store_interval, default=None) + rst_file_types = self.config.resolve(qd.rst_file_types) + rst_store_interval = self.config.resolve(qd.rst_store_interval, default=None) # Determine whether to store as a symlink for this cycle # -------------------------------------------------------- diff --git a/src/swell/tasks/stage_jedi.py b/src/swell/tasks/stage_jedi.py index 3c47bdafc..399683b05 100644 --- a/src/swell/tasks/stage_jedi.py +++ b/src/swell/tasks/stage_jedi.py @@ -11,7 +11,7 @@ import os from swell.swell_path import get_swell_path -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.filehandler import get_file_handler from swell.utilities.exceptions import SwellError @@ -33,10 +33,10 @@ def execute(self) -> None: """ # Extract potential template variables from config - horizontal_resolution = self.config.resolve(horizontal_resolution) + horizontal_resolution = self.config.resolve(qd.horizontal_resolution) - swell_static_files_user = self.config.resolve(swell_static_files_user, default=None) - swell_static_files = self.config.resolve(swell_static_files) + swell_static_files_user = self.config.resolve(qd.swell_static_files_user, default=None) + swell_static_files = self.config.resolve(qd.swell_static_files) # Use static_files_user if present in config and contains files # ------------------------------------------------------------- @@ -46,13 +46,13 @@ def execute(self) -> None: self.logger.info(f'Using swell static files in {swell_static_files_user}') swell_static_files = swell_static_files_user - vertical_resolution = self.config.resolve(vertical_resolution) - npx_proc = self.config.resolve(npx_proc, default=None) - npy_proc = self.config.resolve(npy_proc, default=None) - gsibec_configuration = self.config.resolve(gsibec_configuration, default=None) - gsibec_nlats = self.config.resolve(gsibec_nlats, default=None) - gsibec_nlons = self.config.resolve(gsibec_nlons, default=None) - saber_central_block = self.config.resolve(saber_central_block, default=None) + vertical_resolution = self.config.resolve(qd.vertical_resolution) + npx_proc = self.config.resolve(qd.npx_proc, default=None) + npy_proc = self.config.resolve(qd.npy_proc, default=None) + gsibec_configuration = self.config.resolve(qd.gsibec_configuration, default=None) + gsibec_nlats = self.config.resolve(qd.gsibec_nlats, default=None) + gsibec_nlons = self.config.resolve(qd.gsibec_nlons, default=None) + saber_central_block = self.config.resolve(qd.saber_central_block, default=None) # Add jedi interface template keys self.jedi_rendering.add_key('horizontal_resolution', horizontal_resolution) From e7cb8acd3d71163ea7a7f4cf70684081f29e7d69 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 26 Aug 2026 11:25:09 -0400 Subject: [PATCH 11/15] add to geos-marine tasks --- .../geos_marine/get_background_geos_marine.py | 16 ++++++++-------- .../geos_marine/get_ensemble_geos_marine.py | 10 +++++----- .../run_compress_forecast_geos_marine.py | 12 ++++++------ .../geos_marine/save_forecast_geos_marine.py | 12 ++++++------ 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/src/swell/tasks/geos_marine/get_background_geos_marine.py b/src/swell/tasks/geos_marine/get_background_geos_marine.py index fb02f5609..9b9fcfde8 100644 --- a/src/swell/tasks/geos_marine/get_background_geos_marine.py +++ b/src/swell/tasks/geos_marine/get_background_geos_marine.py @@ -13,7 +13,7 @@ import r2d2 from swell.tasks.base.task_base import taskBase -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.utilities.r2d2 import load_r2d2_credentials # -------------------------------------------------------------------------------------------------- @@ -55,15 +55,15 @@ def execute(self) -> None: load_r2d2_credentials( self.logger, self.platform(), - r2d2_server=self.config.resolve(r2d2_server, default=None) + r2d2_server=self.config.resolve(qd.r2d2_server, default=None) ) - r2d2_datastore = self.config.resolve(r2d2_datastore, default=None) - marine_models = self.config.resolve(marine_models, default=None) or [] - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) - horizontal_resolution = self.config.resolve(horizontal_resolution) - background_experiment = self.config.resolve(background_experiment) + r2d2_datastore = self.config.resolve(qd.r2d2_datastore, default=None) + marine_models = self.config.resolve(qd.marine_models, default=None) or [] + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) + horizontal_resolution = self.config.resolve(qd.horizontal_resolution) + background_experiment = self.config.resolve(qd.background_experiment) local_background_time, local_background_time_dto = \ self.da_window_params.local_background_time(window_length, window_type, dto=True) diff --git a/src/swell/tasks/geos_marine/get_ensemble_geos_marine.py b/src/swell/tasks/geos_marine/get_ensemble_geos_marine.py index ad99bafc3..a4e1fb6fd 100644 --- a/src/swell/tasks/geos_marine/get_ensemble_geos_marine.py +++ b/src/swell/tasks/geos_marine/get_ensemble_geos_marine.py @@ -11,7 +11,7 @@ import tarfile from swell.tasks.base.task_base import taskBase -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.utilities.datetime_util import datetime_formats # -------------------------------------------------------------------------------------------------- @@ -36,13 +36,13 @@ def execute(self) -> None: # Parse configuration # ------------------- - horizontal_resolution = self.config.resolve(horizontal_resolution) - vertical_resolution = self.config.resolve(vertical_resolution) - background_experiment = self.config.resolve(background_experiment) + horizontal_resolution = self.config.resolve(qd.horizontal_resolution) + vertical_resolution = self.config.resolve(qd.vertical_resolution) + background_experiment = self.config.resolve(qd.background_experiment) # Get the path and pattern for the ensemble members # ------------------------------------------------- - ensemble_path = self.config.resolve(path_to_ensemble) + ensemble_path = self.config.resolve(qd.path_to_ensemble) # For 3D window, analysis time is the cycle time # ------------------------------------------------- diff --git a/src/swell/tasks/geos_marine/run_compress_forecast_geos_marine.py b/src/swell/tasks/geos_marine/run_compress_forecast_geos_marine.py index 95921953c..0642bdf08 100644 --- a/src/swell/tasks/geos_marine/run_compress_forecast_geos_marine.py +++ b/src/swell/tasks/geos_marine/run_compress_forecast_geos_marine.py @@ -13,8 +13,8 @@ import tarfile from swell.tasks.base.task_base import taskBase -from swell.configuration.question_defaults import * from swell.utilities.compress import compress_file +import swell.configuration.question_defaults as qd # -------------------------------------------------------------------------------------------------- @@ -53,16 +53,16 @@ def execute(self) -> None: state files into a tar.gz archive. """ - marine_models = self.config.resolve(marine_models, default=None) or [] - window_type = self.config.resolve(window_type) - window_length = self.config.resolve(window_length) + marine_models = self.config.resolve(qd.marine_models, default=None) or [] + window_type = self.config.resolve(qd.window_type) + window_length = self.config.resolve(qd.window_length) window_begin_iso = self.da_window_params.window_begin_iso(window_length) - horizontal_resolution = self.config.resolve(horizontal_resolution) + horizontal_resolution = self.config.resolve(qd.horizontal_resolution) is_4d = window_type == '4D' or 'fgat' in self.suite_name() if is_4d: - background_frequency = self.config.resolve(background_frequency) + background_frequency = self.config.resolve(qd.background_frequency) (local_background_time, local_background_time_dto) = \ self.da_window_params.local_background_time(window_length, window_type, dto=True) diff --git a/src/swell/tasks/geos_marine/save_forecast_geos_marine.py b/src/swell/tasks/geos_marine/save_forecast_geos_marine.py index f49146a1c..b483e1efb 100644 --- a/src/swell/tasks/geos_marine/save_forecast_geos_marine.py +++ b/src/swell/tasks/geos_marine/save_forecast_geos_marine.py @@ -12,7 +12,7 @@ from r2d2 import store from swell.tasks.base.task_base import taskBase -from swell.configuration.question_defaults import * +import swell.configuration.question_defaults as qd from swell.utilities.datetime_util import datetime_formats from swell.utilities.r2d2 import load_r2d2_credentials @@ -60,10 +60,10 @@ def execute(self) -> None: window type (and/or suite type) and length. """ - self.marine_models = self.config.resolve(marine_models, default=None) or [] - window_type = self.config.resolve(window_type) - self.window_length = self.config.resolve(window_length) - self.horizontal_resolution = self.config.resolve(horizontal_resolution) + self.marine_models = self.config.resolve(qd.marine_models, default=None) or [] + window_type = self.config.resolve(qd.window_type) + self.window_length = self.config.resolve(qd.window_length) + self.horizontal_resolution = self.config.resolve(qd.horizontal_resolution) load_r2d2_credentials(self.logger, self.platform()) @@ -102,7 +102,7 @@ def _store_forecast(self, model_name: str, file_type: str) -> None: store( item='forecast', model=model_name, - experiment=self.config.resolve(r2d2_experiment_id), + experiment=self.config.resolve(qd.r2d2_experiment_id), resolution=self.horizontal_resolution, date=self.local_background_time_dto.strftime('%Y-%m-%d %H:%M:%S'), source_file=archive_path, From a228a4d5e3f42115fe5acb70cc1a0a4ea15f6a84 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 26 Aug 2026 11:35:51 -0400 Subject: [PATCH 12/15] Fix config --- src/swell/utilities/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index dea439908..715f5d279 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -117,7 +117,7 @@ def __init__(self, input_file: str, logger: Logger, task_name: str, model: str) # Add suite questions if they aren't already set for question in suite_questions: - if question not in suite_questions: + if question not in self.question_list: self.question_list.append(question) # Find the questions associated with the task @@ -130,7 +130,7 @@ def resolve(self, question: SwellQuestion, default='LrZRExPGcQ'): question_obj = question() name = question_obj.question_name - if name in self.question_list: + if name in self.question_list and name in self.experiment_dict: default = self.experiment_dict[name] elif name in self.experiment_dict: raise KeyError(f'Value {name} is present in config but this task has not been assigned' From 26cfe8f37dfe93d234792d1f014d6ab5b118c7bd Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 26 Aug 2026 11:47:32 -0400 Subject: [PATCH 13/15] Add descriptions for questions and docstrings --- src/swell/configuration/question_defaults.py | 38 ++++++++++++++++++++ src/swell/utilities/config.py | 7 ++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py index 1776f8028..a69df5e95 100644 --- a/src/swell/configuration/question_defaults.py +++ b/src/swell/configuration/question_defaults.py @@ -21,6 +21,7 @@ @dataclass class comparison_experiment_paths(SuiteQuestion): + '''Paths to two `experiment.yaml` files to be compared by the suite''' default_value: list = mutable_field([]) prompt: str = "Provide paths to two experiments to run comparison tests on." data_type: DType = DType.LIST @@ -29,6 +30,7 @@ class comparison_experiment_paths(SuiteQuestion): @dataclass class cycle_times(SuiteQuestion): + '''Cycle times that cylc will execute tasks for between the start and end cycle points''' default_value: str = "defer_to_model" options: str = "defer_to_model" models: List[str] = mutable_field([ @@ -41,6 +43,10 @@ class cycle_times(SuiteQuestion): @dataclass class cycling_varbc(SuiteQuestion): + ''' + Cycling Variational Bias Correction - controls whether previous + cycle point will be used as forecast. + ''' default_value: str = "defer_to_model" models: List[str] = mutable_field([ "geos_atmosphere" @@ -52,6 +58,11 @@ class cycling_varbc(SuiteQuestion): @dataclass class download_convert_pipeline(SuiteQuestion): + ''' + Boolean option in `flow.cylc` deciding whether or not to run DownloadObs + and ConvertObsToIoda tasks. + ''' + default_value: bool = False prompt: str = ("Run the DownloadObs and ConvertObsToIoda tasks?" "(DownloadObs -> ConvertObsToIoda) -> IngestObs to R2D2") @@ -61,6 +72,10 @@ class download_convert_pipeline(SuiteQuestion): @dataclass class ensemble_hofx_packets(SuiteQuestion): + ''' + Number of ensemble packets to run for HofX experiment + ''' + default_value: str = "defer_to_model" models: List[str] = mutable_field([ "all_models" @@ -72,6 +87,9 @@ class ensemble_hofx_packets(SuiteQuestion): @dataclass class ensemble_hofx_strategy(SuiteQuestion): + ''' + Strategy to use for ensemble HofX experiment, serial or parallel. + ''' default_value: str = "defer_to_model" models: List[str] = mutable_field([ "all_models" @@ -83,6 +101,9 @@ class ensemble_hofx_strategy(SuiteQuestion): @dataclass class experiment_id(SuiteQuestion): + ''' + ID that swell and cylc will use to reference the experiment, defaults to `swell-` + ''' default_value: str = "defer_to_code" prompt: str = "What is the experiment id?" data_type: DType = DType.STRING @@ -91,6 +112,9 @@ class experiment_id(SuiteQuestion): @dataclass class experiment_root(SuiteQuestion): + ''' + Root directory to place experiment in, defaults to `$NOBACKUP/SwellExperiments` on Discover. + ''' default_value: str = "defer_to_platform" prompt: str = ("What is the experiment root (the directory where the " "experiment will be stored)?") @@ -100,6 +124,9 @@ class experiment_root(SuiteQuestion): @dataclass class final_cycle_point(SuiteQuestion): + ''' + Final cycle point of experiment. + ''' default_value: str = "2023-10-10T06:00:00Z" prompt: str = "What is the time of the final cycle (middle of the window)?" data_type: DType = DType.ISO_DATETIME @@ -108,6 +135,9 @@ class final_cycle_point(SuiteQuestion): @dataclass class marine_models(SuiteQuestion): + ''' + Marine models, relevant to `geos_marine` applications, includes `cice6` and `soca`. + ''' default_value: str = "defer_to_model" options: str = "defer_to_model" models: List[str] = mutable_field([ @@ -120,6 +150,9 @@ class marine_models(SuiteQuestion): @dataclass class mock_experiment(SuiteQuestion): + ''' + Dry-run option that simulates directories and other variables, used in comparing JEDI configs. + ''' default_value: bool = False prompt: str = "Dry-run option for comparing configs." data_type: DType = DType.BOOLEAN @@ -128,6 +161,10 @@ class mock_experiment(SuiteQuestion): @dataclass class model_components(SuiteQuestion): + ''' + GEOS model components that the experiment will be running, + such as (geos_marine, geos_atmosphere, geos_cf) + ''' default_value: str = "defer_to_code" options: str = "defer_to_code" prompt: str = "Enter the model components for this model." @@ -386,6 +423,7 @@ class bufr_obs_classes(TaskQuestion): @dataclass class bundles(TaskQuestion): + '''Selected JEDI bundles to build.''' default_value: List[str] = mutable_field([ "fv3-jedi", "soca", diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 715f5d279..2eec5d277 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -130,19 +130,22 @@ def resolve(self, question: SwellQuestion, default='LrZRExPGcQ'): question_obj = question() name = question_obj.question_name + # Check that the task has access to this question if name in self.question_list and name in self.experiment_dict: default = self.experiment_dict[name] elif name in self.experiment_dict: raise KeyError(f'Value {name} is present in config but this task has not been assigned' ' it in `task_questions.py`') + # Check that a default has been provided if default == 'LrZRExPGcQ': raise KeyError(f'Trying to reference value {name} in config but this key does not ' 'exist and no default has been provided') - - data_type = question_obj.data_type + # Check that the provided value conforms to the expected type. + data_type = question_obj.data_type if isinstance(data_type, list): + # Check for multiple items in list if not any([dtype.is_type(default) for dtype in data_type]): self.__logger__.warning(f'Warning: Experiment key {name} does not conform to any expected' f' types {" ".join(data_type)}') From 6af526321de9433aa3cebfbf23704b07c48c98f9 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Wed, 26 Aug 2026 13:54:53 -0400 Subject: [PATCH 14/15] Add class spacing --- src/swell/configuration/question_defaults.py | 139 ++++++++++++++++++- 1 file changed, 138 insertions(+), 1 deletion(-) diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py index a69df5e95..bb6eba622 100644 --- a/src/swell/configuration/question_defaults.py +++ b/src/swell/configuration/question_defaults.py @@ -19,6 +19,7 @@ # Suite question defaults go here # -------------------------------------------------------------------------------------------------- + @dataclass class comparison_experiment_paths(SuiteQuestion): '''Paths to two `experiment.yaml` files to be compared by the suite''' @@ -28,6 +29,7 @@ class comparison_experiment_paths(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class cycle_times(SuiteQuestion): '''Cycle times that cylc will execute tasks for between the start and end cycle points''' @@ -41,6 +43,7 @@ class cycle_times(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class cycling_varbc(SuiteQuestion): ''' @@ -56,6 +59,7 @@ class cycling_varbc(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class download_convert_pipeline(SuiteQuestion): ''' @@ -70,6 +74,7 @@ class download_convert_pipeline(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_packets(SuiteQuestion): ''' @@ -85,6 +90,7 @@ class ensemble_hofx_packets(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_strategy(SuiteQuestion): ''' @@ -99,7 +105,7 @@ class ensemble_hofx_strategy(SuiteQuestion): # -------------------------------------------------------------------------------------------------- -@dataclass + class experiment_id(SuiteQuestion): ''' ID that swell and cylc will use to reference the experiment, defaults to `swell-` @@ -110,6 +116,7 @@ class experiment_id(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class experiment_root(SuiteQuestion): ''' @@ -122,6 +129,7 @@ class experiment_root(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class final_cycle_point(SuiteQuestion): ''' @@ -133,6 +141,7 @@ class final_cycle_point(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class marine_models(SuiteQuestion): ''' @@ -148,6 +157,7 @@ class marine_models(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class mock_experiment(SuiteQuestion): ''' @@ -159,6 +169,7 @@ class mock_experiment(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class model_components(SuiteQuestion): ''' @@ -172,6 +183,7 @@ class model_components(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ingest_background_pipeline(SuiteQuestion): default_value: bool = False @@ -180,6 +192,7 @@ class ingest_background_pipeline(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class parser_options(SuiteQuestion): default_value: list = mutable_field(['fgrep_residual_norm']) @@ -189,6 +202,7 @@ class parser_options(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class r2d2_experiment_id(SuiteQuestion): default_value: str = "defer_to_code" @@ -197,6 +211,7 @@ class r2d2_experiment_id(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class r2d2_server(SuiteQuestion): default_value: str | None = None @@ -208,6 +223,7 @@ class r2d2_server(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class r2d2_datastore(SuiteQuestion): default_value: str | None = None @@ -222,6 +238,7 @@ class r2d2_datastore(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class runahead_limit(SuiteQuestion): default_value: str = "P4" @@ -232,6 +249,7 @@ class runahead_limit(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class saber_central_block(SuiteQuestion): default_value: str = "defer_to_model" @@ -244,6 +262,7 @@ class saber_central_block(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class saber_outer_block(SuiteQuestion): default_value: str = "defer_to_model" @@ -256,6 +275,7 @@ class saber_outer_block(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class skip_ensemble_hofx(SuiteQuestion): default_value: str = "defer_to_model" @@ -267,6 +287,7 @@ class skip_ensemble_hofx(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class skip_r2d2(SuiteQuestion): default_value: bool = False @@ -275,6 +296,7 @@ class skip_r2d2(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class start_cycle_point(SuiteQuestion): default_value: str = "2023-10-10T00:00:00Z" @@ -283,6 +305,7 @@ class start_cycle_point(SuiteQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class window_type(SuiteQuestion): default_value: str = "defer_to_model" @@ -300,6 +323,7 @@ class window_type(SuiteQuestion): # Task question defaults go here # -------------------------------------------------------------------------------------------------- + @dataclass class analysis_variables(TaskQuestion): default_value: str = "defer_to_model" @@ -312,6 +336,7 @@ class analysis_variables(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class background_error_model(TaskQuestion): default_value: str = "defer_to_model" @@ -324,6 +349,7 @@ class background_error_model(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class background_experiment(TaskQuestion): default_value: str = "defer_to_model" @@ -335,6 +361,7 @@ class background_experiment(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class background_frequency(TaskQuestion): default_value: str = "defer_to_model" @@ -349,6 +376,7 @@ class background_frequency(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class background_time_offset(TaskQuestion): default_value: str = "defer_to_model" @@ -361,6 +389,7 @@ class background_time_offset(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ebkg_time_offset(TaskQuestion): default_value: str = "defer_to_model" @@ -373,6 +402,7 @@ class ebkg_time_offset(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class rst_experiment(TaskQuestion): default_value: str = "defer_to_model" @@ -384,6 +414,7 @@ class rst_experiment(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class rst_file_types(TaskQuestion): default_value: str = "defer_to_model" @@ -396,6 +427,7 @@ class rst_file_types(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class rst_store_interval(TaskQuestion): default_value: str = None @@ -409,6 +441,7 @@ class rst_store_interval(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class bufr_obs_classes(TaskQuestion): default_value: str = "defer_to_model" @@ -421,6 +454,7 @@ class bufr_obs_classes(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class bundles(TaskQuestion): '''Selected JEDI bundles to build.''' @@ -447,6 +481,7 @@ class bundles(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class check_for_obs(TaskQuestion): default_value: bool = True @@ -459,6 +494,7 @@ class check_for_obs(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class clean_patterns(TaskQuestion): default_value: str = "defer_to_model" @@ -471,6 +507,7 @@ class clean_patterns(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class comparison_log_type(TaskQuestion): default_value: str = "variational" @@ -486,6 +523,7 @@ class comparison_log_type(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class crtm_coeff_dir(TaskQuestion): default_value: str = "defer_to_platform" @@ -497,6 +535,7 @@ class crtm_coeff_dir(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_packets(TaskQuestion): default_value: str = "defer_to_model" @@ -509,6 +548,7 @@ class ensemble_hofx_packets(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_hofx_strategy(TaskQuestion): default_value: str = "defer_to_model" @@ -521,6 +561,7 @@ class ensemble_hofx_strategy(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensemble_num_members(TaskQuestion): default_value: str = "defer_to_model" @@ -533,6 +574,7 @@ class ensemble_num_members(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class obs_pert_amplitude(TaskQuestion): default_value: str = "defer_to_model" @@ -545,6 +587,7 @@ class obs_pert_amplitude(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensmean_only(TaskQuestion): default_value: bool = False @@ -560,6 +603,7 @@ class ensmean_only(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensmeanvariance_only(TaskQuestion): default_value: bool = False @@ -575,6 +619,7 @@ class ensmeanvariance_only(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ensmeanvariance_spec(TaskQuestion): default_value: List[Dict[str, str]] = field(default_factory=lambda: [{}]) @@ -587,6 +632,7 @@ class ensmeanvariance_spec(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class diffstates_spec(TaskQuestion): default_value: Dict[str, Any] = field(default_factory=dict) @@ -598,6 +644,7 @@ class diffstates_spec(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_geos_gcm_build_path(TaskQuestion): default_value: str = "defer_to_platform" @@ -609,6 +656,7 @@ class existing_geos_gcm_build_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_geos_gcm_source_path(TaskQuestion): default_value: str = "defer_to_platform" @@ -620,6 +668,7 @@ class existing_geos_gcm_source_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_jedi_build_directory(TaskQuestion): default_value: str = "defer_to_platform" @@ -631,6 +680,7 @@ class existing_jedi_build_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_jedi_build_directory_pinned(TaskQuestion): default_value: str = "defer_to_platform" @@ -642,6 +692,7 @@ class existing_jedi_build_directory_pinned(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_jedi_source_directory(TaskQuestion): default_value: str = "defer_to_platform" @@ -653,6 +704,7 @@ class existing_jedi_source_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_jedi_source_directory_pinned(TaskQuestion): default_value: str = "defer_to_platform" @@ -664,6 +716,7 @@ class existing_jedi_source_directory_pinned(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class existing_perllib_path(TaskQuestion): default_value: str = 'defer_to_platform' @@ -673,6 +726,7 @@ class existing_perllib_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class gmao_perllib_tag(TaskQuestion): default_value: str = 'g1.0.1' @@ -682,6 +736,7 @@ class gmao_perllib_tag(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class forecast_duration(TaskQuestion): default_value: str = "PT12H" @@ -690,6 +745,7 @@ class forecast_duration(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class forecast_length(TaskQuestion): default_value: str = "PT12H" @@ -701,6 +757,7 @@ class forecast_length(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class forecast_output_frequency(TaskQuestion): default_value: str = "PT1H" @@ -712,6 +769,7 @@ class forecast_output_frequency(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class generate_yaml_and_exit(TaskQuestion): default_value: bool = False @@ -720,6 +778,7 @@ class generate_yaml_and_exit(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_build_method(TaskQuestion): default_value: str = "create" @@ -732,6 +791,7 @@ class geos_build_method(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_homdir(TaskQuestion): default_value: str = "defer_to_platform" @@ -741,6 +801,7 @@ class geos_homdir(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_expdir_different(TaskQuestion): default_value: str = False @@ -754,6 +815,7 @@ class geos_expdir_different(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_expdir(TaskQuestion): default_value: str = "/dev/null/" @@ -767,6 +829,7 @@ class geos_expdir(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_cf_install_dir(TaskQuestion): default_value: str = "defer_to_platform" @@ -778,6 +841,7 @@ class geos_cf_install_dir(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_cf_run_dir(TaskQuestion): default_value: str = "defer_to_platform" @@ -789,6 +853,7 @@ class geos_cf_run_dir(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geosfp_exp(TaskQuestion): default_value: str = "f5295_fp" @@ -800,6 +865,7 @@ class geosfp_exp(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geosfp_path(TaskQuestion): default_value: str = "defer_to_platform" @@ -811,6 +877,7 @@ class geosfp_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_gcm_tag(TaskQuestion): default_value: str = "v11.6.0" @@ -822,6 +889,7 @@ class geos_gcm_tag(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_x_background_directory(TaskQuestion): default_value: str = "/dev/null/" @@ -837,6 +905,7 @@ class geos_x_background_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geos_x_ensemble_directory(TaskQuestion): default_value: str = "/dev/null/" @@ -852,6 +921,7 @@ class geos_x_ensemble_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geovals_experiment(TaskQuestion): default_value: str = "defer_to_model" @@ -863,6 +933,7 @@ class geovals_experiment(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class geovals_provider(TaskQuestion): default_value: str = "defer_to_model" @@ -874,6 +945,7 @@ class geovals_provider(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class gradient_norm_reduction(TaskQuestion): default_value: str = "defer_to_model" @@ -885,6 +957,7 @@ class gradient_norm_reduction(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class gsibec_configuration(TaskQuestion): default_value: str = "defer_to_model" @@ -896,6 +969,7 @@ class gsibec_configuration(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class gsibec_nlats(TaskQuestion): default_value: str = "defer_to_model" @@ -907,6 +981,7 @@ class gsibec_nlats(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class gsibec_nlons(TaskQuestion): default_value: str = "defer_to_model" @@ -918,6 +993,7 @@ class gsibec_nlons(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class horizontal_resolution(TaskQuestion): default_value: str = "defer_to_model" @@ -930,6 +1006,7 @@ class horizontal_resolution(TaskQuestion): # ------------------------------------------------------------------------------------------------ + @dataclass class dry_run(TaskQuestion): default_value: bool = True @@ -941,6 +1018,7 @@ class dry_run(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class store_as_symlink(TaskQuestion): default_value: bool = True @@ -952,6 +1030,7 @@ class store_as_symlink(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class obs_rc_path(TaskQuestion): default_value: str = 'GEOS_mksi/ObsClass/obsys-nccs.rc' @@ -961,6 +1040,7 @@ class obs_rc_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class obs_to_ingest(TaskQuestion): default_value: list = mutable_field([]) @@ -973,6 +1053,7 @@ class obs_to_ingest(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class obs_to_download(TaskQuestion): default_value: list = mutable_field([]) @@ -984,6 +1065,7 @@ class obs_to_download(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class converter_path(TaskQuestion): default_value: str = "" @@ -996,6 +1078,7 @@ class converter_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class initial_restarts_method(TaskQuestion): default_value: str = "defer_to_platform" @@ -1009,6 +1092,7 @@ class initial_restarts_method(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ioda_locations_not_in_r2d2(TaskQuestion): default_value: str = "defer_to_platform" @@ -1021,6 +1105,7 @@ class ioda_locations_not_in_r2d2(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class iau(TaskQuestion): default_value: bool = True @@ -1036,6 +1121,7 @@ class iau(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class inc_template(TaskQuestion): default_value: str = "defer_to_platform" @@ -1047,6 +1133,7 @@ class inc_template(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class jedi_build_method(TaskQuestion): default_value: str = "use_existing" @@ -1061,6 +1148,7 @@ class jedi_build_method(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class jedi_forecast_model(TaskQuestion): default_value: str = "defer_to_model" @@ -1076,6 +1164,7 @@ class jedi_forecast_model(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_inflation_mult(TaskQuestion): default_value: str = "defer_to_model" @@ -1087,6 +1176,7 @@ class local_ensemble_inflation_mult(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_inflation_rtpp(TaskQuestion): default_value: str = "defer_to_model" @@ -1098,6 +1188,7 @@ class local_ensemble_inflation_rtpp(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_inflation_rtps(TaskQuestion): default_value: str = "defer_to_model" @@ -1109,6 +1200,7 @@ class local_ensemble_inflation_rtps(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_save_posterior_ensemble(TaskQuestion): default_value: bool = False @@ -1124,6 +1216,7 @@ class local_ensemble_save_posterior_ensemble(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): default_value: bool = False @@ -1139,6 +1232,7 @@ class local_ensemble_save_posterior_ensemble_increments(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_save_posterior_mean(TaskQuestion): default_value: bool = False @@ -1154,6 +1248,7 @@ class local_ensemble_save_posterior_mean(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_save_posterior_mean_increment(TaskQuestion): default_value: bool = True @@ -1169,6 +1264,7 @@ class local_ensemble_save_posterior_mean_increment(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_solver(TaskQuestion): default_value: str = "defer_to_model" @@ -1181,6 +1277,7 @@ class local_ensemble_solver(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class local_ensemble_use_linear_observer(TaskQuestion): default_value: str = "defer_to_model" @@ -1193,6 +1290,7 @@ class local_ensemble_use_linear_observer(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class minimizer(TaskQuestion): default_value: str = "defer_to_model" @@ -1205,6 +1303,7 @@ class minimizer(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class mom6_iau(TaskQuestion): default_value: str = "defer_to_model" @@ -1220,6 +1319,7 @@ class mom6_iau(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class mom6_iau_nhours(TaskQuestion): default_value: str = "defer_to_model" @@ -1236,6 +1336,7 @@ class mom6_iau_nhours(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class ncdiag_experiments(TaskQuestion): default_value: str = "defer_to_model" @@ -1248,6 +1349,7 @@ class ncdiag_experiments(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class npx_proc(TaskQuestion): default_value: str = "defer_to_model" @@ -1261,6 +1363,7 @@ class npx_proc(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class npy_proc(TaskQuestion): default_value: str = "defer_to_model" @@ -1274,6 +1377,7 @@ class npy_proc(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class npx(TaskQuestion): default_value: str = "defer_to_model" @@ -1285,6 +1389,7 @@ class npx(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class npy(TaskQuestion): default_value: str = "defer_to_model" @@ -1296,6 +1401,7 @@ class npy(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class number_of_iterations(TaskQuestion): default_value: str = "defer_to_model" @@ -1309,6 +1415,7 @@ class number_of_iterations(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class obs_experiment(TaskQuestion): default_value: str = "defer_to_model" @@ -1320,6 +1427,7 @@ class obs_experiment(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class observation_providers(TaskQuestion): default_value: Dict[str, str] = mutable_field({}) @@ -1331,6 +1439,7 @@ class observation_providers(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class obs_thinning_rej_fraction(TaskQuestion): default_value: float = 0.75 @@ -1342,6 +1451,7 @@ class obs_thinning_rej_fraction(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class observations(TaskQuestion): default_value: str = "defer_to_model" @@ -1354,6 +1464,7 @@ class observations(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class observing_system_records_mksi_path(TaskQuestion): default_value: str = "defer_to_model" @@ -1365,6 +1476,7 @@ class observing_system_records_mksi_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class observing_system_records_mksi_path_tag(TaskQuestion): default_value: str = "defer_to_model" @@ -1376,6 +1488,7 @@ class observing_system_records_mksi_path_tag(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class observing_system_records_path(TaskQuestion): default_value: str = "defer_to_model" @@ -1387,6 +1500,7 @@ class observing_system_records_path(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class path_to_ensemble(TaskQuestion): default_value: str = "defer_to_model" @@ -1398,6 +1512,7 @@ class path_to_ensemble(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class path_to_geos_adas_background(TaskQuestion): default_value: str = "defer_to_model" @@ -1410,6 +1525,7 @@ class path_to_geos_adas_background(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class path_to_gsi_bc_coefficients(TaskQuestion): default_value: str = "defer_to_model" @@ -1421,6 +1537,7 @@ class path_to_gsi_bc_coefficients(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class path_to_gsi_nc_diags(TaskQuestion): default_value: str = "defer_to_model" @@ -1432,6 +1549,7 @@ class path_to_gsi_nc_diags(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class perhost(TaskQuestion): default_value: str = None @@ -1443,6 +1561,7 @@ class perhost(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class produce_geovals(TaskQuestion): default_value: str = "defer_to_model" @@ -1459,6 +1578,7 @@ class produce_geovals(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class publish_directory(TaskQuestion): default_value: str = None @@ -1467,6 +1587,7 @@ class publish_directory(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class cache_fetch(TaskQuestion): default_value: bool = True @@ -1479,6 +1600,7 @@ class cache_fetch(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class fetch_obs_from_public_s3(TaskQuestion): default_value: bool = False @@ -1491,6 +1613,7 @@ class fetch_obs_from_public_s3(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class save_geovals(TaskQuestion): default_value: bool = False @@ -1503,6 +1626,7 @@ class save_geovals(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class single_observations(TaskQuestion): default_value: bool = False @@ -1518,6 +1642,7 @@ class single_observations(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class swell_static_files(TaskQuestion): default_value: str = "defer_to_platform" @@ -1529,6 +1654,7 @@ class swell_static_files(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class swell_static_files_user(TaskQuestion): default_value: str = "None" @@ -1537,6 +1663,7 @@ class swell_static_files_user(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class total_processors(TaskQuestion): default_value: str = "defer_to_model" @@ -1548,6 +1675,7 @@ class total_processors(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_apply_log_transform(TaskQuestion): default_value: bool = True @@ -1565,6 +1693,7 @@ class vertical_localization_apply_log_transform(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_function(TaskQuestion): default_value: str = "defer_to_model" @@ -1577,6 +1706,7 @@ class vertical_localization_function(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_ioda_vertical_coord(TaskQuestion): default_value: str = "defer_to_model" @@ -1589,6 +1719,7 @@ class vertical_localization_ioda_vertical_coord(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_ioda_vertical_coord_group(TaskQuestion): default_value: str = "defer_to_model" @@ -1602,6 +1733,7 @@ class vertical_localization_ioda_vertical_coord_group(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_lengthscale(TaskQuestion): default_value: str = "defer_to_model" @@ -1613,6 +1745,7 @@ class vertical_localization_lengthscale(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_localization_method(TaskQuestion): default_value: str = "defer_to_model" @@ -1626,6 +1759,7 @@ class vertical_localization_method(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class vertical_resolution(TaskQuestion): default_value: str = "defer_to_model" @@ -1639,6 +1773,7 @@ class vertical_resolution(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class window_length(TaskQuestion): default_value: str = "defer_to_model" @@ -1650,6 +1785,7 @@ class window_length(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class window_type(TaskQuestion): default_value: str = "defer_to_model" @@ -1665,6 +1801,7 @@ class window_type(TaskQuestion): # -------------------------------------------------------------------------------------------------- + @dataclass class background_source_path(TaskQuestion): default_value: str = ( From 5a14bc7135321e0ef76f6f5597f8aded613ce3b6 Mon Sep 17 00:00:00 2001 From: Michael Anstett Date: Fri, 28 Aug 2026 15:58:36 -0400 Subject: [PATCH 15/15] test fixes --- pycodestyle.cfg | 2 +- src/swell/configuration/question_defaults.py | 48 +++++++++---------- .../question_and_answer_defaults.py | 4 +- src/swell/tasks/bufr_to_ioda.py | 2 +- src/swell/tasks/eva_jedi_log.py | 2 +- src/swell/tasks/prep_coupled_geos_run_dir.py | 4 +- src/swell/tasks/save_restart.py | 2 +- src/swell/utilities/config.py | 1 - src/swell/utilities/swell_questions.py | 7 ++- 9 files changed, 34 insertions(+), 38 deletions(-) diff --git a/pycodestyle.cfg b/pycodestyle.cfg index c8aad235a..9e0e82f45 100644 --- a/pycodestyle.cfg +++ b/pycodestyle.cfg @@ -5,7 +5,7 @@ # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. [pycodestyle] -max-line-length = 100 +max-line-length = 120 indent-size = 4 statistics = True exclude = ._*, build, .venv, venv, .git, __pycache__, .tox, .eggs, *.egg-info, dist diff --git a/src/swell/configuration/question_defaults.py b/src/swell/configuration/question_defaults.py index bb6eba622..3761ba8f8 100644 --- a/src/swell/configuration/question_defaults.py +++ b/src/swell/configuration/question_defaults.py @@ -66,10 +66,10 @@ class download_convert_pipeline(SuiteQuestion): Boolean option in `flow.cylc` deciding whether or not to run DownloadObs and ConvertObsToIoda tasks. ''' - + default_value: bool = False prompt: str = ("Run the DownloadObs and ConvertObsToIoda tasks?" - "(DownloadObs -> ConvertObsToIoda) -> IngestObs to R2D2") + "(DownloadObs -> ConvertObsToIoda) -> IngestObs to R2D2") data_type: DType = DType.BOOLEAN # -------------------------------------------------------------------------------------------------- @@ -106,6 +106,7 @@ class ensemble_hofx_strategy(SuiteQuestion): # -------------------------------------------------------------------------------------------------- +@dataclass class experiment_id(SuiteQuestion): ''' ID that swell and cylc will use to reference the experiment, defaults to `swell-` @@ -124,7 +125,7 @@ class experiment_root(SuiteQuestion): ''' default_value: str = "defer_to_platform" prompt: str = ("What is the experiment root (the directory where the " - "experiment will be stored)?") + "experiment will be stored)?") data_type: DType = DType.STRING # -------------------------------------------------------------------------------------------------- @@ -243,8 +244,8 @@ class r2d2_datastore(SuiteQuestion): class runahead_limit(SuiteQuestion): default_value: str = "P4" prompt: str = ("Set the Cylc runahead limit: the maximum number of cycles " - "that may be active ahead of the current cycle " - "(e.g. P1: up to 1 cycle ahead, P3: up to 3 cycles ahead, default P4).") + "that may be active ahead of the current cycle " + "(e.g. P1: up to 1 cycle ahead, P3: up to 3 cycles ahead, default P4).") data_type: DType = DType.STRING # -------------------------------------------------------------------------------------------------- @@ -384,7 +385,7 @@ class background_time_offset(TaskQuestion): "all_models" ]) prompt: str = ("How long before the middle of the analysis window did" - " the background providing forecast begin?") + " the background providing forecast begin?") data_type: DType = DType.ISO_DURATION # -------------------------------------------------------------------------------------------------- @@ -397,7 +398,7 @@ class ebkg_time_offset(TaskQuestion): "all_models" ]) prompt: str = ("How long before the middle of the analysis window did" - " the ensemble background providing forecast begin?") + " the ensemble background providing forecast begin?") data_type: DType = DType.ISO_DURATION # -------------------------------------------------------------------------------------------------- @@ -435,8 +436,8 @@ class rst_store_interval(TaskQuestion): "geos_cf" ]) prompt: str = ("After how many cycles should restart files be stored as real files " - "(not symlinks)? E.g. 28 means every 28th cycle (and multiples) stores " - "real files. Leave unset to always store as symlinks.") + "(not symlinks)? E.g. 28 means every 28th cycle (and multiples) stores " + "real files. Leave unset to always store as symlinks.") data_type: DType = DType.INTEGER # -------------------------------------------------------------------------------------------------- @@ -626,7 +627,7 @@ class ensmeanvariance_spec(TaskQuestion): models: List[str] = mutable_field([ "all_models" ]) - + prompt: str = "Configure the ensemble mean and variance specifications:" data_type: DType = DType.STRING @@ -796,7 +797,7 @@ class geos_build_method(TaskQuestion): class geos_homdir(TaskQuestion): default_value: str = "defer_to_platform" prompt: str = ("What is the location for the HOME Directory (HOMDIR in gcm_run and " - "gcm_setup) that contains model settings and RC files?") + "gcm_setup) that contains model settings and RC files?") data_type: DType = DType.STRING # -------------------------------------------------------------------------------------------------- @@ -810,7 +811,7 @@ class geos_expdir_different(TaskQuestion): False ]) prompt: str = ("Is your GEOS EXPERIMENT Directory, where restarts and scratch is located, " - "different than your GEOS HOME Directory?") + "different than your GEOS HOME Directory?") data_type: DType = DType.BOOLEAN # -------------------------------------------------------------------------------------------------- @@ -823,8 +824,8 @@ class geos_expdir(TaskQuestion): "geos_expdir_different": True }) prompt: str = ("What is the location for the EXPERIMENT Directory (to contain model " - "output and restart files), if it is different than your GEOS HOME " - "Directory?") + "output and restart files), if it is different than your GEOS HOME " + "Directory?") data_type: DType = DType.STRING # -------------------------------------------------------------------------------------------------- @@ -1073,7 +1074,7 @@ class converter_path(TaskQuestion): "all_models" ]) prompt: str = ("Path to directory containing ioda-converter scripts" - " (leave blank to use jedi_bin)") + " (leave blank to use jedi_bin)") data_type: DType = DType.STRING # -------------------------------------------------------------------------------------------------- @@ -1353,7 +1354,6 @@ class ncdiag_experiments(TaskQuestion): @dataclass class npx_proc(TaskQuestion): default_value: str = "defer_to_model" - models: List[str] = mutable_field([ "geos_atmosphere", "geos_cf" @@ -1367,7 +1367,6 @@ class npx_proc(TaskQuestion): @dataclass class npy_proc(TaskQuestion): default_value: str = "defer_to_model" - models: List[str] = mutable_field([ "geos_atmosphere", "geos_cf" @@ -1573,7 +1572,7 @@ class produce_geovals(TaskQuestion): "geos_atmosphere" ]) prompt: str = ("When running the ncdiag to ioda converted do you " - "want to produce GeoVaLs files?") + "want to produce GeoVaLs files?") data_type: DType = DType.BOOLEAN # -------------------------------------------------------------------------------------------------- @@ -1687,8 +1686,8 @@ class vertical_localization_apply_log_transform(TaskQuestion): "geos_atmosphere" ]) prompt: str = ("Should a log (base 10) transformation be applied " - "to vertical coordinate when " - "constructing vertical localization?") + "to vertical coordinate when " + "constructing vertical localization?") data_type: DType = DType.BOOLEAN # -------------------------------------------------------------------------------------------------- @@ -1728,7 +1727,7 @@ class vertical_localization_ioda_vertical_coord_group(TaskQuestion): "geos_atmosphere" ]) prompt: str = ("Which vertical coordinate group should be used " - "in constructing vertical localization?") + "in constructing vertical localization?") data_type: DType = DType.STRING # -------------------------------------------------------------------------------------------------- @@ -1754,7 +1753,7 @@ class vertical_localization_method(TaskQuestion): "geos_atmosphere" ]) prompt: str = ("What localization scheme should be applied in " - "constructing a vertical localization?") + "constructing a vertical localization?") data_type: DType = DType.STRING # -------------------------------------------------------------------------------------------------- @@ -1763,7 +1762,6 @@ class vertical_localization_method(TaskQuestion): @dataclass class vertical_resolution(TaskQuestion): default_value: str = "defer_to_model" - options: str = "defer_to_model" models: List[str] = mutable_field([ "all_models" @@ -1810,8 +1808,8 @@ class background_source_path(TaskQuestion): ) models: List[str] = mutable_field(['geos_cf']) prompt: str = ("Path template for background files. Uses Python strftime format codes, " - "e.g. Y%Y/M%m/D%d gives Y2025/M10/D02 and %Y%m%d_%H%Mz gives " - "20251002_0900z.") + "e.g. Y%Y/M%m/D%d gives Y2025/M10/D02 and %Y%m%d_%H%Mz gives " + "20251002_0900z.") data_type: DType = DType.STRING # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py b/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py index 2601f78c2..3e4a035da 100644 --- a/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py +++ b/src/swell/deployment/prepare_config_and_suite/question_and_answer_defaults.py @@ -27,8 +27,8 @@ def get_answer(self, logger: Logger, key: str, val: dict, elif not data_type.is_type(default): logger.warning(f'Default value for {key}, {default}, does not conform to type ' - f'{data_type.value}, check the override file or ' - 'suite configuration.') + f'{data_type.value}, check the override file or ' + 'suite configuration.') return default diff --git a/src/swell/tasks/bufr_to_ioda.py b/src/swell/tasks/bufr_to_ioda.py index 32ec279dd..c6997a5e9 100644 --- a/src/swell/tasks/bufr_to_ioda.py +++ b/src/swell/tasks/bufr_to_ioda.py @@ -13,7 +13,7 @@ import subprocess import shutil -import swell.configuration.question_defaults as qd +# import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/eva_jedi_log.py b/src/swell/tasks/eva_jedi_log.py index eab6ae81c..2632978ff 100644 --- a/src/swell/tasks/eva_jedi_log.py +++ b/src/swell/tasks/eva_jedi_log.py @@ -13,7 +13,7 @@ from eva.eva_driver import eva -import swell.configuration.question_defaults as qd +# import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase from swell.utilities.jinja2 import template_string_jinja2 diff --git a/src/swell/tasks/prep_coupled_geos_run_dir.py b/src/swell/tasks/prep_coupled_geos_run_dir.py index 0d5e2269b..da0a99673 100644 --- a/src/swell/tasks/prep_coupled_geos_run_dir.py +++ b/src/swell/tasks/prep_coupled_geos_run_dir.py @@ -207,8 +207,8 @@ def mom6_iau(self) -> None: mom_oda_incupd = self.forecast_dir('MOM_oda_incupd') mom6_config = self.geos.parse_mom6_input(mom_oda_incupd) # P50D is just a random input for get_key_for_model to function -self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)6self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)hself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)=self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)lself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',).self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)cself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',).self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)tself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)kself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)yself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)fself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)dself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)lself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)(self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)6self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)hself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)uself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',),self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)gself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)oself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)sself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)_self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)mself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)aself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)rself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)iself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)nself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)eself.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',)'self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',),self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) -self.config.resolve(qd.get_key_for_model, default='mom6_iau_nhours', 'geos_marine',) 'PT50D') + mom6_iau_nhours = self.config.get_key_for_model('mom6_iau_nhours', 'geos_marine', + 'PT50D') # convert ISO to 3.0 duration = isodate.parse_duration(mom6_iau_nhours) diff --git a/src/swell/tasks/save_restart.py b/src/swell/tasks/save_restart.py index d46116a56..764221308 100644 --- a/src/swell/tasks/save_restart.py +++ b/src/swell/tasks/save_restart.py @@ -7,7 +7,7 @@ # -------------------------------------------------------------------------------------------------- -import swell.configuration.question_defaults as qd +# import swell.configuration.question_defaults as qd from swell.tasks.base.task_base import taskBase # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/config.py b/src/swell/utilities/config.py index 2eec5d277..4a6967e73 100644 --- a/src/swell/utilities/config.py +++ b/src/swell/utilities/config.py @@ -8,7 +8,6 @@ from ruamel.yaml import YAML -from typing import Callable from swell.tasks.task_questions import TaskQuestions as task_questions from swell.utilities.logger import Logger diff --git a/src/swell/utilities/swell_questions.py b/src/swell/utilities/swell_questions.py index ecc02d3ec..5936dd4d2 100644 --- a/src/swell/utilities/swell_questions.py +++ b/src/swell/utilities/swell_questions.py @@ -16,7 +16,6 @@ from swell.utilities.datetime_util import is_datetime, is_duration from swell.swell_path import get_swell_path -from swell.utilities.logger import Logger # -------------------------------------------------------------------------------------------------- @@ -49,13 +48,13 @@ def is_type(self, value) -> bool: if self == DataType.STRING: return isinstance(value, str) - + if self == DataType.BOOLEAN: return isinstance(value, bool) if self == DataType.INTEGER: return isinstance(value, int) - + if self == DataType.INTEGER_LIST: if isinstance(value, list): return all([isinstance(item, int) for item in value]) @@ -63,7 +62,7 @@ def is_type(self, value) -> bool: return False if self == DataType.FLOAT: - return isinstance(value, float) + return isinstance(value, float) if self == DataType.LIST: return isinstance(value, list)