diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index d5d4690b38..dec1425738 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -214,14 +214,16 @@ def _get_use_case_cmds(host_name, use_case_by_requirement, work_dir, group_name, use_case_cmds.append('status=0') for use_case in use_case_by_requirement.use_cases: # add parm/use_cases path to config args if they are conf files - config_args = _get_config_args(use_case.config_args, work_dir, host_name) + config_args, ci_overrides = ( + _get_config_args(use_case.config_args, work_dir, host_name) + ) output_base = os.path.join(output_top_dir, group_name.split('-')[0], use_case.name) use_case_cmd = (f"run_metplus.py" f" {' '.join(config_args)}" - f" {py_embed_arg}{test_settings_conf}" + f" {py_embed_arg}{test_settings_conf} {ci_overrides}" f" config.OUTPUT_BASE={output_base}") use_case_cmds.append(use_case_cmd) # check exit code from use case command and @@ -238,7 +240,7 @@ def _get_use_case_cmds(host_name, use_case_by_requirement, work_dir, group_name, def _get_config_args(input_config_args, work_dir, host_name): config_args = [] - ci_overrides = None + ci_overrides = '' for config_arg in input_config_args: if config_arg.endswith('.conf'): config_arg = os.path.join(work_dir, 'parm', @@ -252,11 +254,11 @@ def _get_config_args(input_config_args, work_dir, host_name): config_args.append(config_arg) - # add CI overrides config file if running in docker - if ci_overrides and host_name == 'docker': - config_args.append(ci_overrides) + # remove CI overrides config file if not running in docker + if ci_overrides and host_name != 'docker': + ci_overrides = '' - return config_args + return config_args, ci_overrides def handle_command_line_args(): diff --git a/.github/parm/test_settings.conf b/.github/parm/test_settings.conf index 6bd0227586..c6f03b9101 100644 --- a/.github/parm/test_settings.conf +++ b/.github/parm/test_settings.conf @@ -14,6 +14,8 @@ FILE_LISTS_DIR = {STAGING_DIR}/file_lists SCRUB_STAGING_DIR=False +EXIT_IF_WARN = True + # also set path to GempakToCF.jar for GEMPAK use cases GEMPAKTOCF_JAR = /data/input/GempakToCF.jar diff --git a/.idea/modules.xml b/.idea/modules.xml index 9fd0385cc9..0a9b5744ba 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -2,7 +2,7 @@ - + \ No newline at end of file diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 803e3abdde..6b942d43d6 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -15168,6 +15168,462 @@ METplus Configuration Glossary | *Used by:* GenEnsProd + WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run. + Set to False to suppress it. + + | *Used by:* All + + ASCII2NC_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for ASCII2NC. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* ASCII2NC + + DATA_INGEST_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for DataIngest. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* DataIngest + + ENSEMBLE_STAT_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for EnsembleStat. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* EnsembleStat + + EXTRACT_TILES_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for ExtractTiles. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* ExtractTiles + + GEN_VX_MASK_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for GenVxMask. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* GenVxMask + + GEN_ENS_PROD_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for GenEnsProd. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* GenEnsProd + + GRID_DIAG_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for GridDiag. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* GridDiag + + GRID_STAT_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for GridStat. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* GridStat + + IODA2NC_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for IODA2NC. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* IODA2NC + + MADIS2NC_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for MADIS2NC. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* MADIS2NC + + MODE_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for MODE. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* MODE + + MTD_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for MTD. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* MTD + + PAIR_STAT_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for PairStat. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* PairStat + + PB2NC_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for PB2NC. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* PB2NC + + PCP_COMBINE_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for PCPCombine. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* PCPCombine + + PLOT_DATA_PLANE_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for PlotDataPlane. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* PlotDataPlane + + PLOT_POINT_OBS_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for PlotPointObs. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* PlotPointObs + + POINT2GRID_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for Point2Grid. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* Point2Grid + + POINT_STAT_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for PointStat. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* PointStat + + PY_EMBED_INGEST_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for PyEmbedIngest. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* PyEmbedIngest + + REGRID_DATA_PLANE_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for RegridDataPlane. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* RegridDataPlane + + RMW_ANALYSIS_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for RMWAnalysis. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* RMWAnalysis + + SERIES_ANALYSIS_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for SeriesAnalysis. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* SeriesAnalysis + + STAT_ANALYSIS_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for StatAnalysis. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* StatAnalysis + + TC_DIAG_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for TCDiag. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* TCDiag + + TC_GEN_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for TCGen. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* TCGen + + TC_PAIRS_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for TCPairs. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* TCPairs + + TCRMW_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for TCRMW. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* TCRMW + + TC_STAT_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for TCStat. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* TCStat + + WAVELET_STAT_WARN_IF_DUPLICATE_OUTPUT + Log a warning message when an output file is written + more than once during a single METplus run for WaveletStat. + Set to False to suppress it. + See also :term:`WARN_IF_DUPLICATE_OUTPUT`. + + | *Used by:* WaveletStat + + WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* All + + ASCII2NC_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for ASCII2NC + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* ASCII2NC + + DATA_INGEST_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for DataIngest. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* DataIngest + + ENSEMBLE_STAT_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for EnsembleStat. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* EnsembleStat + + EXTRACT_TILES_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for ExtractTiles. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* ExtractTiles + + GEN_VX_MASK_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for GenVxMask. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* GenVxMask + + GEN_ENS_PROD_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for GenEnsProd. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* GenEnsProd + + GRID_DIAG_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for GridDiag. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* GridDiag + + GRID_STAT_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for GridStat. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* GridStat + + IODA2NC_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for IODA2NC. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* IODA2NC + + MADIS2NC_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for MADIS2NC. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* MADIS2NC + + MODE_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for MODE. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* MODE + + MTD_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for MTD. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* MTD + + PAIR_STAT_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for PairStat. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* PairStat + + PB2NC_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for PB2NC. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* PB2NC + + PCP_COMBINE_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for PCPCombine. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* PCPCombine + + PLOT_DATA_PLANE_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for PlotDataPlane. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* PlotDataPlane + + PLOT_POINT_OBS_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for PlotPointObs. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* PlotPointObs + + POINT2GRID_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for Point2Grid. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* Point2Grid + + POINT_STAT_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for PointStat. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* PointStat + + PY_EMBED_INGEST_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for PyEmbedIngest. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* PyEmbedIngest + + REGRID_DATA_PLANE_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for RegridDataPlane. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* RegridDataPlane + + RMW_ANALYSIS_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for RMWAnalysis. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* RMWAnalysis + + SERIES_ANALYSIS_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for SeriesAnalysis. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* SeriesAnalysis + + STAT_ANALYSIS_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for StatAnalysis. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* StatAnalysis + + TC_DIAG_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for TCDiag. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* TCDiag + + TC_GEN_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for TCGen. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* TCGen + + TC_PAIRS_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for TCPairs. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* TCPairs + + TCRMW_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for TCRMW. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* TCRMW + + TC_STAT_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for TCStat. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* TCStat + + WAVELET_STAT_WARN_IF_OUTPUT_EXISTS + Log a warning message when an output file already exists for WaveletStat. + Set to True to output warning. See also :term:`WARN_IF_OUTPUT_EXISTS`. + + | *Used by:* WaveletStat + + EXIT_IF_WARN + Immediately exit METplus run when a warning message is encountered. + + | *Used by:* All + + WARN_IF_ENSEMBLE_IS_MISSING + Log a warning if an ensemble member was not found. + Set to False to suppress it. + + | *Used by:* GenEnsProd, EnsembleStat + + GEN_ENS_PROD_WARN_IF_ENSEMBLE_IS_MISSING + Log a warning if an ensemble member was not found for GenEnsProd. + Set to False to suppress it. + + | *Used by:* GenEnsProd + + ENSEMBLE_STAT_WARN_IF_ENSEMBLE_IS_MISSING + Log a warning if an ensemble member was not found for EnsembleStat. + Set to False to suppress it. + + | *Used by:* EnsembleStat + POINT_STAT_KDE_REF_ANGLE Specify the value for 'kde_ref_angle' in the MET configuration file for PointStat. diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index 596f3e7d4a..592ce189ca 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -1481,14 +1481,14 @@ This is the behavior that was previously executed when LOOP_ORDER = processes. VALID_END = 20190203 VALID_INCREMENT = 1d -will run in the following order:: +will run in the following order: - * PCPCombine at 2019-02-01 - * PCPCombine at 2019-02-02 - * PCPCombine at 2019-02-03 - * GridStat at 2019-02-01 - * GridStat at 2019-02-02 - * GridStat at 2019-02-03 +* PCPCombine at 2019-02-01 +* PCPCombine at 2019-02-02 +* PCPCombine at 2019-02-03 +* GridStat at 2019-02-01 +* GridStat at 2019-02-02 +* GridStat at 2019-02-03 .. _Custom_Looping: @@ -2232,6 +2232,10 @@ how these variables affect how the data is processed. FCST_SERIES_ANALYSIS_INPUT_TEMPLATE = I{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}_V{valid?fmt=%H} + SERIES_ANALYSIS_OUTPUT_DIR = /my/output/dir + + SERIES_ANALYSIS_OUTPUT_TEMPLATE = I{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}_V{valid?fmt=%H}.nc + In this example, the wrapper will go through all initialization and forecast lead times and find any files that match the template under /my/fcst/dir: @@ -2251,8 +2255,11 @@ Example 1: Run Once:: For this configuration, a single command will be built to call SeriesAnalysis. The wildcard character '\*' will replace init, valid, and lead in the template when attempting to find data to process. +The output files will use 'all' in place of the filename template tags that +are ambiguous. Template Used: I\*_F\*_V\* + Files Processed:: I2020101712_F003_V15 @@ -2262,6 +2269,10 @@ Files Processed:: I2020101912_F003_V15 I2020101912_F006_V18 +Output Created:: + + Iall_Fall_Vall.nc + Example 2 Run Once Per Initialization Time:: [config] @@ -2273,26 +2284,45 @@ The wildcard character '\*' will replace valid and lead in the template when attempting to find data to process. Runtime: Init: 2020-10-17 12Z + Template Used: I2020101712_F\*_V\* + Files Processed:: I2020101712_F003_V15 I2020101712_F006_V18 +Output Created:: + + I2020101712_Fall_Vall.nc + Runtime: Init: 2020-10-18 12Z + Template Used: I2020101812_F\*_V\* + Files Processed:: I2020101812_F003_V15 I2020101812_F006_V18 +Output Created:: + + I2020101812_Fall_Vall.nc + + Runtime: Init: 2020-10-19 12Z + Template Used: I2020101912_F\*_V\* + Files Processed:: I2020101912_F003_V15 I2020101912_F006_V18 +Output Created:: + + I2020101912_Fall_Vall.nc + .. note:: If LOOP_BY was set to VALID, then the values defined by VALID_BEG, VALID_END, and VALID_INCREMENT would be substituted for the valid time @@ -2309,21 +2339,33 @@ The wildcard character '\*' will replace valid and init in the template when attempting to find data to process. Runtime: Lead: 3 hour + Template Used: I\*_F003*_V\* + Files Processed:: I2020101712_F003_V15 I2020101812_F003_V15 I2020101912_F003_V15 +Output Created:: + + Iall_F003_Vall.nc + Runtime: Lead: 6 hour + Template Used: I\*_F006*_V\* + Files Processed:: I2020101712_F006_V18 I2020101812_F006_V18 I2020101912_F006_V18 +Output Created:: + + Iall_F006_Vall.nc + Example 4 Run Once For Each Time:: [config] @@ -2335,41 +2377,77 @@ The wildcard character '\*' will replace valid only in the template when attempting to find data to process. Runtime: Init: 2020-10-17 12Z, Forecast: 3 hour + Template Used: I2020101712_F003_V\* + Files Processed:: I2020101712_F003_V15 +Output Created:: + + I2020101712_F003_V15.nc + Runtime: Init: 2020-10-17 12Z, Forecast: 6 hour + Template Used: I2020101712_F006_V\* + Files Processed:: I2020101712_F006_V18 +Output Created:: + + I2020101712_F006_V18.nc + Runtime: Init: 2020-10-18 12Z, Forecast: 3 hour + Template Used: I2020101812_F003_V\* + Files Processed:: I2020101812_F003_V15 +Output Created:: + + I2020101812_F003_V15.nc + Runtime: Init: 2020-10-18 12Z, Forecast: 6 hour + Template Used: I2020101812_F006_V\* + Files Processed:: I2020101812_F006_V18 +Output Created:: + + I2020101812_F006_V18.nc + Runtime: Init: 2020-10-19 12Z, Forecast: 3 hour + Template Used: I2020101912_F003_V\* + Files Processed:: I2020101912_F003_V15 +Output Created:: + + I2020101912_F003_V15.nc + Runtime: Init: 2020-10-19 12Z, Forecast: 6 hour + Template Used: I2020101912_F006_V\* + Files Processed:: I2020101912_F006_V18 +Output Created:: + + I2020101912_F006_V18.nc + .. _config-utilities: diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 00644a0c1e..bcd88ab5ca 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -67,7 +67,8 @@ METplus Configuration | :term:`ASCII2NC_INC_INIT_TIMES` | :term:`ASCII2NC_ALLOW_MISSING_INPUTS` | :term:`ASCII2NC_INPUT_THRESH` - +| :term:`ASCII2NC_WARN_IF_DUPLICATE_OUTPUT` +| :term:`ASCII2NC_WARN_IF_OUTPUT_EXISTS` | .. _ascii2nc-met-conf: @@ -214,6 +215,8 @@ Optional: | :term:`DATA_INGEST__AUTO_DECOMPRESS` | :term:`DATA_INGEST__USERNAME` | :term:`DATA_INGEST__PASSWORD` +| :term:`DATA_INGEST_WARN_IF_DUPLICATE_OUTPUT` +| :term:`DATA_INGEST_WARN_IF_OUTPUT_EXISTS` .. _ensemble_stat_wrapper: @@ -472,7 +475,9 @@ METplus Configuration | :term:`ENSEMBLE_STAT_INC_INIT_TIMES` | :term:`ENSEMBLE_STAT_ALLOW_MISSING_INPUTS` | :term:`ENSEMBLE_STAT_INPUT_THRESH` -| +| :term:`ENSEMBLE_STAT_WARN_IF_DUPLICATE_OUTPUT` +| :term:`ENSEMBLE_STAT_WARN_IF_OUTPUT_EXISTS` +| :term:`ENSEMBLE_STAT_WARN_IF_ENSEMBLE_IS_MISSING` .. warning:: **DEPRECATED:** @@ -1476,6 +1481,8 @@ the dimensions and density of the tiles comprising the subregion: | :term:`EXTRACT_TILES_VAR_LIST` | :term:`EXTRACT_TILES_SKIP_IF_OUTPUT_EXISTS` | :term:`EXTRACT_TILES_CUSTOM_LOOP_LIST` +| :term:`EXTRACT_TILES_WARN_IF_DUPLICATE_OUTPUT` +| :term:`EXTRACT_TILES_WARN_IF_OUTPUT_EXISTS` | .. warning:: **DEPRECATED:** @@ -1632,7 +1639,9 @@ METplus Configuration | :term:`GEN_ENS_PROD_INPUT_THRESH` | :term:`GEN_ENS_PROD_N_MEMBERS` | :term:`GEN_ENS_PROD_VLD_THRESH` - +| :term:`GEN_ENS_PROD_WARN_IF_DUPLICATE_OUTPUT` +| :term:`GEN_ENS_PROD_WARN_IF_OUTPUT_EXISTS` +| :term:`GEN_ENS_PROD_WARN_IF_ENSEMBLE_IS_MISSING` .. _gen-ens-prod-met-conf: @@ -2059,6 +2068,8 @@ Configuration | :term:`GEN_VX_MASK_INC_INIT_TIMES` | :term:`GEN_VX_MASK_ALLOW_MISSING_INPUTS` | :term:`GEN_VX_MASK_INPUT_THRESH` +| :term:`GEN_VX_MASK_WARN_IF_DUPLICATE_OUTPUT` +| :term:`GEN_VX_MASK_WARN_IF_OUTPUT_EXISTS` | .. _gfdl_tracker_wrapper: @@ -3372,6 +3383,8 @@ METplus Configuration | :term:`GRID_DIAG_OUTPUT_FLAG_HISTOGRAM_1D` | :term:`GRID_DIAG_OUTPUT_FLAG_HISTOGRAM_2D` | :term:`GRID_DIAG_OUTPUT_FLAG_INFO_THEORY` +| :term:`GRID_DIAG_WARN_IF_DUPLICATE_OUTPUT` +| :term:`GRID_DIAG_WARN_IF_OUTPUT_EXISTS` .. _grid-diag-met-conf: @@ -3741,7 +3754,8 @@ METplus Configuration | :term:`GRID_STAT_INPUT_THRESH` | :term:`GRID_STAT_GRADIENT_DX` | :term:`GRID_STAT_GRADIENT_DY` -| +| :term:`GRID_STAT_WARN_IF_DUPLICATE_OUTPUT` +| :term:`GRID_STAT_WARN_IF_OUTPUT_EXISTS` .. warning:: **DEPRECATED** @@ -4535,6 +4549,9 @@ METplus Configuration | :term:`IODA2NC_INC_INIT_TIMES` | :term:`IODA2NC_ALLOW_MISSING_INPUTS` | :term:`IODA2NC_INPUT_THRESH` +| :term:`IODA2NC_WARN_IF_DUPLICATE_OUTPUT` +| :term:`IODA2NC_WARN_IF_OUTPUT_EXISTS` + .. _ioda2nc-met-conf: @@ -4819,7 +4836,9 @@ METplus Configuration | :term:`MADIS2NC_INC_INIT_TIMES` | :term:`MADIS2NC_ALLOW_MISSING_INPUTS` | :term:`MADIS2NC_INPUT_THRESH` -| +| :term:`MADIS2NC_WARN_IF_DUPLICATE_OUTPUT` +| :term:`MADIS2NC_WARN_IF_OUTPUT_EXISTS` + .. _madis2nc-met-conf: @@ -5248,7 +5267,9 @@ METplus Configuration | :term:`MODE_TIME_OFFSET_WARNING` | :term:`MODE_ALLOW_MISSING_INPUTS` | :term:`MODE_INPUT_THRESH` -| +| :term:`MODE_WARN_IF_DUPLICATE_OUTPUT` +| :term:`MODE_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -6010,7 +6031,9 @@ METplus Configuration | :term:`MTD_INC_INIT_TIMES` | :term:`MTD_ALLOW_MISSING_INPUTS` | :term:`MTD_INPUT_THRESH` -| +| :term:`MTD_WARN_IF_DUPLICATE_OUTPUT` +| :term:`MTD_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -6434,6 +6457,9 @@ Configuration | :term:`PAIR_STAT_INC_INIT_TIMES` | :term:`PAIR_STAT_ALLOW_MISSING_INPUTS` | :term:`PAIR_STAT_INPUT_THRESH` +| :term:`PAIR_STAT_WARN_IF_DUPLICATE_OUTPUT` +| :term:`PAIR_STAT_WARN_IF_OUTPUT_EXISTS` + .. _pair-stat-met-conf: @@ -7390,6 +7416,9 @@ METplus Configuration | :term:`PB2NC_INC_INIT_TIMES` | :term:`PB2NC_ALLOW_MISSING_INPUTS` | :term:`PB2NC_INPUT_THRESH` +| :term:`PB2NC_WARN_IF_DUPLICATE_OUTPUT` +| :term:`PB2NC_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -7749,7 +7778,8 @@ METplus Configuration | :term:`OBS_PCP_COMBINE_VLD_THRESH` | :term:`PCP_COMBINE_ALLOW_MISSING_INPUTS` | :term:`PCP_COMBINE_INPUT_THRESH` -| +| :term:`PCP_COMBINE_WARN_IF_DUPLICATE_OUTPUT` +| :term:`PCP_COMBINE_WARN_IF_OUTPUT_EXISTS` .. warning:: **DEPRECATED:** @@ -7815,6 +7845,8 @@ Configuration | :term:`PLOT_DATA_PLANE_INC_INIT_TIMES` | :term:`PLOT_DATA_PLANE_ALLOW_MISSING_INPUTS` | :term:`PLOT_DATA_PLANE_INPUT_THRESH` +| :term:`PLOT_DATA_PLANE_WARN_IF_DUPLICATE_OUTPUT` +| :term:`PLOT_DATA_PLANE_WARN_IF_OUTPUT_EXISTS` .. _plot_point_obs_wrapper: @@ -7884,6 +7916,8 @@ Configuration | :term:`PLOT_POINT_OBS_INC_INIT_TIMES` | :term:`PLOT_POINT_OBS_ALLOW_MISSING_INPUTS` | :term:`PLOT_POINT_OBS_INPUT_THRESH` +| :term:`PLOT_POINT_OBS_WARN_IF_DUPLICATE_OUTPUT` +| :term:`PLOT_POINT_OBS_WARN_IF_OUTPUT_EXISTS` .. _plot-point-obs-met-conf: @@ -8273,7 +8307,9 @@ METplus Configuration | :term:`POINT2GRID_MET_CONFIG_OVERRIDES` | :term:`POINT2GRID_ALLOW_MISSING_INPUTS` | :term:`POINT2GRID_INPUT_THRESH` -| +| :term:`POINT2GRID_WARN_IF_DUPLICATE_OUTPUT` +| :term:`POINT2GRID_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -8649,7 +8685,9 @@ Configuration | :term:`POINT_STAT_WRITE_WEIGHTS` | :term:`POINT_STAT_ALLOW_MISSING_INPUTS` | :term:`POINT_STAT_INPUT_THRESH` -| +| :term:`POINT_STAT_WARN_IF_DUPLICATE_OUTPUT` +| :term:`POINT_STAT_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -9494,7 +9532,9 @@ METplus Configuration | :term:`PY_EMBED_INGEST_CUSTOM_LOOP_LIST` | :term:`PY_EMBED_INGEST__OUTPUT_FIELD_NAME` | :term:`PY_EMBED_INGEST_SKIP_IF_OUTPUT_EXISTS` -| +| :term:`PY_EMBED_INGEST_WARN_IF_DUPLICATE_OUTPUT` +| :term:`PY_EMBED_INGEST_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -9550,7 +9590,9 @@ METplus Configuration | :term:`REGRID_DATA_PLANE_INC_INIT_TIMES` | :term:`REGRID_DATA_PLANE_ALLOW_MISSING_INPUTS` | :term:`REGRID_DATA_PLANE_INPUT_THRESH` -| +| :term:`REGRID_DATA_PLANE_WARN_IF_DUPLICATE_OUTPUT` +| :term:`REGRID_DATA_PLANE_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -9606,7 +9648,9 @@ METplus Configuration | :term:`RMW_ANALYSIS_INC_VALID_TIMES` | :term:`RMW_ANALYSIS_SKIP_INIT_TIMES` | :term:`RMW_ANALYSIS_INC_INIT_TIMES` -| +| :term:`RMW_ANALYSIS_WARN_IF_DUPLICATE_OUTPUT` +| :term:`RMW_ANALYSIS_WARN_IF_OUTPUT_EXISTS` + .. _rmw-analysis-met-conf: @@ -10121,7 +10165,9 @@ METplus Configuration | :term:`SERIES_ANALYSIS_INPUT_THRESH` | :term:`SERIES_ANALYSIS_GRADIENT_DX` | :term:`SERIES_ANALYSIS_GRADIENT_DY` -| +| :term:`SERIES_ANALYSIS_WARN_IF_DUPLICATE_OUTPUT` +| :term:`SERIES_ANALYSIS_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -10938,6 +10984,9 @@ The following values are optional in the METplus configuration file: | :term:`STAT_ANALYSIS_INC_VALID_TIMES` | :term:`STAT_ANALYSIS_SKIP_INIT_TIMES` | :term:`STAT_ANALYSIS_INC_INIT_TIMES` +| :term:`STAT_ANALYSIS_WARN_IF_DUPLICATE_OUTPUT` +| :term:`STAT_ANALYSIS_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -11530,7 +11579,9 @@ METplus Configuration | :term:`TC_DIAG_INC_INIT_TIMES` | :term:`TC_DIAG_ALLOW_MISSING_INPUTS` | :term:`TC_DIAG_INPUT_THRESH` -| +| :term:`TC_DIAG_WARN_IF_DUPLICATE_OUTPUT` +| :term:`TC_DIAG_WARN_IF_OUTPUT_EXISTS` + .. _tc-diag-met-conf: @@ -12064,6 +12115,9 @@ METplus Configuration | :term:`TC_GEN_NC_PAIRS_GRID` | :term:`TC_GEN_ALLOW_MISSING_INPUTS` | :term:`TC_GEN_INPUT_THRESH` +| :term:`TC_GEN_WARN_IF_DUPLICATE_OUTPUT` +| :term:`TC_GEN_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -12725,7 +12779,9 @@ METplus Configuration | :term:`TC_PAIRS_INC_VALID_TIMES` | :term:`TC_PAIRS_SKIP_INIT_TIMES` | :term:`TC_PAIRS_INC_INIT_TIMES` -| +| :term:`TC_PAIRS_WARN_IF_DUPLICATE_OUTPUT` +| :term:`TC_PAIRS_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -13135,7 +13191,9 @@ METplus Configuration | :term:`TC_RMW_INC_VALID_TIMES` | :term:`TC_RMW_SKIP_INIT_TIMES` | :term:`TC_RMW_INC_INIT_TIMES` -| +| :term:`TCRMW_WARN_IF_DUPLICATE_OUTPUT` +| :term:`TCRMW_WARN_IF_OUTPUT_EXISTS` + .. _tc-rmw-met-conf: @@ -13564,7 +13622,9 @@ METplus Configuration | :term:`TC_STAT_INC_VALID_TIMES` | :term:`TC_STAT_SKIP_INIT_TIMES` | :term:`TC_STAT_INC_INIT_TIMES` -| +| :term:`TC_STAT_WARN_IF_DUPLICATE_OUTPUT` +| :term:`TC_STAT_WARN_IF_OUTPUT_EXISTS` + .. warning:: **DEPRECATED:** @@ -14330,6 +14390,9 @@ METplus Configuration | :term:`WAVELET_STAT_TIME_OFFSET_WARNING` | :term:`WAVELET_STAT_ALLOW_MISSING_INPUTS` | :term:`WAVELET_STAT_INPUT_THRESH` +| :term:`WAVELET_STAT_WARN_IF_DUPLICATE_OUTPUT` +| :term:`WAVELET_STAT_WARN_IF_OUTPUT_EXISTS` + .. _wavelet-stat-met-conf: diff --git a/docs/requirements.txt b/docs/requirements.txt index f470529137..29a78e3c10 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -pillow==12.2.0 +pillow==12.3.0 sphinx==8.2.3 sphinx-copybutton==0.5.2 sphinx-design==0.6.1 diff --git a/internal/scripts/dev_tools/add_met_config_helper.py b/internal/scripts/dev_tools/add_met_config_helper.py index a6a4d95188..91afa7bcf1 100755 --- a/internal/scripts/dev_tools/add_met_config_helper.py +++ b/internal/scripts/dev_tools/add_met_config_helper.py @@ -9,15 +9,18 @@ import sys import os +from typing import Any try: - from metplus.util.string_manip import get_wrapper_name + from metplus.util.string_manip import get_wrapper_name, camel_to_underscore + from metplus.util.constants import LOWER_TO_WRAPPER_NAME except ImportError: # if metplus package is not installed, find util relative to this script metplus_home = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir) sys.path.insert(0, os.path.abspath(metplus_home)) - from metplus.util.string_manip import get_wrapper_name + from metplus.util.string_manip import get_wrapper_name, camel_to_underscore + from metplus.util.constants import LOWER_TO_WRAPPER_NAME SCRIPT_INFO_TEXT = ( 'This script is intended to help developers add support for setting ' @@ -29,8 +32,16 @@ 'added or replaced in the files. Do not include the dash lines.' ) +WRAPPERS_TO_SKIP = ( + 'CyclonePlotter', + 'Example', + 'GempakToCF', + 'GFDLTracker', + 'Usage', +) + -def print_doc_text(tool_name, input_dict): +def print_doc_text(tool_name, input_dict, skip_met_config=False): """! Format documentation for adding support for a new MET config variable through METplus wrappers. @@ -39,27 +50,38 @@ def print_doc_text(tool_name, input_dict): @param dict_items (optional) list of MET dictionary var items if met_var is a dictionary """ + tool_names = [] + if tool_name == "all": + for wrapper_name in LOWER_TO_WRAPPER_NAME.values(): + if wrapper_name not in WRAPPERS_TO_SKIP and camel_to_underscore(wrapper_name) not in tool_names: + tool_names.append(camel_to_underscore(wrapper_name)) + else: + tool_names = tool_name.split() + _print_script_info_text() - wrapper_camel = get_wrapper_name(tool_name) - if wrapper_camel is None: - print(f'ERROR: Invalid tool name: {tool_name}') - sys.exit(1) + wrapper_camel_list = [] + for tool in tool_names: + wrapper_camel = get_wrapper_name(tool) + if wrapper_camel is None: + print(f'ERROR: Invalid tool name: {tool}') + sys.exit(1) + wrapper_camel_list.append(wrapper_camel) # get info for each variable and store it in a dictionary - met_vars = _get_met_vars(tool_name, input_dict) + met_vars_dict = _get_met_vars(tool_names, wrapper_camel_list, input_dict) - _step_add_wrapper_env_var_keys(met_vars, tool_name) - _step_add_wrapper_config_dict(met_vars, tool_name) + _step_add_wrapper_content(met_vars_dict, skip_met_config) - _step_add_parm_metplus_config(met_vars, wrapper_camel) - _step_add_parm_met_config(met_vars, wrapper_camel) + _step_add_parm_metplus_config(met_vars_dict) + if not skip_met_config: + _step_add_parm_met_config(met_vars_dict) - _step_add_doc_metplus_config(met_vars, wrapper_camel) - _step_add_doc_met_config(met_vars, wrapper_camel) - _step_add_doc_glossary(met_vars, wrapper_camel) - _step_add_unit_tests(tool_name, met_vars) - _step_test_met_tool(wrapper_camel) + _step_add_unit_tests(met_vars_dict, skip_met_config) + _step_test_met_tool(met_vars_dict.keys()) + + _step_add_doc_wrappers(met_vars_dict, skip_met_config) + _step_add_doc_glossary(met_vars_dict, skip_met_config) _print_script_end_text() @@ -77,35 +99,41 @@ def _print_script_end_text(): _print_divider_line(before=False) -def _step_add_wrapper_env_var_keys(met_vars, tool_name): +def _step_add_wrapper_content(met_vars_dict, skip_met_config): _print_divider_line() - print(f'In metplus/wrappers/{tool_name}_wrapper.py\n\n' - f'In the {get_wrapper_name(tool_name)}Wrapper ' - f'class, add the following to the WRAPPER_ENV_VAR_KEYS class ' - f"variable list:\n") - _print_divider_line(char='-', after=False) - for var in met_vars: - print(f" '{var['env_var_name']}',") - _print_divider_line(char='-', before=False) + if skip_met_config: + print("Add handling of the METplus config variable as needed\n") + for _, (tool_name, met_vars) in met_vars_dict.items(): + print(f'In metplus/wrappers/{tool_name}_wrapper.py') + if skip_met_config: + continue -def _step_add_wrapper_config_dict(met_vars, tool_name): - _print_divider_line() - print(f'In metplus/wrappers/{tool_name}_wrapper.py\n') - print(f'In the create_c_dict function for ' - f'{get_wrapper_name(tool_name)}Wrapper, add a ' - 'function call to read the new METplus config variables and save ' - 'the value to be added to the wrapped MET config file.\n') - - _print_divider_line(char='-', after=False) - for var in met_vars: - _print_add_met_config(var) - _print_divider_line(char='-', before=False) + print(f'\nIn the {get_wrapper_name(tool_name)}Wrapper ' + f'class, add the following to the WRAPPER_ENV_VAR_KEYS class ' + f"variable list:") + _print_divider_line(char='-', after=False) + for var in met_vars: + print(f" '{var['env_var_name']}',") + _print_divider_line(char='-', before=False) + + print(f'In the create_c_dict function for ' + f'{get_wrapper_name(tool_name)}Wrapper, add a ' + 'function call to read the new METplus config variables and save ' + 'the value to be added to the wrapped MET config file.') + + _print_divider_line(char='-', after=False) + for var in met_vars: + _print_add_met_config(var) + _print_divider_line(char='-', before=False) + + if skip_met_config: + return - print("where DATA_TYPE can be string, list, int, float, bool, " + print("DATA_TYPE can be string, list, int, float, bool, " "or thresh. Refer to the METplus Contributor's Guide " "Basic Components section to see how to add additional info.\n") - print("Sometimes a function is written to handle MET config dictionary" + print("NOTE: Sometimes a function is written to handle MET config dictionary" " items that are complex and common to many wrappers." " Search for functions that start with handle_ in " "CommandBuilder or other parent class wrappers to see if a " @@ -113,99 +141,110 @@ def _step_add_wrapper_config_dict(met_vars, tool_name): "as an example to write a new one.") -def _step_add_parm_metplus_config(met_vars, wrapper_camel): +def _step_add_parm_metplus_config(met_vars_dict): _print_divider_line() - print(f'In parm/use_cases/met_tool_wrapper/{wrapper_camel}/{wrapper_camel}.conf') - print('\nAdd the new variables commented out in the basic use case') + for wrapper_camel, (tool_name, met_vars) in met_vars_dict.items(): + print(f'In parm/use_cases/met_tool_wrapper/{wrapper_camel}/{wrapper_camel}.conf') + print('\nAdd the new variables commented out in the basic use case') - _print_divider_line(char='-', after=False) + _print_divider_line(char='-', after=False) - for var in met_vars: - for mp_config in var['metplus_config_names']: - print(f'#{mp_config} =') + for var in met_vars: + for mp_config in var['metplus_config_names']: + print(f'#{mp_config} =') - _print_divider_line(char='-', before=False) + _print_divider_line(char='-', before=False) -def _step_add_parm_met_config(met_vars, wrapper_camel): +def _step_add_parm_met_config(met_vars_dict): + _print_divider_line() + for wrapper_camel, (tool_name, met_vars) in met_vars_dict.items(): + var_names = '/'.join([var['name'] for var in met_vars]) + print(f"In parm/met_config/{wrapper_camel}Config_wrapped\n\n" + "IMPORTANT: Compare the default values set for " + f"{var_names} " + "to the version" + f" in share/met/config/{wrapper_camel}Config_default. If " + "they do differ, make sure to add variables to the use case " + "config files so that they produce the same output.\n\n") + + for var in met_vars: + print("REPLACE:") + _print_divider_line(char='-', after=False) + print(f"{var['name']} = ...") + _print_divider_line(char='-', before=False) + print('with:') + _print_divider_line(char='-', after=False) + print(f"//{var['name']} ={' {' if var['dict_items'] else ''}") + print(f"${{{var['env_var_name']}}}") + _print_divider_line(char='-', before=False) + + +def _step_add_doc_wrappers(met_vars_dict, skip_met_config): _print_divider_line() + print("In docs/Users_Guide/wrappers.rst\n") + + for wrapper_camel, (tool_name, met_vars) in met_vars_dict.items(): + print(f"Under {wrapper_camel} => METplus Configuration section, add:") - var_names = '/'.join([var['name'] for var in met_vars]) - print(f"In parm/met_config/{wrapper_camel}Config_wrapped\n\n" - "IMPORTANT: Compare the default values set for " - f"{var_names} " - "to the version" - f" in share/met/config/{wrapper_camel}Config_default. If " - "they do differ, make sure to add variables to the use case " - "config files so that they produce the same output.\n\n") - - for var in met_vars: - print("REPLACE:") - _print_divider_line(char='-', after=False) - print(f"{var['name']} = ...") - _print_divider_line(char='-', before=False) - print('with:') _print_divider_line(char='-', after=False) - print(f"//{var['name']} ={' {' if var['dict_items'] else ''}") - print(f"${{{var['env_var_name']}}}") + for var in met_vars: + for metplus_config_name in var['metplus_config_names']: + print(f'| :term:`{metplus_config_name}`') _print_divider_line(char='-', before=False) + if skip_met_config: + continue -def _step_add_doc_metplus_config(met_vars, wrapper_camel): - _print_divider_line() - print(f"In docs/Users_Guide/wrappers.rst\n\n" - f"Under {wrapper_camel} => METplus Configuration section, add:") - - _print_divider_line(char='-', after=False) - for var in met_vars: - for metplus_config_name in var['metplus_config_names']: - print(f'| :term:`{metplus_config_name}`') - _print_divider_line(char='-', before=False) - - -def _step_add_doc_met_config(met_vars, wrapper_camel): - _print_divider_line() - print(f"In docs/Users_Guide/wrappers.rst\n\n" - f"Under {wrapper_camel} => MET Configuration section, add:") - _print_divider_line(char='-') - for var in met_vars: - _print_met_config_table(var) - _print_divider_line(char='-', before=False) + print(f"Under {wrapper_camel} => MET Configuration section, add:") + _print_divider_line(char='-') + for var in met_vars: + _print_met_config_table(var) + _print_divider_line(char='-', before=False) -def _step_add_doc_glossary(met_vars, wrapper_camel): +def _step_add_doc_glossary(met_vars_dict, skip_met_config): _print_divider_line() print("In docs/Users_Guide/glossary.rst\n\n" "Add the following anywhere in the file:") _print_divider_line(char='-') - for var in met_vars: - _print_glossary_entry(var, wrapper_camel) + for wrapper_camel, (_, met_vars) in met_vars_dict.items(): + for var in met_vars: + _print_glossary_entry(var, wrapper_camel, skip_met_config) _print_divider_line(char='-', before=False) -def _step_add_unit_tests(tool_name, met_vars): +def _step_add_unit_tests(met_vars_dict, skip_met_config): _print_divider_line() - print(f"In internal/tests/pytests/wrappers/{tool_name}/" - f"test_{tool_name}_wrapper.py" - "\n\nAdd the following items to " - "the tests to ensure the new items are set properly. Note: " - "if the tool does not have unit tests to check the handling of " - "MET config variables, you will need to add those tests. See " - "grid_stat/test_grid_stat_wrapper.py for an example. Change " - "VALUE to an appropriate value for the variable.\n") - - _print_divider_line(char='-', after=False) - for var in met_vars: - _print_unit_test(var) - _print_divider_line(char='-', before=False) + if skip_met_config: + print("Add any unit tests as needed") + for _, (tool_name, met_vars) in met_vars_dict.items(): + print(f"In internal/tests/pytests/wrappers/{tool_name}/test_{tool_name}_wrapper.py") + if skip_met_config: + continue -def _step_test_met_tool(wrapper_camel): + print("\nAdd the following items to " + "the tests to ensure the new items are set properly. Note: " + "if the tool does not have unit tests to check the handling of " + "MET config variables, you will need to add those tests. See " + "grid_stat/test_grid_stat_wrapper.py for an example. Change " + "VALUE to an appropriate value for the variable.\n") + + _print_divider_line(char='-', after=False) + for var in met_vars: + _print_unit_test(var) + _print_divider_line(char='-', before=False) + + +def _step_test_met_tool(wrapper_camel_list): # add note to test setting a valid value in the basic use case config file # to ensure that it is formatted properly when read by the MET tool _print_divider_line() - print(f"In parm/use_cases/met_tool_wrapper/{wrapper_camel}" - "\n\nVerify that the new METplus configuration variable(s) " + for wrapper_camel in wrapper_camel_list: + print(f"In parm/use_cases/met_tool_wrapper/{wrapper_camel}") + + print("\nVerify that the new METplus configuration variable(s) " "will be formatted properly when read by the MET tool by " "setting the variable(s) in the basic use case config files " "to a valid value " @@ -214,38 +253,47 @@ def _step_test_met_tool(wrapper_camel): "after you have confirmed this step.") -def _get_met_vars(tool_name, input_dict): - met_vars = [] - for var_name, dict_list in input_dict.items(): - metplus_var = f"{tool_name.upper()}_{var_name.upper().replace('.', '_')}" - env_var_name = f'METPLUS_{var_name.upper()}' - met_var = {'name': var_name, 'dict_items': dict_list, - 'metplus_config_names': [], 'met_config_names': []} - if not dict_list: - met_var['env_var_name'] = env_var_name - met_var['metplus_config_names'].append(metplus_var) - met_var['met_config_names'].append(var_name) - else: - met_var['env_var_name'] = f'{env_var_name}_DICT' - for item_name in dict_list: - metplus_config = f"{metplus_var}_{item_name.upper().replace('.', '_')}" - met_config = f"{var_name}.{item_name}" - met_var['metplus_config_names'].append(metplus_config) - met_var['met_config_names'].append(met_config) - - met_vars.append(met_var) - +def _get_met_vars(tool_names, wrapper_camel_list, input_dict): _print_divider_line() print('Generating instructions for adding support for:\n') - print(f"Wrapper: {get_wrapper_name(tool_name)}\n") + + met_vars_dict = {} + for tool_name, wrapper_camel in zip(tool_names, wrapper_camel_list): + met_vars = [] + for var_name, dict_list in input_dict.items(): + metplus_var = f"{tool_name.upper()}_{var_name.upper().replace('.', '_')}" + env_var_name = f'METPLUS_{var_name.upper()}' + met_var = {'name': var_name, 'dict_items': dict_list, + 'metplus_config_names': [], 'met_config_names': []} + if not dict_list: + met_var['env_var_name'] = env_var_name + met_var['metplus_config_names'].append(metplus_var) + met_var['met_config_names'].append(var_name) + else: + met_var['env_var_name'] = f'{env_var_name}_DICT' + for item_name in dict_list: + metplus_config = f"{metplus_var}_{item_name.upper().replace('.', '_')}" + met_config = f"{var_name}.{item_name}" + met_var['metplus_config_names'].append(metplus_config) + met_var['met_config_names'].append(met_config) + + met_vars.append(met_var) + + met_vars_dict[wrapper_camel] = (tool_name, met_vars) + + _print_met_vars(met_vars, tool_name) + + return met_vars_dict + + +def _print_met_vars(met_vars: list[Any], tool_name): + print(f"Wrapper: {get_wrapper_name(tool_name)}") for index, var in enumerate(met_vars, 1): - print(f"MET Variable {index}: {var['name']}") + print(f" MET Variable {index}: {var['name']}") if var['dict_items']: - print(f" Dictionary Items: {', '.join(var['dict_items'])}") + print(f" Dictionary Items: {', '.join(var['dict_items'])}") print() - return met_vars - def _print_divider_line(char='=', count=80, before=True, after=True): value = char * count @@ -290,14 +338,19 @@ def _print_met_config_table(var): print(list_table_text) -def _print_glossary_entry(var, wrapper_camel): +def _print_glossary_entry(var, wrapper_camel, skip_met_config): metplus_names = var['metplus_config_names'] met_names = var['met_config_names'] for metplus_config_name, met_config_name in zip(metplus_names, met_names): + variable_info = " REPLACE ME" + if not skip_met_config: + variable_info = ( + f" Specify the value for '{met_config_name}' " + f"in the MET configuration file" + ) + glossary_entry = ( - f" {metplus_config_name}\n" - f" Specify the value for '{met_config_name}' " - f"in the MET configuration file for {wrapper_camel}.\n\n" + f" {metplus_config_name}\n{variable_info} for {wrapper_camel}.\n\n" f" | *Used by:* {wrapper_camel}" ) print(f'{glossary_entry}\n') @@ -364,34 +417,54 @@ def _get_output_item(dict_items, met_config_name): def doc_util_usage(): """! Print usage statement for script """ + file = os.path.basename(__file__) print( 'Usage:\n' - f'{__file__} " []" ' - '" []"\n' - f"\nExample: {__file__} grid_stat output_prefix " + f'{file} " []" ' + '" []" [--skip_met_config]\n' + f"\nExample: {file} grid_stat output_prefix " "\n (simple variable named output_prefix)\n" - f'\nExample: {__file__} grid_stat "output_flag fho ctc mctc" ' + f'\nExample: {file} grid_stat "output_flag fho ctc mctc" ' '\n (dictionary named output_flag containing fho, ctc, and mctc)\n' - f'\nExample: {__file__} grid_stat "output_flag fho ctc mctc" ' + f'\nExample: {file} grid_stat "output_flag fho ctc mctc" ' 'output_prefix \n (both of the variables from the previous ' 'examples)\n' - f'\nExample: {__file__} point_stat "topo_mask interp.method interp.width" ' + f'\nExample: {file} point_stat "topo_mask interp.method interp.width" ' '\n (dictionary named topo_mask containing dictionary interp containing method and width)\n' + f"\nExample: {file} \"grid_stat point_stat\" output_prefix " + "\n (variable named output_prefix for multiple wrappers)\n" + f"\nExample: {file} all skip_warn_output_overwrite --skip_met_config" + "\n (variable named skip_warn_output_overwrite for all wrappers, skipping MET config steps)\n" ) if __name__ == "__main__": - # sys.argv[1] is MET tool name, i.e. grid_stat - # sys.argv[2+] is MET variable name, i.e. output_flag or a MET variable - # name followed by a list of MET dictionary var items separated by spaces - if len(sys.argv) < 3: - doc_util_usage() - sys.exit(1) - - TOOL_NAME = sys.argv[1] - INPUT_DICT = {} - for arg in sys.argv[2:]: + import argparse + + + # custom ArgumentParser for custom usage statement function + class CustomParser(argparse.ArgumentParser): + def error(self, message): + # Override standard error handling to print usage statement and exit + doc_util_usage() + sys.exit(1) + + parser = CustomParser(add_help=False) + + # Add optional boolean flag + parser.add_argument( + "--skip_met_config", + action="store_true", + help="Skip steps specific to MET config variables", + ) + + parser.add_argument("TOOL_NAME") + parser.add_argument("raw_vars", nargs="+") + args = parser.parse_args() + + variable_info = {} + for arg in args.raw_vars: variable_name, *dictionary_items = arg.split() - INPUT_DICT[variable_name] = dictionary_items + variable_info[variable_name] = dictionary_items - print_doc_text(TOOL_NAME, INPUT_DICT) + print_doc_text(args.TOOL_NAME, variable_info, skip_met_config=args.skip_met_config) diff --git a/internal/tests/pytests/conftest.py b/internal/tests/pytests/conftest.py index 480bbfb7ed..bb69c00144 100644 --- a/internal/tests/pytests/conftest.py +++ b/internal/tests/pytests/conftest.py @@ -341,3 +341,21 @@ def run_all_and_check_missing_run_error(wrapper, missing, run, errors): assert wrapper.errors == errors return run_all_and_check_missing_run_error + +@pytest.fixture(scope="module") +def check_warn_output_overwrite(): + def check_warn_output_overwrite_wrapper(wrapper, expected_warning, expected_path): + if expected_warning: + # Verify that a warning was logged + assert wrapper.logger.warning.called + + # Extract and verify the warning message content + # Using the same pattern as existing tests in the file + last_msg = wrapper.logger.warning.call_args_list[-1][0][0] + assert "Output has already been written" in last_msg + assert expected_path in last_msg + else: + # Verify that no warning was logged for new files + assert not wrapper.logger.warning.called + + return check_warn_output_overwrite_wrapper diff --git a/internal/tests/pytests/pytest.ini b/internal/tests/pytests/pytest.ini deleted file mode 100644 index 2851d20601..0000000000 --- a/internal/tests/pytests/pytest.ini +++ /dev/null @@ -1,12 +0,0 @@ -[pytest] -markers = - run_metplus: custom marker for testing run_metplus.py script - util: custom marker for testing metplus/util logic - wrapper_a: custom marker for testing metplus/wrapper logic - A group - wrapper_b: custom marker for testing metplus/wrapper logic - B group - wrapper_c: custom marker for testing metplus/wrapper logic - C group - wrapper_d: custom marker for testing metplus/wrapper logic - D group - wrapper: custom marker for testing metplus/wrapper logic - all others - long: custom marker for tests that take a long time to run - plotting: custom marker for tests that involve plotting - diff: custom marker for diff util tests that require additional packages diff --git a/internal/tests/pytests/requirements_3.12.txt b/internal/tests/pytests/requirements_3.12.txt index 4c5a871dfd..76f895081a 100644 --- a/internal/tests/pytests/requirements_3.12.txt +++ b/internal/tests/pytests/requirements_3.12.txt @@ -1,7 +1,7 @@ netCDF4>=1.7.2 pandas>=2.2.3 pdf2image>=1.17.0 -pillow>=12.2.0 +pillow>=12.3.0 pytest>=9.0.3 pytest-cov>=6.0.0 python-dateutil>=2.9.0.post0 diff --git a/internal/tests/pytests/requirements_3.14.txt b/internal/tests/pytests/requirements_3.14.txt index 9b7f850d65..0d75e79cfb 100644 --- a/internal/tests/pytests/requirements_3.14.txt +++ b/internal/tests/pytests/requirements_3.14.txt @@ -1,7 +1,7 @@ netCDF4>=1.7.4 pandas>=3.0.2 pdf2image>=1.17.0 -pillow>=12.2.0 +pillow>=12.3.0 pytest>=9.0.3 pytest-cov>=7.1.0 python-dateutil>=2.9.0.post0 diff --git a/internal/tests/pytests/util/run_util/test_run_util.py b/internal/tests/pytests/util/run_util/test_run_util.py index f409a82fbe..a2bb97bb50 100644 --- a/internal/tests/pytests/util/run_util/test_run_util.py +++ b/internal/tests/pytests/util/run_util/test_run_util.py @@ -50,6 +50,7 @@ 'METPLUS_VERSION', 'ALLOW_MISSING_INPUTS', 'INPUT_THRESH', + 'EXIT_IF_WARN', ] def remove_output_base(config): @@ -473,3 +474,92 @@ def test_post_run_cleanup_errors(post_run_config): post_run_config, ['Check the log file for more information: /log/file.log'] ) assert mock.call(err_msg) in post_run_config.logger.error.call_args_list + +success_settings = { + 'PROCESS_LIST': 'GridStat, PCPCombine', + 'LOOP_BY': 'INIT', + 'INIT_TIME_FMT': '%Y%m%d%H', + 'INIT_BEG': '2005080700', + 'INIT_END': '2005080700', + 'INIT_INCREMENT': '12H', + 'LEAD_SEQ': '12H', + 'FCST_GRID_STAT_INPUT_DIR': '{INPUT_BASE}/met_test/data/sample_fcst', + 'FCST_GRID_STAT_INPUT_TEMPLATE': '{init?fmt=%Y%m%d%H}/wrfprs_ruc13_{lead?fmt=%HH}.tm00_G212', + 'OBS_GRID_STAT_INPUT_DIR': '{INPUT_BASE}/met_test/new', + 'OBS_GRID_STAT_INPUT_TEMPLATE': 'ST2ml{valid?fmt=%Y%m%d%H}_A03h.nc', + 'GRID_STAT_OUTPUT_TEMPLATE': '{init?fmt=%Y%m%d%H}', + 'FCST_PCP_COMBINE_RUN': True, + 'FCST_PCP_COMBINE_METHOD': 'SUM', + 'FCST_PCP_COMBINE_INPUT_DIR': '/some/dir', + 'FCST_PCP_COMBINE_INPUT_TEMPLATE': '{init?fmt=%Y%m%d}/file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc', + 'FCST_PCP_COMBINE_OUTPUT_TEMPLATE': 'file.{init?fmt=%Y%m%d%H}.f{lead?fmt=%HHH}.nc', + 'BOTH_VAR1_NAME': 'APCP', + 'BOTH_VAR1_LEVELS': 'A03', + 'BOTH_VAR1_THRESH': 'gt1, gt5', + 'INPUT_MUST_EXIST': 'False', + 'DO_NOT_RUN_EXE': 'True', +} + +@pytest.mark.parametrize( + 'exit_on_warn, config_overrides, expected_init_errors, expected_run_errors', + [ + # successful run without warnings or errors + (False, {}, 0, 0), + (True, {}, 0, 0), + # warning in init - GridStat (1st process) + (False, {'GRID_STAT_FCST_INPUT_TEMPLATE': 'template.nc'}, 0, 0), + (True, {'GRID_STAT_FCST_INPUT_TEMPLATE': 'template.nc'}, 1, 0), + # warning in init - PCPCombine (2nd process) + (False, {'PCP_COMBINE_FCST_INPUT_TEMPLATE': 'template.nc'}, 0, 0), + (True, {'PCP_COMBINE_FCST_INPUT_TEMPLATE': 'template.nc'}, 1, 0), + # warning in init - both processes + (False, {'GRID_STAT_FCST_INPUT_TEMPLATE': 'template.nc','PCP_COMBINE_FCST_INPUT_TEMPLATE': 'template.nc'}, 0, 0), + (True, {'GRID_STAT_FCST_INPUT_TEMPLATE': 'template.nc', 'PCP_COMBINE_FCST_INPUT_TEMPLATE': 'template.nc'}, 1, 0), + # warning in run - GridStat (1st process) + (False, {'GRID_STAT_WARN_IF_OUTPUT_EXISTS': True}, 0, 0), + (True, {'GRID_STAT_WARN_IF_OUTPUT_EXISTS': True}, 0, 1), + # warning in run - PCPCombine (2nd process) + (False, {'PCP_COMBINE_WARN_IF_OUTPUT_EXISTS': True}, 0, 0), + (True, {'PCP_COMBINE_WARN_IF_OUTPUT_EXISTS': True}, 0, 1), + # warning in run - both processes + (False, {'GRID_STAT_WARN_IF_OUTPUT_EXISTS': True, 'PCP_COMBINE_WARN_IF_OUTPUT_EXISTS': True}, 0, 0), + (True, {'GRID_STAT_WARN_IF_OUTPUT_EXISTS': True, 'PCP_COMBINE_WARN_IF_OUTPUT_EXISTS': True}, 0, 1), + ], +) +@pytest.mark.util +def test_exit_on_warn(metplus_config_files, tmp_path_factory, make_dummy_empty, exit_on_warn, config_overrides, expected_init_errors, expected_run_errors): + # create fake output directories to test warnings when output already exists + fake_output_dir = tmp_path_factory.mktemp("output_dir") + make_dummy_empty(fake_output_dir, 'grid_stat/2005080700/grid_stat_120000L_20050807_120000V.stat') + make_dummy_empty(fake_output_dir, 'pcp_combine/file.2005080700.f012.nc') + + initial_overrides = [ + f'config.GRID_STAT_OUTPUT_DIR={fake_output_dir}/grid_stat', + f'config.FCST_PCP_COMBINE_OUTPUT_DIR={fake_output_dir}/pcp_combine', + ] + + # set EXIT_IF_WARN before config init because it is currently read in config_metplus.launch + # after refactor to support wrapper-specific EXIT_IF_WARN, this can be set in config_overrides instead + + if exit_on_warn: + initial_overrides.append('config.EXIT_IF_WARN=True') + config = metplus_config_files(initial_overrides) + + for key, value in success_settings.items(): + config.set('config', key, value) + + for key, value in config_overrides.items(): + config.set('config', key, value) + + process_list = ru.get_process_list(config) + processes = ru._load_all_wrappers(config, process_list) + + # initialization warning when EXIT_IF_WARN=True cause processes to be None + if expected_init_errors: + assert processes is None + else: + assert processes + + ru._run_processes(processes) + total_errors = ru._get_total_errors_and_log_counts(processes, config.logger) + assert total_errors == expected_run_errors diff --git a/internal/tests/pytests/wrappers/command_builder/test_command_builder.py b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py index 13613b8807..9f366a78c1 100644 --- a/internal/tests/pytests/wrappers/command_builder/test_command_builder.py +++ b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py @@ -1255,3 +1255,51 @@ def test_errors_and_defaults(metplus_config): assert actual is None assert _in_last_err('bar', cb.logger) + +TEST_SEARCH_PATH = ( + '/output/path/precip24_2026070112.nc', + '/output/path/precip24_2026070118.nc', + '/output/path/grid_stat_120000L_20050807_120000V*', + '/output/path/grid_stat_WRF_APCP_vs_MC_PCP_APCP_03_120000L_20050807_120000V*', +) +@pytest.mark.parametrize( + 'output_paths, expected_warnings', [ + # Case 1: First time writing a file (no warning) + ([TEST_SEARCH_PATH[0]], [False]), + + # Case 2: Writing the same file twice (warning on second call) + ([TEST_SEARCH_PATH[0], TEST_SEARCH_PATH[0]], [False, True]), + + # Case 3: Multiple different files (no warnings) + ([TEST_SEARCH_PATH[0], TEST_SEARCH_PATH[1]], [False, False]), + + # Case 4: Search format/wildcard path (warning on repeat) + ([TEST_SEARCH_PATH[2], TEST_SEARCH_PATH[2]], [False, True]), + + # Case 5: Search different wildcard paths + ([TEST_SEARCH_PATH[2], TEST_SEARCH_PATH[3]], [False, False]), + ] +) +@pytest.mark.wrapper +def test_check_if_output_has_been_written(metplus_config, check_warn_output_overwrite, + output_paths, expected_warnings): + """!Test that CommandBuilder tracks written files and warns on overwrites. + """ + # Create a CommandBuilder instance using the existing metplus_config fixture + cb = CommandBuilder(metplus_config) + + # Iterate through the sequence of file write attempts + for path, should_warn in zip(output_paths, expected_warnings): + # Reset the mock logger for each path to isolate the check + cb.logger.warning.reset_mock() + + # Call the new logic + cb._check_if_output_has_been_written(path) + + check_warn_output_overwrite(cb, should_warn, path) + + # Final check: verify all unique paths were added to the tracking list + unique_paths = list(dict.fromkeys(output_paths)) + assert len(cb.output_written) == len(unique_paths) + for path in unique_paths: + assert path in cb.output_written diff --git a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py index 3eb4200a20..dcd007f2cd 100644 --- a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py @@ -951,3 +951,48 @@ def test_get_config_file(metplus_config): config.set('config', 'GRID_STAT_CONFIG_FILE', fake_config_name) wrapper = GridStatWrapper(config) assert wrapper.c_dict['CONFIG_FILE'] == fake_config_name + +@pytest.mark.parametrize( + 'config_overrides, expected_warning, expected_path', [ + ({}, False, 'grid_stat_120000L_20050808_000000V*'), + # run once per field without unique prefix + ({'GRID_STAT_ONCE_PER_FIELD': True}, + True, 'grid_stat_120000L_20050808_000000V*'), + # run once per field with unique prefix + ({'GRID_STAT_ONCE_PER_FIELD': True, 'GRID_STAT_OUTPUT_PREFIX': '{fcst_name}_vs_{obs_name}'}, + False, 'grid_stat_ABCD_vs_OBCD_120000L_20050808_000000V*'), + ] +) +@pytest.mark.wrapper_b +def test_grid_stat_warn_on_overwrite(metplus_config, check_warn_output_overwrite, + config_overrides, expected_warning, expected_path): + config = metplus_config + set_minimum_config_settings(config) + + # set additional settings + config.set('config', 'FCST_VAR2_NAME', 'FBCD') + config.set('config', 'FCST_VAR2_LEVELS', fcst_level) + config.set('config', 'OBS_VAR2_NAME', 'OBCD') + config.set('config', 'OBS_VAR2_LEVELS', obs_level) + config.set('config', 'BOTH_VAR2_THRESH', both_thresh) + config.set('config', 'GRID_STAT_OUTPUT_TEMPLATE', '') + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + wrapper = GridStatWrapper(config) + + # create one of the expected output files to ensure that the existence + # of this file does not affect the logging of the warning message + output_dir = wrapper.config.getdir('GRID_STAT_OUTPUT_DIR') + os.makedirs(output_dir, exist_ok=True) + for output_file in ('grid_stat_120000L_20050808_000000V.stat', 'grid_stat_FBCD_vs_OBCD_120000L_20050808_000000V.stat'): + output_path = os.path.join(output_dir, output_file) + with open(output_path, 'w') as file_handle: + file_handle.write('') + assert os.path.exists(output_path) + + wrapper.run_all_times() + + check_warn_output_overwrite(wrapper, expected_warning, expected_path) diff --git a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index 87ccbc3f77..f3f4a8d0d4 100644 --- a/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -6,6 +6,7 @@ from metplus.wrappers.series_analysis_wrapper import SeriesAnalysisWrapper from metplus.wrappers import series_analysis_wrapper as saw +from metplus.util.run_util import get_process_list, _load_all_wrappers, _run_processes fcst_dir = '/some/fcst/dir' obs_dir = '/some/obs/dir' @@ -95,7 +96,7 @@ def set_minimum_config_settings(config): config.set('config', 'SERIES_ANALYSIS_OUTPUT_DIR', '{OUTPUT_BASE}/SeriesAnalysis/output') config.set('config', 'SERIES_ANALYSIS_OUTPUT_TEMPLATE', - '{init?fmt=%Y%m%d%H}') + 'sa_{init?fmt=%Y%m%d%H}.nc') config.set('config', 'FCST_VAR1_NAME', fcst_name) config.set('config', 'FCST_VAR1_LEVELS', fcst_level) @@ -647,7 +648,7 @@ def test_series_analysis_single_field(metplus_config, config_overrides, expected_cmds = [] for run_time in run_times: - cmd = (f"{app_path} {file_args} -out {out_dir}/{extra_args}" + cmd = (f"{app_path} {file_args} -out {out_dir}/sa_.nc{extra_args}" f"-config {config_file} {verbosity}") expected_cmds.append(cmd.replace('', run_time)) @@ -1304,3 +1305,75 @@ def test_get_fcst_and_obs_path(metplus_config, actual = wrapper._get_fcst_and_obs_path(time_info, '*', lead_group) assert actual == expected +@pytest.mark.parametrize( + 'config_overrides, expected_warning, expected_path', [ + # no warning + ({}, False, None), + # 2 fields/vars, so output is overwritten + ({'FCST_VAR2_NAME': 'FBCD', 'FCST_VAR2_LEVELS': fcst_level, + 'OBS_VAR2_NAME': 'OBCD', 'OBS_VAR2_LEVELS': obs_level,}, + True, '2005080712.nc'), + # 2 fields/vars, output template includes field, so output is not overwritten + ({'FCST_VAR2_NAME': 'FBCD', 'FCST_VAR2_LEVELS': fcst_level, + 'OBS_VAR2_NAME': 'OBCD', 'OBS_VAR2_LEVELS': obs_level, + 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': 'sa_{init?fmt=%Y%m%d%H}_{fcst_name}.nc',}, + False, None), + # output template does not include init + ({'SERIES_ANALYSIS_OUTPUT_TEMPLATE': 'sa_F{lead?fmt=%H}.nc'}, True, 'sa_Fall.nc'), + # run once per lead, template does not include lead + ({'SERIES_ANALYSIS_RUNTIME_FREQ': 'RUN_ONCE_PER_LEAD', 'LEAD_SEQ': '12H, 18H'}, True, 'sa_all.nc'), + # run once per lead, template includes lead + ({'SERIES_ANALYSIS_OUTPUT_TEMPLATE': 'sa_F{lead?fmt=%H}.nc', + 'SERIES_ANALYSIS_RUNTIME_FREQ': 'RUN_ONCE_PER_LEAD', 'LEAD_SEQ': '12H, 18H'}, + False, None), + + # custom loop list causes multiple duplicate runs + ({'SERIES_ANALYSIS_CUSTOM_LOOP_LIST': 'A,B'}, True, '2005080712.nc'), + # custom loop list with custom string in output template + ({'SERIES_ANALYSIS_CUSTOM_LOOP_LIST': 'A,B', + 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': 'sa_{init?fmt=%Y%m%d%H}_{custom}.nc'}, False, None), + # custom loop list with custom string in output template but no init in template + ({'SERIES_ANALYSIS_CUSTOM_LOOP_LIST': 'A,B', + 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': 'sa_{custom}.nc'}, True, 'sa_B.nc'), + + # multiple instances cause multiple duplicate runs + ({'PROCESS_LIST': 'SeriesAnalysis(A), SeriesAnalysis(B)'}, True, '2005080712.nc'), + # multiple instances with instance string in output template + ({'PROCESS_LIST': 'SeriesAnalysis(A), SeriesAnalysis(B)', + 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': 'sa_{init?fmt=%Y%m%d%H}_{instance}.nc'}, False, None), + # multiple instances with instance string in output template but no init in template + ({'PROCESS_LIST': 'SeriesAnalysis(A), SeriesAnalysis(B)', + 'SERIES_ANALYSIS_OUTPUT_TEMPLATE': 'sa_{instance}.nc'}, True, 'sa_B.nc'), + + ] +) +@pytest.mark.wrapper_b +def test_series_analysis_warn_on_overwrite(metplus_config, check_warn_output_overwrite, + config_overrides, expected_warning, expected_path): + config = metplus_config + set_minimum_config_settings(config) + + # set config variable overrides + for key, value in config_overrides.items(): + config.set('config', key, value) + + process_list = get_process_list(config) + processes = _load_all_wrappers(config, process_list) + assert processes + + for wrapper in processes: + + # create one of the expected output files to ensure that the existence + # of this file does not affect the logging of the warning message + output_dir = wrapper.config.getdir('SERIES_ANALYSIS_OUTPUT_DIR') + os.makedirs(output_dir, exist_ok=True) + for output_file in ('2005080700.nc', '2005080712.nc'): + output_path = os.path.join(output_dir, output_file) + with open(output_path, 'w') as file_handle: + file_handle.write('') + assert os.path.exists(output_path) + + _run_processes(processes) + + for wrapper in processes: + check_warn_output_overwrite(wrapper, expected_warning, expected_path) diff --git a/metplus/util/__init__.py b/metplus/util/__init__.py index edd8a2a0d6..5ec9dbcd85 100644 --- a/metplus/util/__init__.py +++ b/metplus/util/__init__.py @@ -1,5 +1,6 @@ from .metplus_check import * from .constants import * +from .exceptions import * from .string_manip import * from .system_util import * from .time_util import * diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 667ad634f9..5ca8805ebb 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -21,6 +21,7 @@ import uuid from .constants import RUNTIME_CONFS, MISSING_DATA_VALUE +from .exceptions import MPWarningError from .string_template_substitution import do_string_sub from .string_manip import getlist, remove_quotes from .string_manip import validate_thresholds, find_indices_in_config_section @@ -87,6 +88,16 @@ # set all loggers to use UTC logging.Formatter.converter = time.gmtime +# logging handler for exiting on warnings if EXIT_IF_WARN is True +class ExitOnWarningHandler(logging.Handler): + def __init__(self): + super().__init__(level=logging.WARNING) + + def emit(self, record): + if record.levelno == logging.WARNING: + self.format(record) + raise MPWarningError() + def setup(args, base_confs=None): """!Setup the METplusConfig by reading in default configurations and any @@ -272,6 +283,11 @@ def launch(config_list): with open(final_conf, 'wt') as file_handle: config.write(file_handle) + # add warning handler to exit when a warning occurs if EXIT_IF_WARN is True + if config.getbool('config', 'EXIT_IF_WARN', False): + warning_handler = ExitOnWarningHandler() + config.logger.addHandler(warning_handler) + return config diff --git a/metplus/util/constants.py b/metplus/util/constants.py index cd1c003964..98db6dc5a9 100644 --- a/metplus/util/constants.py +++ b/metplus/util/constants.py @@ -192,3 +192,13 @@ def debug(self, message): pass # skip logging def info(self, message): pass # skip logging def warning(self, message): pass # skip logging def error(self, message): pass # skip logging + +# config variable names that involve warnings and their default values +# generic and wrapper-specific versions are supported +# If explicit apps are listed, the wrapper-specific version is only available +# for that wrapper. +WARNING_CONFIGS = { + 'WARN_IF_DUPLICATE_OUTPUT': (True, ()), + 'WARN_IF_OUTPUT_EXISTS': (False, ()), + 'WARN_IF_ENSEMBLE_IS_MISSING': (True, ('ensemble_stat', 'gen_ens_prod')), +} diff --git a/metplus/util/exceptions.py b/metplus/util/exceptions.py new file mode 100644 index 0000000000..9feb58588d --- /dev/null +++ b/metplus/util/exceptions.py @@ -0,0 +1,9 @@ +class MPWarningError(Exception): + """Raised when a warning is encountered and EXIT_IF_WARN is True""" + + def __init__(self): + # Pass a clear error message to the base Exception class + super().__init__( + "Encountered a warning when EXIT_IF_WARN is set." + " Ending METplus run now." + ) diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index 17a1a16290..81542e029e 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -16,6 +16,7 @@ from .. import get_metplus_version from .config_metplus import setup from . import get_wrapper_instance +from .exceptions import MPWarningError class RunArgs(NamedTuple): @@ -201,6 +202,7 @@ def run_metplus(config): # Use config object to get the list of processes to call process_list = get_process_list(config) + # catch any unexpected exceptions in the code and error/exit try: # if Usage is in process list, run it and exit if 'Usage' in [p[0] for p in process_list]: @@ -218,19 +220,15 @@ def run_metplus(config): if init_errors: return init_errors - all_commands = [] - for process in processes: - new_commands = process.run_all_times() - if new_commands: - all_commands.extend(new_commands) + all_commands = _run_processes(processes) # write out all commands and environment variables to file write_all_commands(all_commands, config) # compute total number of errors that occurred and output results return _get_total_errors_and_log_counts(processes, config.logger) - except Exception: - config.logger.exception("Fatal error occurred") + except Exception as err: + config.logger.exception(f"Fatal error occurred: {err}") config.logger.info("Check the log file for more information: " f"{get_logfile_info(config)}") return 1 @@ -285,6 +283,38 @@ def _check_wrapper_init_errors(processes, logger=None): return errors +def _run_processes(processes): + """!Loop over the list of processes and call run_all_times on each. + Track all commands that were run and output file regexes that were written. + Pass output file regexes to next process in the list so it can check if + duplicate paths are being overwritten. + + @param processes list of processes to run + @returns list of tuples with all commands that were run and + the env vars that were set for them + """ + # track output written by each process and add to next wrapper + # to ensure the same output is not being written by multiple processes + output_written = [] + + all_commands = [] + for process in processes: + process.output_written.extend(output_written) + try: + new_commands = process.run_all_times() + + # log error count if warning-on-error occurs, add any commands run + # before warning, then exit without running any more processes + except MPWarningError as err: + process.log_error(err) + all_commands.extend(process.all_commands) + return all_commands + + output_written.extend(process.output_written) + if new_commands: + all_commands.extend(new_commands) + + return all_commands def _get_total_errors_and_log_counts(processes, logger=None): total_errors = 0 diff --git a/metplus/util/wrapper_init.py b/metplus/util/wrapper_init.py index df87279e84..4a74cbb6f4 100644 --- a/metplus/util/wrapper_init.py +++ b/metplus/util/wrapper_init.py @@ -1,6 +1,7 @@ from importlib import import_module from . import camel_to_underscore +from .exceptions import MPWarningError def get_wrapper_class(config, process): @@ -17,11 +18,14 @@ def get_wrapper_class(config, process): module = import_module(package_name) wrapper_class = getattr(module, f"{process}Wrapper") except AttributeError as err: - config.logger.error(f"There was a problem loading {process} wrapper: {err}") + config.logger.error( + f"There was a problem loading {process} wrapper: {err}" + ) return None except ModuleNotFoundError: - config.logger.error(f"Could not load {process} wrapper. " - "Wrapper may have been disabled.") + config.logger.error( + f"Could not load {process} wrapper. Wrapper may have been disabled." + ) return None return wrapper_class @@ -40,4 +44,11 @@ def get_wrapper_instance(config, process, instance=None): wrapper_class = get_wrapper_class(config, process) if wrapper_class is None: return None - return wrapper_class(config, instance=instance) + + try: + wrapper_instance = wrapper_class(config, instance=instance) + except MPWarningError as err: + config.logger.error(f"{err} Could not initialize {process} wrapper.") + wrapper_instance = None + + return wrapper_instance diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 6994997af9..754cf28284 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -14,9 +14,11 @@ from datetime import datetime from abc import ABCMeta from inspect import getframeinfo, stack +from typing import Any from ..util.constants import PYTHON_EMBEDDING_TYPES, COMPRESSION_EXTENSIONS from ..util.constants import MULTIPLE_INPUT_WRAPPERS, TIME_OFFSET_WARNING_WRAPPERS +from ..util.constants import WARNING_CONFIGS from ..util import getlist, preprocess_file from ..util import do_string_sub, ti_calculate, get_seconds_from_string from ..util import shift_time_seconds, seconds_to_met_time @@ -64,6 +66,7 @@ def __init__(self, config, instance=None): self.outfile = "" self.param = "" self.all_commands = [] + self.output_written = [] # set app name to empty string if not set by wrapper # needed to create instance of parent wrapper for unit tests @@ -169,9 +172,10 @@ def check_for_unused_env_vars(self): for env_var_key in self.WRAPPER_ENV_VAR_KEYS: env_var_string = f"${{{env_var_key}}}" if env_var_string not in content: - self.logger.warning(f"Environment variable {env_var_string} " - "is not utilized in MET config file: " - f"{config_file}") + self.logger.warning( + f"Environment variable {env_var_string} is not utilized in " + f"MET config file: {config_file}" + ) def create_c_dict(self): c_dict = dict() @@ -218,6 +222,18 @@ def create_c_dict(self): 'DO_NOT_RUN_EXE', False) + # read generic or wrapper-specific config for + # warning skips and exit on warning (all default to False) + for name, (default, app_list) in WARNING_CONFIGS.items(): + # skip reading warning config if only certain apps use it, + # and this app is not one of them + if app_list and self.app_name not in app_list: + continue + + c_dict[name] = self.get_wrapper_or_generic_config( + name, var_type='bool', default=default + ) + return c_dict def clear(self, clear_input_files=True): @@ -491,10 +507,7 @@ def find_obs_offset(self, time_info, mandatory=True, return_list=False, allow_di # error should already be reported self.logger.error(log_message) else: - if self.c_dict.get('SUPPRESS_WARNINGS', False): - self.logger.debug(log_message) - else: - self.logger.warning(log_message) + self.log_warn_or_debug(log_message, not self.c_dict.get('SUPPRESS_WARNINGS', False)) return None, time_info @@ -627,10 +640,7 @@ def _log_message_dynamic_level(self, msg, mandatory): """ # warn instead of error if it is not mandatory to find files if self._is_optional_input(mandatory): - if self.c_dict.get('SUPPRESS_WARNINGS', False): - self.logger.debug(msg) - else: - self.logger.warning(msg) + self.log_warn_or_debug(msg, not self.c_dict.get('SUPPRESS_WARNINGS', False)) else: self.log_error(msg) @@ -880,8 +890,17 @@ def find_input_files_ensemble(self, time_info, fill_missing=True): self.infiles.append(file_list_path) return True + # suppress warnings if requested + suppress_warnings = self.c_dict.get('SUPPRESS_WARNINGS', False) + if not self.c_dict.get('WARN_IF_ENSEMBLE_IS_MISSING', True): + self.c_dict['SUPPRESS_WARNINGS'] = True + # get list of ensemble files to process input_files = self.find_model(time_info, return_list=True, mandatory=False) + + # restore suppress warnings setting + self.c_dict['SUPPRESS_WARNINGS'] = suppress_warnings + if not input_files: msg = "Could not find any input files" self._log_message_dynamic_level(msg, True) @@ -893,8 +912,9 @@ def find_input_files_ensemble(self, time_info, fill_missing=True): # check if control file is found in ensemble list if ctrl_file in input_files: # warn and remove control file if found - self.logger.warning(f"Control file found in ensemble list: " - f"{ctrl_file}. Removing from list.") + self.logger.warning( + f"Control file found in ensemble list: {ctrl_file}. Removing from list." + ) input_files.remove(ctrl_file) # compare number of files found to expected number of members @@ -944,14 +964,17 @@ def _check_expected_ensembles(self, input_files): # if fewer files found than expected, warn and add fake files if num_found < num_expected: - self.logger.warning( + self.log_warn_or_debug( f"Found fewer files than expected. " - f"Found {num_found} expected {num_expected}" + f"Found {num_found} expected {num_expected}", + self.c_dict.get('WARN_IF_ENSEMBLE_IS_MISSING', True) ) # add fake files to list for ens_thresh checking diff = num_expected - num_found - self.logger.warning(f'Adding {diff} fake files to ' - 'ensure ens_thresh check is accurate') + self.log_warn_or_debug( + f'Adding {diff} fake files to ensure ens_thresh check is accurate', + self.c_dict.get('WARN_IF_ENSEMBLE_IS_MISSING', True) + ) for _ in range(0, diff, 1): input_files.append('MISSING') @@ -1008,28 +1031,7 @@ def find_and_check_output_file(self, time_info=None, to output file path specified) @returns True if the app should be run or False if it should not """ - output_path = output_path_template - - # if output path template not specified, get it from - # c_dict keys OUTPUT_DIR and OUTPUT_TEMPLATE - if not output_path: - output_dir = self.c_dict.get('OUTPUT_DIR', '') - output_template = self.c_dict.get('OUTPUT_TEMPLATE', '') - - # remove trailing path separator if necessary (directories) - output_template = output_template.rstrip(os.path.sep) - output_path = os.path.join(output_dir, output_template) - - # substitute time info if provided - if time_info: - output_path = do_string_sub(output_path, **time_info) - - # replace wildcard character * with all - output_path = output_path.replace('*', 'all') - - # replace any whitespace with an underscore - output_path = '_'.join(output_path.split()) - + output_path = self._get_output_path_to_check(output_path_template, time_info) skip_if_output_exists = self.c_dict.get('SKIP_IF_OUTPUT_EXISTS', False) # get directory that the output file will exist @@ -1065,15 +1067,79 @@ def find_and_check_output_file(self, time_info=None, self.logger.debug(f"Creating output directory: {parent_dir}") mkdir_p(parent_dir) - if not output_exists or not skip_if_output_exists: + self._check_if_output_has_been_written(output_path, skip_if_output_exists) + + if not output_exists: return True - # if the output file exists and we are supposed to skip, don't run tool - self.logger.debug(f'Skip writing output {output_path} because it already ' - 'exists. Remove file or change ' - f'{self.app_name.upper()}_SKIP_IF_OUTPUT_EXISTS to False ' - 'to process') - return False + warn_if_exists = self.c_dict.get('WARN_IF_OUTPUT_EXISTS', False) + + if skip_if_output_exists: + # if the output file exists and we are supposed to skip, don't run tool + skip_config_name = f"{self.app_name.upper()}_SKIP_IF_OUTPUT_EXISTS" + msg = ( + f"Skip writing output {output_path} because it already exists. " + f"Remove file or change {skip_config_name} to False to process" + ) + self.log_warn_or_debug(msg, self.c_dict.get('WARN_IF_OUTPUT_EXISTS', False)) + return False + + if warn_if_exists: + self.logger.warning(f"Output {output_path} already exists and will be overwritten. " + f"Set {self.app_name.upper()}_WARN_IF_OUTPUT_EXISTS=False to turn " + "off this warning.") + + return True + + def _get_output_path_to_check(self, output_path_template, time_info): + output_path = output_path_template + + # if output path template not specified, get it from + # c_dict keys OUTPUT_DIR and OUTPUT_TEMPLATE + if not output_path: + output_dir = self.c_dict.get('OUTPUT_DIR', '') + output_template = self.c_dict.get('OUTPUT_TEMPLATE', '') + + # remove trailing path separator if necessary (directories) + output_template = output_template.rstrip(os.path.sep) + output_path = os.path.join(output_dir, output_template) + + # substitute time info if provided + if time_info: + output_path = do_string_sub(output_path, **time_info) + + # replace wildcard character * with all + output_path = output_path.replace('*', 'all') + + # replace any whitespace with an underscore + output_path = '_'.join(output_path.split()) + return output_path + + def _check_if_output_has_been_written(self, output_path, skip_if_output_exists=False): + """!Check if output file has already been written during this METplus run. + Log a warning if it has already been written. Otherwise add output path + to the list of output files that have been written. + + @param output_path path to output file or search string for apps that + write multiple output files + @param skip_if_output_exists boolean to skip writing output files if + they already exist + """ + if (not self.c_dict.get('WARN_IF_DUPLICATE_OUTPUT', True) + or skip_if_output_exists): + return + + if output_path in self.output_written: + self.logger.warning( + "Output has already been written during this METplus run and " + f"will be overwritten: {output_path}. Disable this warning by " + "setting WARN_IF_DUPLICATE_OUTPUT=False " + f"or {self.app_name.upper()}_WARN_IF_DUPLICATE_OUTPUT=False. " + "Check that the *_OUTPUT_TEMPLATE and/or *_OUTPUT_PREFIX config" + " options are set to produce unique output for each run" + ) + else: + self.output_written.append(output_path) @staticmethod def _get_valid_and_lead_from_time_info(time_info): @@ -1491,20 +1557,22 @@ def handle_climo_dict(self, sub_groups=('fcst', 'obs')): sub_groups=sub_groups): self.errors += 1 - def get_wrapper_or_generic_config(self, generic_name, var_type='str'): + def get_wrapper_or_generic_config(self, generic_name, var_type='str', + default: Any=''): """! Check for config variable with _ prepended first. If set use that value. If not, check for config without prefix. @param generic_name name of variable to read from config @param var_type type of variable to read, e.g. str, bool, int, or float. Default is str. + @param default value to return if variables are not set @returns value if set or empty string if not """ name = self.config.get_mp_config_name( [f'{self.app_name}_{generic_name}'.upper(), generic_name.upper()] ) if not name: - return '' + return default if var_type == 'bool': return self.config.getbool('config', name) if var_type == 'float': @@ -1751,3 +1819,8 @@ def handle_file_type(self, type_list=('FCST', 'OBS'), env_var_override=None): env_var_name=env_var_name, metplus_configs=metplus_configs, extra_args={'constant': True}) + + def log_warn_or_debug(self, msg: str, should_it_warn: bool) -> None: + """Logs a message at WARNING level if condition is True, otherwise at DEBUG level.""" + log_method = getattr(self.logger, 'warning' if should_it_warn else 'debug') + log_method(msg) diff --git a/metplus/wrappers/example_wrapper.py b/metplus/wrappers/example_wrapper.py index 7d39f81030..8030089621 100755 --- a/metplus/wrappers/example_wrapper.py +++ b/metplus/wrappers/example_wrapper.py @@ -36,10 +36,11 @@ def create_c_dict(self): self.logger.debug('EXAMPLE_INPUT_DIR was not set') if not c_dict['INPUT_TEMPLATE']: - self.logger.warning('EXAMPLE_INPUT_TEMPLATE was not set. ' - 'You should set this variable to see how the ' - 'runtime is substituted. ' - 'For example: {valid?fmt=%Y%m%d%H}.ext') + self.logger.warning( + 'EXAMPLE_INPUT_TEMPLATE was not set. ' + 'You should set this variable to see how the ' + 'runtime is substituted. For example: {valid?fmt=%Y%m%d%H}.ext' + ) full_path = os.path.join(c_dict['INPUT_DIR'], c_dict['INPUT_TEMPLATE']) self.logger.info(f"Input directory is {c_dict['INPUT_DIR']}") diff --git a/metplus/wrappers/extract_tiles_wrapper.py b/metplus/wrappers/extract_tiles_wrapper.py index bc7b06c9ff..cecdf7dbc5 100755 --- a/metplus/wrappers/extract_tiles_wrapper.py +++ b/metplus/wrappers/extract_tiles_wrapper.py @@ -23,8 +23,8 @@ class ExtractTilesWrapper(LoopTimesWrapper): """! Takes tc-pairs data and regrids paired data to an n x m grid as specified in the config file. """ - RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' - RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_PER_INIT_OR_VALID' + RUNTIME_FREQ_SUPPORTED = 'ALL' COLUMNS_OF_INTEREST = { 'TC_STAT': [ @@ -154,6 +154,7 @@ def regrid_data_plane_init(self): overrides[f'{data_type}_{rdp}_OUTPUT_DIR'] = self.c_dict['OUTPUT_DIR'] overrides[f'{rdp}_SKIP_IF_OUTPUT_EXISTS'] = self.c_dict['SKIP_IF_OUTPUT_EXISTS'] + overrides[f'{rdp}_WARN_IF_DUPLICATE_OUTPUT'] = self.c_dict['WARN_IF_DUPLICATE_OUTPUT'] overrides[f'{rdp}_ONCE_PER_FIELD'] = False overrides[f'{rdp}_MANDATORY'] = False diff --git a/metplus/wrappers/pcp_combine_wrapper.py b/metplus/wrappers/pcp_combine_wrapper.py index a001ef3d90..30997c5ab3 100755 --- a/metplus/wrappers/pcp_combine_wrapper.py +++ b/metplus/wrappers/pcp_combine_wrapper.py @@ -394,12 +394,13 @@ def _get_lookback_seconds(self, time_info, var_info, data_src): lookback = self.c_dict[f"{data_src}_LOOKBACK"] elif var_info: lookback = var_info[f'{data_src.lower()}_level'] - self.logger.warning( + self.logger.debug( f'{data_src}_PCP_COMBINE_LOOKBACK is ' f'not set. Using {lookback} from ' f'{data_src}_VAR{var_info.get("index")}_LEVELS' '. It is recommended that you explicitly set ' - 'the output accumulation.') + 'the output accumulation.' + ) else: lookback = '0' @@ -760,7 +761,7 @@ def _handle_name_argument(self, var_info, data_src): output_name = self.c_dict.get(f'{data_src}_OUTPUT_NAME') if not output_name and var_info: output_name = var_info.get(f"{data_src.lower()}_name") - self.logger.warning( + self.logger.debug( f'{data_src}_PCP_COMBINE_OUTPUT_NAME is ' f'not set. Using {output_name} from ' f'{data_src}_VAR{var_info.get("index")}_NAME.' diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index 017af65192..3cc4ce9e91 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -329,8 +329,9 @@ def find_input_files(self, time_info): # if there is more than 1 file, create file list file if len(model_path) > 1: - self.logger.warning('Multiple forecast files found.' - 'Using the first one') + self.logger.warning( + 'Multiple forecast files found. Using the first one' + ) self.infiles.append(model_path[0]) diff --git a/metplus/wrappers/py_embed_ingest_wrapper.py b/metplus/wrappers/py_embed_ingest_wrapper.py index c135e4b9f7..a4bc08244a 100755 --- a/metplus/wrappers/py_embed_ingest_wrapper.py +++ b/metplus/wrappers/py_embed_ingest_wrapper.py @@ -90,6 +90,8 @@ def create_c_dict(self): self.config.set(instance, 'REGRID_DATA_PLANE_SKIP_IF_OUTPUT_EXISTS', c_dict['SKIP_IF_OUTPUT_EXISTS']) + self.config.set(instance, 'REGRID_DATA_PLANE_WARN_IF_DUPLICATE_OUTPUT', + c_dict['WARN_IF_DUPLICATE_OUTPUT']) # set config variable to prevent incorrect error in RegridDataPlane # RDP requires either FCST_ or OBS_ options when run directly, but diff --git a/metplus/wrappers/regrid_data_plane_wrapper.py b/metplus/wrappers/regrid_data_plane_wrapper.py index c084cf2c99..5f11a30b80 100755 --- a/metplus/wrappers/regrid_data_plane_wrapper.py +++ b/metplus/wrappers/regrid_data_plane_wrapper.py @@ -212,9 +212,10 @@ def get_output_names(self, var_list, data_type): # if there are any duplicates, clear list and build names using # input name and input level if len(output_names) != len(set(output_names)): - self.logger.warning("Duplicate output names found. " - "Replacing output names to use " - "{input_name}_{input_level} instead.") + self.logger.warning( + "Duplicate output names found. Replacing output names to use " + "{input_name}_{input_level} instead." + ) output_names.clear() for field_info in var_list: input_name = field_info[f'{data_type.lower()}_name'] diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 7d73d748f3..3a4056b092 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -106,9 +106,7 @@ def validate_runtime_freq(self, c_dict): err_msg = (f"{self.app_name.upper()}_RUNTIME_FREQ=" f"{c_dict['RUNTIME_FREQ']} not supported.") if hasattr(self, 'RUNTIME_FREQ_DEFAULT'): - self.logger.warning( - f"{err_msg} Using {self.RUNTIME_FREQ_DEFAULT}" - ) + self.logger.warning(f"{err_msg} Using {self.RUNTIME_FREQ_DEFAULT}") c_dict['RUNTIME_FREQ'] = self.RUNTIME_FREQ_DEFAULT else: self.log_error(err_msg) @@ -171,8 +169,10 @@ def _get_templates_from_prefixes(self, prefix_list, c_dict, label): # If templates were found with this prefix, use it if templates: if found_prefix: - self.logger.warning(f'{prefix_list[0]}_INPUT_TEMPLATE and ' - f'{prefix}_INPUT_TEMPLATE are set. Using the former. ') + self.logger.warning( + f'{prefix_list[0]}_INPUT_TEMPLATE and ' + f'{prefix}_INPUT_TEMPLATE are set. Using the former.' + ) continue return_templates = templates.copy() @@ -194,8 +194,10 @@ def _handle_input_file_list(self, prefix_list, c_dict, template_dict, label): if template: # log a warning if multiple formats of config are set if template_found: - self.logger.warning(f'{prefix_list[0]}_INPUT_FILE_LIST and ' - f'{prefix}_INPUT_TEMPLATE are set. Using the former. ') + self.logger.warning( + f'{prefix_list[0]}_INPUT_FILE_LIST and ' + f'{prefix}_INPUT_TEMPLATE are set. Using the former.' + ) c_dict['EXPLICIT_FILE_LIST'] = True template_dict[label.rstrip('_')] = (template, True, False) template_found = True @@ -363,6 +365,9 @@ def run_once_per_init_or_valid(self, custom): def _get_leads_as_group(self, time_input): """!""" + if self.c_dict.get('SKIP_LEAD_SEQ', False): + return {'': [0]} + lead_groups = get_lead_sequence_groups(self.config) if lead_groups: return lead_groups @@ -625,11 +630,7 @@ def _check_input_files(self, skip_log=False): msg = 'A problem occurred trying to obtain input files' if self.c_dict['ALLOW_MISSING_INPUTS']: - if self.c_dict.get('SUPPRESS_WARNINGS', False): - self.logger.debug(msg) - else: - self.logger.warning(msg) - + self.log_warn_or_debug(msg, not self.c_dict.get('SUPPRESS_WARNINGS', False)) return False # increment error counter for GridDiag because it does not log error for each missing file @@ -906,8 +907,9 @@ def _handle_fcst_inputs_for_ensemble(self, all_input_files, fcst_files): ctrl_file = all_input_files.get('CTRL') if ctrl_file in fcst_files: # warn and remove control file if found - self.logger.warning(f"Control file found in ensemble list: " - f"{ctrl_file}. Removing from list.") + self.logger.warning( + f"Control file found in ensemble list: {ctrl_file}. Removing from list." + ) fcst_files.remove(ctrl_file) # check EnsembleStat number of files diff --git a/metplus/wrappers/stat_analysis_wrapper.py b/metplus/wrappers/stat_analysis_wrapper.py index e9e8f53059..734a0568d9 100755 --- a/metplus/wrappers/stat_analysis_wrapper.py +++ b/metplus/wrappers/stat_analysis_wrapper.py @@ -202,8 +202,9 @@ def create_c_dict(self): # if MODEL_LIST was not set, populate it from the model info list if not c_dict['MODEL_LIST'] and c_dict['MODEL_INFO_LIST']: - self.logger.warning("MODEL_LIST was left blank, " - + "creating with MODELn information.") + self.logger.debug( + "MODEL_LIST was left blank, creating with MODELn information." + ) for model_info in c_dict['MODEL_INFO_LIST']: c_dict['MODEL_LIST'].append(model_info['name']) @@ -579,8 +580,10 @@ def _set_lists_loop_or_group(self, c_dict): if c_dict[list_name]: continue - self.logger.warning(f'{list_name} was found in LOOP_LIST_ITEMS' - ' but is empty. Moving to group list') + self.logger.debug( + f"{list_name} was found in LOOP_LIST_ITEMS but is empty. " + "Moving to group list" + ) c_dict['GROUP_LIST_ITEMS'].append(list_name) c_dict['LOOP_LIST_ITEMS'].remove(list_name) diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index 10d58b00fe..1677f1edcd 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -321,20 +321,15 @@ def validate_runtime_freq(self, c_dict): # check deprecated TC_PAIRS_RUN_ONCE, warn and handle if set elif self.config.has_option('config', 'TC_PAIRS_RUN_ONCE'): - self.logger.warning('TC_PAIRS_RUN_ONCE is deprecated.') run_once = self.config.getbool('config', 'TC_PAIRS_RUN_ONCE', True) - if run_once: - self.logger.warning('Setting TC_PAIRS_RUNTIME_FREQ=RUN_ONCE.' - 'Please remove TC_PAIRS_RUN_ONCE and ' - 'set TC_PAIRS_RUNTIME_FREQ=RUN_ONCE ' - 'to remove this warning') - c_dict['RUNTIME_FREQ'] = 'RUN_ONCE' - else: - self.logger.warning('Setting TC_PAIRS_RUNTIME_FREQ=RUN_ONCE_FOR_EACH.' - 'Please remove TC_PAIRS_RUN_ONCE and ' - 'set TC_PAIRS_RUNTIME_FREQ=RUN_ONCE_FOR_EACH ' - 'to remove this warning') - c_dict['RUNTIME_FREQ'] = 'RUN_ONCE_FOR_EACH' + freq_override = 'RUN_ONCE' if run_once else 'RUN_ONCE_FOR_EACH' + c_dict['RUNTIME_FREQ'] = freq_override + msg = ( + f"TC_PAIRS_RUN_ONCE is deprecated. Setting TC_PAIRS_RUNTIME_FREQ={freq_override}. " + f"Please remove TC_PAIRS_RUN_ONCE and set TC_PAIRS_RUNTIME_FREQ={freq_override} " + "to remove this warning" + ) + self.logger.warning(msg) # if runtime frequency set to run once for each time, check skip lead if c_dict['RUNTIME_FREQ'] == 'RUN_ONCE_FOR_EACH': diff --git a/metplus/wrappers/tc_stat_wrapper.py b/metplus/wrappers/tc_stat_wrapper.py index 503ce76962..8884402edd 100755 --- a/metplus/wrappers/tc_stat_wrapper.py +++ b/metplus/wrappers/tc_stat_wrapper.py @@ -185,6 +185,9 @@ def create_c_dict(self): c_dict['FIND_FILES'] = False # force error if inputs are missing c_dict['ALLOW_MISSING_INPUTS'] = False + # ignore forecast leads + c_dict['SKIP_LEAD_SEQ'] = True + return c_dict def set_met_config_for_environment_variables(self): diff --git a/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf b/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf index 39e06016db..cf3e88ce14 100644 --- a/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf +++ b/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf @@ -48,10 +48,13 @@ ASCII2NC_OUTPUT_DIR = ASCII2NC_OUTPUT_TEMPLATE = {OUTPUT_BASE}/ascii2nc/precip24_{valid?fmt=%Y%m%d%H}.nc ASCII2NC_SKIP_IF_OUTPUT_EXISTS = False +#ASCII2NC_WARN_IF_DUPLICATE_OUTPUT = +#ASCII2NC_WARN_IF_OUTPUT_EXISTS = #ASCII2NC_FILE_WINDOW_BEGIN = 0 #ASCII2NC_FILE_WINDOW_END = 0 + ### # ASCII2NC Settings # https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#ascii2nc diff --git a/parm/use_cases/met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf b/parm/use_cases/met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf index 931f906fb2..719d2f9bba 100644 --- a/parm/use_cases/met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf +++ b/parm/use_cases/met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf @@ -13,6 +13,9 @@ PROCESS_LIST = CyclonePlotter +LOOP_BY = INIT +INIT_TIME_FMT = %Y%m%d%H +INIT_BEG = 2015030112 ### # File I/O @@ -29,9 +32,9 @@ CYCLONE_PLOTTER_OUTPUT_DIR = {OUTPUT_BASE}/cyclone # https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#cycloneplotter ### -CYCLONE_PLOTTER_INIT_DATE = 20150301 +CYCLONE_PLOTTER_INIT_DATE = {init?fmt=%Y%m%d} -CYCLONE_PLOTTER_INIT_HR = 12 +CYCLONE_PLOTTER_INIT_HR = {init?fmt=%H} CYCLONE_PLOTTER_MODEL = GFSO CYCLONE_PLOTTER_PLOT_TITLE = Model Forecast Storm Tracks diff --git a/parm/use_cases/met_tool_wrapper/DataIngest/DataIngest.conf b/parm/use_cases/met_tool_wrapper/DataIngest/DataIngest.conf index 6ef9afc72f..11cdd90d30 100644 --- a/parm/use_cases/met_tool_wrapper/DataIngest/DataIngest.conf +++ b/parm/use_cases/met_tool_wrapper/DataIngest/DataIngest.conf @@ -40,6 +40,8 @@ LEAD_SEQ = 0 ### DATA_INGEST_SKIP_IF_OUTPUT_EXISTS = False +#DATA_INGEST_WARN_IF_DUPLICATE_OUTPUT = +#DATA_INGEST_WARN_IF_OUTPUT_EXISTS = DATA_INGEST_1_INPUT_TEMPLATE = https://dtcenter.ucar.edu/dfiles/code/METplus/DataIngest_input/{valid?fmt=%Y/%m/%d}/point/metar/netcdf/{valid?fmt=%Y%m%d_%H%M}.gz DATA_INGEST_1_OUTPUT_TEMPLATE = {OUTPUT_BASE}/data_ingest/madis_metar/{valid?fmt=%Y%m%d_%H%M}.nc diff --git a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf index 27f38bb16a..e3efebf1ac 100644 --- a/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf +++ b/parm/use_cases/met_tool_wrapper/EnsembleStat/EnsembleStat.conf @@ -64,6 +64,9 @@ ENSEMBLE_STAT_CLIMO_STDEV_INPUT_TEMPLATE = ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE}/ensemble ENSEMBLE_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H%M}/ensemble_stat +#ENSEMBLE_STAT_WARN_IF_DUPLICATE_OUTPUT = +#ENSEMBLE_STAT_WARN_IF_OUTPUT_EXISTS = +#ENSEMBLE_STAT_WARN_IF_ENSEMBLE_IS_MISSING = ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/ExtractTiles/ExtractTiles.conf b/parm/use_cases/met_tool_wrapper/ExtractTiles/ExtractTiles.conf index 019209b8a7..3dd4a84307 100644 --- a/parm/use_cases/met_tool_wrapper/ExtractTiles/ExtractTiles.conf +++ b/parm/use_cases/met_tool_wrapper/ExtractTiles/ExtractTiles.conf @@ -52,7 +52,8 @@ FCST_EXTRACT_TILES_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d_%H}/{storm_id}/FCST_TILE_F OBS_EXTRACT_TILES_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d_%H}/{storm_id}/OBS_TILE_F{lead?fmt=%3H}_gfs_4_{valid?fmt=%Y%m%d}_{valid?fmt=%H}00_000.nc EXTRACT_TILES_SKIP_IF_OUTPUT_EXISTS = yes - +#EXTRACT_TILES_WARN_IF_DUPLICATE_OUTPUT = +#EXTRACT_TILES_WARN_IF_OUTPUT_EXISTS = ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf index 5e65053422..48c449a100 100644 --- a/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf +++ b/parm/use_cases/met_tool_wrapper/GenEnsProd/GenEnsProd.conf @@ -41,10 +41,8 @@ LEAD_SEQ = 24H GEN_ENS_PROD_INPUT_DIR = {INPUT_BASE}/met_test/data/sample_fcst GEN_ENS_PROD_INPUT_TEMPLATE = - {init?fmt=%Y%m%d%H}/arw-fer-gep1/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib, {init?fmt=%Y%m%d%H}/arw-sch-gep2/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib, {init?fmt=%Y%m%d%H}/arw-tom-gep3/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib, - {init?fmt=%Y%m%d%H}/arw-tom-gep4/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib, {init?fmt=%Y%m%d%H}/arw-fer-gep5/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib, {init?fmt=%Y%m%d%H}/arw-sch-gep6/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib, {init?fmt=%Y%m%d%H}/arw-tom-gep7/d01_{init?fmt=%Y%m%d%H}_{lead?fmt=%3H}00.grib @@ -56,6 +54,9 @@ GEN_ENS_PROD_CTRL_INPUT_TEMPLATE = GEN_ENS_PROD_OUTPUT_DIR = {OUTPUT_BASE}/gen_ens_prod GEN_ENS_PROD_OUTPUT_TEMPLATE = gen_ens_prod_{valid?fmt=%Y%m%d_%H%M%S}V_ens.nc +#GEN_ENS_PROD_WARN_IF_DUPLICATE_OUTPUT = +#GEN_ENS_PROD_WARN_IF_OUTPUT_EXISTS = +#GEN_ENS_PROD_WARN_IF_ENSEMBLE_IS_MISSING = ### # Field Info @@ -90,7 +91,7 @@ ENS_VAR5_THRESH = >=5.0 # https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#genensprod ### -GEN_ENS_PROD_N_MEMBERS = 6 +GEN_ENS_PROD_N_MEMBERS = 5 #LOG_GEN_ENS_PROD_VERBOSITY = 2 diff --git a/parm/use_cases/met_tool_wrapper/GenVxMask/GenVxMask.conf b/parm/use_cases/met_tool_wrapper/GenVxMask/GenVxMask.conf index 9b49581368..8f041198cd 100644 --- a/parm/use_cases/met_tool_wrapper/GenVxMask/GenVxMask.conf +++ b/parm/use_cases/met_tool_wrapper/GenVxMask/GenVxMask.conf @@ -50,6 +50,8 @@ GEN_VX_MASK_OUTPUT_TEMPLATE = {OUTPUT_BASE}/met_tool_wrapper/GenVxMask/POLY_GFS_ GEN_VX_MASK_SKIP_IF_OUTPUT_EXISTS = False +#GEN_VX_MASK_WARN_IF_DUPLICATE_OUTPUT = +#GEN_VX_MASK_WARN_IF_OUTPUT_EXISTS = GEN_VX_MASK_FILE_WINDOW_BEGIN = 0 GEN_VX_MASK_FILE_WINDOW_END = 0 diff --git a/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf b/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf index 141f819f2b..cc7b1488e6 100644 --- a/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf +++ b/parm/use_cases/met_tool_wrapper/GridDiag/GridDiag.conf @@ -47,6 +47,8 @@ GRID_DIAG_INPUT_TEMPLATE = gfs.subset.t00z.pgrb2.0p25.f{lead?fmt=%H}, gfs.subset GRID_DIAG_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/GridDiag GRID_DIAG_OUTPUT_TEMPLATE = grid_diag_out.nc +#GRID_DIAG_WARN_IF_DUPLICATE_OUTPUT = +#GRID_DIAG_WARN_IF_OUTPUT_EXISTS = ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf index f3871a281e..af8353ff1c 100644 --- a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf +++ b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf @@ -54,6 +54,9 @@ GRID_STAT_CLIMO_STDEV_INPUT_TEMPLATE = GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/GridStat/GridStat GRID_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H} +#GRID_STAT_WARN_IF_DUPLICATE_OUTPUT = +#GRID_STAT_WARN_IF_OUTPUT_EXISTS = + ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf b/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf index e0ab14a4d1..0f54e25f4d 100644 --- a/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf +++ b/parm/use_cases/met_tool_wrapper/IODA2NC/IODA2NC.conf @@ -43,6 +43,8 @@ IODA2NC_INPUT_TEMPLATE = ioda.NC001007.{valid?fmt=%Y%m%d%H}.nc IODA2NC_OUTPUT_DIR = {OUTPUT_BASE}/ioda2nc IODA2NC_OUTPUT_TEMPLATE = ioda.NC001007.{valid?fmt=%Y%m%d%H}.summary.nc +#IODA2NC_WARN_IF_DUPLICATE_OUTPUT = +#IODA2NC_WARN_IF_OUTPUT_EXISTS = ### # IODA2NC Settings diff --git a/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf b/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf index 41b8009075..b6fc3ac071 100644 --- a/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf +++ b/parm/use_cases/met_tool_wrapper/MADIS2NC/MADIS2NC.conf @@ -48,6 +48,8 @@ MADIS2NC_OUTPUT_DIR = MADIS2NC_OUTPUT_TEMPLATE = {OUTPUT_BASE}/madis2nc/met_metar_{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}.nc #MADIS2NC_SKIP_IF_OUTPUT_EXISTS = False +#MADIS2NC_WARN_IF_DUPLICATE_OUTPUT = +#MADIS2NC_WARN_IF_OUTPUT_EXISTS = #MADIS2NC_FILE_WINDOW_BEGIN = 0 #MADIS2NC_FILE_WINDOW_END = 0 diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf index 429b71f6e6..34497a5ed0 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf @@ -48,6 +48,9 @@ OBS_MODE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d%H}/wrfprs_ruc13_00.tm00_G212 MODE_OUTPUT_DIR = {OUTPUT_BASE}/mode MODE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d%H} +#MODE_WARN_IF_DUPLICATE_OUTPUT = +#MODE_WARN_IF_OUTPUT_EXISTS = + FCST_MODE_FILE_WINDOW_BEGIN = 0 FCST_MODE_FILE_WINDOW_END = 0 diff --git a/parm/use_cases/met_tool_wrapper/MTD/MTD.conf b/parm/use_cases/met_tool_wrapper/MTD/MTD.conf index d68d7a63a4..de52447d15 100644 --- a/parm/use_cases/met_tool_wrapper/MTD/MTD.conf +++ b/parm/use_cases/met_tool_wrapper/MTD/MTD.conf @@ -48,6 +48,8 @@ OBS_MTD_INPUT_TEMPLATE = ST2ml{valid?fmt=%Y%m%d%H}_A03h.nc MTD_OUTPUT_DIR = {OUTPUT_BASE}/mtd MTD_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d%H} +#MTD_WARN_IF_DUPLICATE_OUTPUT = +#MTD_WARN_IF_OUTPUT_EXISTS = ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf b/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf index b07193bace..489f0d1f52 100644 --- a/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf +++ b/parm/use_cases/met_tool_wrapper/PB2NC/PB2NC.conf @@ -45,6 +45,8 @@ PB2NC_INPUT_TEMPLATE = ndas.t{da_init?fmt=%H}z.prepbufr.tm{offset?fmt=%2H}.{da_i PB2NC_OFFSETS = 12 PB2NC_SKIP_IF_OUTPUT_EXISTS = True +#PB2NC_WARN_IF_DUPLICATE_OUTPUT = +#PB2NC_WARN_IF_OUTPUT_EXISTS = PB2NC_OUTPUT_DIR = {OUTPUT_BASE}/pb2nc PB2NC_OUTPUT_TEMPLATE = sample_pb.nc diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf index 5c41b22184..9eb64df8a4 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_add.conf @@ -47,6 +47,8 @@ FCST_PCP_COMBINE_INPUT_TEMPLATE = NEWSe_{init?fmt=%Y%m%d}_i{init?fmt=%H%M}_m0_f{ FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/PCPCombine/PCPCombine_add FCST_PCP_COMBINE_OUTPUT_TEMPLATE = NEWSe5min_mem00_lag00.nc +#PCP_COMBINE_WARN_IF_DUPLICATE_OUTPUT = +#PCP_COMBINE_WARN_IF_OUTPUT_EXISTS = ### # PCPCombine Settings diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_bucket.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_bucket.conf index 6325df895b..ca1a00579a 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_bucket.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_bucket.conf @@ -47,6 +47,8 @@ FCST_PCP_COMBINE_INPUT_TEMPLATE = gfs_{init?fmt=%Y%m%d%H}_F{lead?fmt=%3H}.grib FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/PCPCombine/PCPCombine_bucket FCST_PCP_COMBINE_OUTPUT_TEMPLATE = gfs_{valid?fmt=%Y%m%d%H}_A{level?fmt=%3H}.nc +#PCP_COMBINE_WARN_IF_DUPLICATE_OUTPUT = +#PCP_COMBINE_WARN_IF_OUTPUT_EXISTS = ### # PCPCombine Settings diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_derive.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_derive.conf index 831f636ba7..40e1baf5d8 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_derive.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_derive.conf @@ -47,6 +47,8 @@ FCST_PCP_COMBINE_INPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/wrfprs_ruc13_{lead?fmt=%HH FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/PCPCombine/PCPCombine_derive FCST_PCP_COMBINE_OUTPUT_TEMPLATE = wrfprs_ruc13_{init?fmt=%Y%m%d%H}_f{lead?fmt=%HH}_A{level?fmt=%HH}.nc +#PCP_COMBINE_WARN_IF_DUPLICATE_OUTPUT = +#PCP_COMBINE_WARN_IF_OUTPUT_EXISTS = ### # PCPCombine Settings diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_loop_custom.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_loop_custom.conf index 720d6ccb43..e38c70c867 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_loop_custom.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_loop_custom.conf @@ -49,6 +49,8 @@ FCST_PCP_COMBINE_INPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/{custom?fmt=%s}/d01_{init? FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/PCPCombine/PCPCombine_loop_custom FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/d01_{init?fmt=%Y%m%d%H}_0{lead?fmt=%HH}00.nc +#PCP_COMBINE_WARN_IF_DUPLICATE_OUTPUT = +#PCP_COMBINE_WARN_IF_OUTPUT_EXISTS = ### # PCPCombine Settings diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf index ebb7a0127e..34ce7448c1 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf @@ -47,6 +47,8 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = PYTHON_NUMPY OBS_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/PCPCombine/PCPCombine_combine_py_embed OBS_PCP_COMBINE_OUTPUT_TEMPLATE = IMERG.{valid?fmt=%Y%m%d_%H%M}_A{level?fmt=%2H}h +#PCP_COMBINE_WARN_IF_DUPLICATE_OUTPUT = +#PCP_COMBINE_WARN_IF_OUTPUT_EXISTS = ### # PCPCombine Settings diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_subtract.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_subtract.conf index 59174513d1..4f4b505bf9 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_subtract.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_subtract.conf @@ -48,6 +48,8 @@ FCST_PCP_COMBINE_INPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/wrfprs_ruc13_{lead?fmt=%HH FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/PCPCombine/PCPCombine_subtract FCST_PCP_COMBINE_OUTPUT_TEMPLATE = wrfprs_ruc13_{init?fmt=%Y%m%d%H}_f{lead?fmt=%HH}_A03.nc +#PCP_COMBINE_WARN_IF_DUPLICATE_OUTPUT = +#PCP_COMBINE_WARN_IF_OUTPUT_EXISTS = ### # PCPCombine Settings diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_sum.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_sum.conf index 9012e7fb3b..7b95ec5755 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_sum.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_sum.conf @@ -47,6 +47,8 @@ FCST_PCP_COMBINE_INPUT_TEMPLATE = NEWSe_{init?fmt=%Y%m%d}_i{init?fmt=%H%M}_m0_f* FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/PCPCombine/PCPCombine_sum FCST_PCP_COMBINE_OUTPUT_TEMPLATE = NEWSe5min_mem00_lag00.nc +#PCP_COMBINE_WARN_IF_DUPLICATE_OUTPUT = +#PCP_COMBINE_WARN_IF_OUTPUT_EXISTS = ### # PCPCombine Settings diff --git a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_user_defined.conf b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_user_defined.conf index a30bc6bd30..d039b5caf4 100644 --- a/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_user_defined.conf +++ b/parm/use_cases/met_tool_wrapper/PCPCombine/PCPCombine_user_defined.conf @@ -48,6 +48,8 @@ FCST_PCP_COMBINE_INPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/wrfprs_ruc13_{lead?fmt=%HH FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/PCPCombine/PCPCombine_user_defined FCST_PCP_COMBINE_OUTPUT_TEMPLATE = wrfprs_ruc13_{init?fmt=%Y%m%d%H}_f{lead?fmt=%HH}_A{level?fmt=%HH}.nc +#PCP_COMBINE_WARN_IF_DUPLICATE_OUTPUT = +#PCP_COMBINE_WARN_IF_OUTPUT_EXISTS = ### # PCPCombine Settings diff --git a/parm/use_cases/met_tool_wrapper/PairStat/PairStat.conf b/parm/use_cases/met_tool_wrapper/PairStat/PairStat.conf index 46e3f4b0e0..deb9ec3824 100644 --- a/parm/use_cases/met_tool_wrapper/PairStat/PairStat.conf +++ b/parm/use_cases/met_tool_wrapper/PairStat/PairStat.conf @@ -47,6 +47,9 @@ PAIR_STAT_PAIRS_INPUT_TEMPLATE = point_stat_{lead?fmt=%H%M%S}L_{valid?fmt=%Y%m%d PAIR_STAT_OUTPUT_DIR = {OUTPUT_BASE}/pair_stat PAIR_STAT_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d_%H}_out +#PAIR_STAT_WARN_IF_DUPLICATE_OUTPUT = +#PAIR_STAT_WARN_IF_OUTPUT_EXISTS = + #PAIR_STAT_CLIMO_MEAN_INPUT_DIR = #PAIR_STAT_CLIMO_MEAN_INPUT_TEMPLATE = diff --git a/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_grib1.conf b/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_grib1.conf index f67abc6065..cb35913e9c 100644 --- a/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_grib1.conf +++ b/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_grib1.conf @@ -45,6 +45,8 @@ PLOT_DATA_PLANE_INPUT_TEMPLATE = {INPUT_BASE}/met_test/data/sample_fcst/{valid?f PLOT_DATA_PLANE_OUTPUT_DIR = PLOT_DATA_PLANE_OUTPUT_TEMPLATE = {OUTPUT_BASE}/met_tool_wrapper/plot_data_plane/nam.t00z.awip1236.tm{valid?fmt=%H}.{valid?fmt=%Y%m%d}_TMPZ2.ps +#PLOT_DATA_PLANE_WARN_IF_DUPLICATE_OUTPUT = +#PLOT_DATA_PLANE_WARN_IF_OUTPUT_EXISTS = ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_netcdf.conf b/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_netcdf.conf index b98422004c..07aba40d85 100644 --- a/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_netcdf.conf +++ b/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_netcdf.conf @@ -45,6 +45,8 @@ PLOT_DATA_PLANE_INPUT_TEMPLATE = {INPUT_BASE}/met_test/out/pcp_combine/sample_fc PLOT_DATA_PLANE_OUTPUT_DIR = PLOT_DATA_PLANE_OUTPUT_TEMPLATE = {OUTPUT_BASE}/met_tool_wrapper/plot_data_plane/sample_fcst_12L_{valid?fmt=%Y%m%d%H}V_12A_APCP12_NC_MET.ps +#PLOT_DATA_PLANE_WARN_IF_DUPLICATE_OUTPUT = +#PLOT_DATA_PLANE_WARN_IF_OUTPUT_EXISTS = ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_python_embedding.conf b/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_python_embedding.conf index 5240c11cde..7681af6684 100644 --- a/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_python_embedding.conf +++ b/parm/use_cases/met_tool_wrapper/PlotDataPlane/PlotDataPlane_python_embedding.conf @@ -45,6 +45,8 @@ PLOT_DATA_PLANE_INPUT_TEMPLATE = PYTHON_NUMPY PLOT_DATA_PLANE_OUTPUT_DIR = PLOT_DATA_PLANE_OUTPUT_TEMPLATE = {OUTPUT_BASE}/met_tool_wrapper/plot_data_plane/py_embed_fcst.ps +#PLOT_DATA_PLANE_WARN_IF_DUPLICATE_OUTPUT = +#PLOT_DATA_PLANE_WARN_IF_OUTPUT_EXISTS = ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf index 304e1c5253..00b1a2581f 100644 --- a/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf +++ b/parm/use_cases/met_tool_wrapper/PlotPointObs/PlotPointObs.conf @@ -54,6 +54,8 @@ PLOT_POINT_OBS_OUTPUT_DIR = {OUTPUT_BASE}/plot_point_obs PLOT_POINT_OBS_OUTPUT_TEMPLATE = nam_and_ndas.{valid?fmt=%Y%m%d}.t{valid?fmt=%H}z.prepbufr_CONFIG.ps #PLOT_POINT_OBS_SKIP_IF_OUTPUT_EXISTS = False +#PLOT_POINT_OBS_WARN_IF_DUPLICATE_OUTPUT = +#PLOT_POINT_OBS_WARN_IF_OUTPUT_EXISTS = ### diff --git a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf index 03695fcc67..98314486a6 100644 --- a/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf +++ b/parm/use_cases/met_tool_wrapper/Point2Grid/Point2Grid.conf @@ -45,6 +45,9 @@ POINT2GRID_INPUT_TEMPLATE = {INPUT_BASE}/met_test/data/sample_obs/prepbufr/sampl POINT2GRID_OUTPUT_DIR = POINT2GRID_OUTPUT_TEMPLATE = {OUTPUT_BASE}/met_tool_wrapper/Point2Grid/grid.{init?fmt=%Y%d%H}.nc +#POINT2GRID_WARN_IF_DUPLICATE_OUTPUT = +#POINT2GRID_WARN_IF_OUTPUT_EXISTS = + POINT2GRID_FILE_WINDOW_BEGIN = 0 POINT2GRID_FILE_WINDOW_END = 0 diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index db0e4d8eaa..f742a071ba 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -47,6 +47,9 @@ OBS_POINT_STAT_INPUT_TEMPLATE = sample_pb.nc POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/point_stat +#POINT_STAT_WARN_IF_DUPLICATE_OUTPUT = +#POINT_STAT_WARN_IF_OUTPUT_EXISTS = + POINT_STAT_CLIMO_MEAN_INPUT_DIR = POINT_STAT_CLIMO_MEAN_INPUT_TEMPLATE = diff --git a/parm/use_cases/met_tool_wrapper/PyEmbedIngest/PyEmbedIngest.conf b/parm/use_cases/met_tool_wrapper/PyEmbedIngest/PyEmbedIngest.conf index 8064c39b72..4974a0fc13 100644 --- a/parm/use_cases/met_tool_wrapper/PyEmbedIngest/PyEmbedIngest.conf +++ b/parm/use_cases/met_tool_wrapper/PyEmbedIngest/PyEmbedIngest.conf @@ -53,3 +53,6 @@ PY_EMBED_INGEST_2_OUTPUT_TEMPLATE = {OUTPUT_BASE}/met_tool_wrapper/PyEmbedIngest PY_EMBED_INGEST_2_SCRIPT = {MET_INSTALL_DIR}/share/met/python/examples/read_ascii_numpy.py {INPUT_BASE}/met_test/data/python/fcst.txt FCST PY_EMBED_INGEST_2_TYPE = NUMPY PY_EMBED_INGEST_2_OUTPUT_GRID = G130 + +#PY_EMBED_INGEST_WARN_IF_DUPLICATE_OUTPUT = +#PY_EMBED_INGEST_WARN_IF_OUTPUT_EXISTS = diff --git a/parm/use_cases/met_tool_wrapper/RMWAnalysis/RMWAnalysis.conf b/parm/use_cases/met_tool_wrapper/RMWAnalysis/RMWAnalysis.conf index bb8de345d8..e03e17a6ae 100644 --- a/parm/use_cases/met_tool_wrapper/RMWAnalysis/RMWAnalysis.conf +++ b/parm/use_cases/met_tool_wrapper/RMWAnalysis/RMWAnalysis.conf @@ -45,6 +45,8 @@ RMW_ANALYSIS_INPUT_TEMPLATE = tc_rmw_aal14{init?fmt=%Y}.nc RMW_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE}/rmw_analysis RMW_ANALYSIS_OUTPUT_TEMPLATE = rmw_analysis_aal14{init?fmt=%Y}.nc +#RMW_ANALYSIS_WARN_IF_DUPLICATE_OUTPUT = +#RMW_ANALYSIS_WARN_IF_OUTPUT_EXISTS = ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/RegridDataPlane/RegridDataPlane.conf b/parm/use_cases/met_tool_wrapper/RegridDataPlane/RegridDataPlane.conf index 1a9940e237..265e61e49f 100644 --- a/parm/use_cases/met_tool_wrapper/RegridDataPlane/RegridDataPlane.conf +++ b/parm/use_cases/met_tool_wrapper/RegridDataPlane/RegridDataPlane.conf @@ -47,6 +47,8 @@ OBS_REGRID_DATA_PLANE_OUTPUT_DIR = {OUTPUT_BASE}/regrid_data_plane OBS_REGRID_DATA_PLANE_TEMPLATE = ST2ml{valid?fmt=%Y%m%d%H}.Grb_G212 +#REGRID_DATA_PLANE_WARN_IF_DUPLICATE_OUTPUT = +#REGRID_DATA_PLANE_WARN_IF_OUTPUT_EXISTS = ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf b/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf index 91072f8a8d..aee272928b 100644 --- a/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf +++ b/parm/use_cases/met_tool_wrapper/SeriesAnalysis/SeriesAnalysis.conf @@ -60,6 +60,9 @@ SERIES_ANALYSIS_AGGR_INPUT_TEMPLATE = SERIES_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/SeriesAnalysis SERIES_ANALYSIS_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H}_sa.nc +#SERIES_ANALYSIS_WARN_IF_DUPLICATE_OUTPUT = +#SERIES_ANALYSIS_WARN_IF_OUTPUT_EXISTS = + SERIES_ANALYSIS_CLIMO_MEAN_INPUT_DIR = SERIES_ANALYSIS_CLIMO_MEAN_INPUT_TEMPLATE = diff --git a/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf b/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf index 89f4167b6d..7c6e406d1b 100644 --- a/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf +++ b/parm/use_cases/met_tool_wrapper/StatAnalysis/StatAnalysis.conf @@ -45,6 +45,9 @@ MODEL1_STAT_ANALYSIS_DUMP_ROW_TEMPLATE = {fcst_valid_hour?fmt=%H}Z/{MODEL1}/{MOD STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE}/stat_analysis STAT_ANALYSIS_OUTPUT_TEMPLATE = job.out +#STAT_ANALYSIS_WARN_IF_DUPLICATE_OUTPUT = +#STAT_ANALYSIS_WARN_IF_OUTPUT_EXISTS = + ### # StatAnalysis Settings diff --git a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf index b10919341f..3fb0964d7a 100644 --- a/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf +++ b/parm/use_cases/met_tool_wrapper/TCDiag/TCDiag.conf @@ -48,6 +48,9 @@ TC_DIAG_INPUT1_TECH_ID_LIST = AVNO TC_DIAG_OUTPUT_DIR = {OUTPUT_BASE}/tc_diag TC_DIAG_OUTPUT_TEMPLATE = {date?fmt=%Y} +#TC_DIAG_WARN_IF_DUPLICATE_OUTPUT = +#TC_DIAG_WARN_IF_OUTPUT_EXISTS = + ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf b/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf index ace36034eb..e442ed954a 100644 --- a/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf +++ b/parm/use_cases/met_tool_wrapper/TCGen/TCGen.conf @@ -55,6 +55,9 @@ TC_GEN_GENESIS_INPUT_TEMPLATE = {init?fmt=%Y}*/genesis*{init?fmt=%Y}* TC_GEN_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/TCGen TC_GEN_OUTPUT_TEMPLATE = tc_gen_{init?fmt=%Y} +#TC_GEN_WARN_IF_DUPLICATE_OUTPUT = +#TC_GEN_WARN_IF_OUTPUT_EXISTS = + ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf index e5a0bcf1eb..5d5c1308ba 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf @@ -57,6 +57,8 @@ TC_PAIRS_OUTPUT_TEMPLATE = {date?fmt=%Y%m}/{basin?fmt=%s}q{date?fmt=%Y%m%d%H}.gf TC_PAIRS_SKIP_IF_OUTPUT_EXISTS = yes TC_PAIRS_SKIP_IF_REFORMAT_EXISTS = yes +#TC_PAIRS_WARN_IF_DUPLICATE_OUTPUT = +#TC_PAIRS_WARN_IF_OUTPUT_EXISTS = TC_PAIRS_READ_ALL_FILES = no diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf index 9c3415b5ee..598715a12a 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf @@ -56,6 +56,9 @@ TC_PAIRS_OUTPUT_DIR = {OUTPUT_BASE}/tc_pairs TC_PAIRS_OUTPUT_TEMPLATE = tc_pairs_{basin?fmt=%s}{date?fmt=%Y%m%d%H}.dat TC_PAIRS_SKIP_IF_OUTPUT_EXISTS = no +#TC_PAIRS_WARN_IF_DUPLICATE_OUTPUT = +#TC_PAIRS_WARN_IF_OUTPUT_EXISTS = + TC_PAIRS_READ_ALL_FILES = no TC_PAIRS_REFORMAT_DECK = no diff --git a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf index 3e118c961c..c65a205fae 100644 --- a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf +++ b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf @@ -46,6 +46,8 @@ TC_RMW_INPUT_TEMPLATE = gfs.subset.t00z.pgrb2.0p25.f* TC_RMW_OUTPUT_DIR = {OUTPUT_BASE}/met_tool_wrapper/TCRMW TC_RMW_OUTPUT_TEMPLATE = tc_rmw_aal14{date?fmt=%Y}.nc +#TCRMW_WARN_IF_DUPLICATE_OUTPUT = +#TCRMW_WARN_IF_OUTPUT_EXISTS = ### # Field Info diff --git a/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf b/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf index ec1705915b..3a62944a02 100644 --- a/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf +++ b/parm/use_cases/met_tool_wrapper/TCStat/TCStat.conf @@ -42,6 +42,8 @@ TC_STAT_LOOKIN_DIR = {INPUT_BASE}/met_test/tc_pairs TC_STAT_OUTPUT_DIR = {OUTPUT_BASE}/tc_stat TC_STAT_OUTPUT_TEMPLATE = job.out +#TC_STAT_WARN_IF_DUPLICATE_OUTPUT = +#TC_STAT_WARN_IF_OUTPUT_EXISTS = ### # TCStat Settings diff --git a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf index 704b434f44..2945b37af8 100644 --- a/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf +++ b/parm/use_cases/met_tool_wrapper/UserScript/UserScript_run_once.conf @@ -37,6 +37,8 @@ LEAD_SEQ = 0H, 12H, 15H, 24H, 120H USER_SCRIPT_CUSTOM_LOOP_LIST = nc +USER_SCRIPT_SKIP_INIT_TIMES = "%Y%m%d%H%M%S: 20141030183015,20141031063015,20141031183015" + ### # File I/O diff --git a/parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf b/parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf index 1cf908fd50..dac469a2d6 100644 --- a/parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf +++ b/parm/use_cases/met_tool_wrapper/WaveletStat/WaveletStat.conf @@ -48,6 +48,9 @@ OBS_WAVELET_STAT_INPUT_TEMPLATE = ST2ml{valid?fmt=%Y%m%d%H}_A03h.nc WAVELET_STAT_OUTPUT_DIR = {OUTPUT_BASE}/wavelet_stat WAVELET_STAT_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H} +#WAVELET_STAT_WARN_IF_DUPLICATE_OUTPUT = +#WAVELET_STAT_WARN_IF_OUTPUT_EXISTS = + ### # Field Info diff --git a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf index 8e8eb73c06..6fb9b131f1 100644 --- a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf +++ b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf @@ -54,7 +54,7 @@ GRID_STAT_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d%H%M}/grid_stat MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {OUTPUT_BASE}/met_out/{MODEL1}/anom/*/grid_stat STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE}/gather_by_date/stat_analysis/grid2grid/anom -MODEL1_STAT_ANALYSIS_DUMP_ROW_TEMPLATE = {fcst_valid_hour?fmt=%H}Z/{MODEL1}/{MODEL1}_{valid?fmt=%Y%m%d}.stat +MODEL1_STAT_ANALYSIS_DUMP_ROW_TEMPLATE = {fcst_valid_hour?fmt=%H}Z/{MODEL1}/{MODEL1}_{fcst_var}_{fcst_level}_{valid?fmt=%Y%m%d}.stat ### diff --git a/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf b/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf index ea00e64932..00fb181170 100644 --- a/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf +++ b/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf @@ -38,6 +38,8 @@ SERIES_ANALYSIS_RUNTIME_FREQ = RUN_ONCE_PER_LEAD SERIES_ANALYSIS_RUN_ONCE_PER_STORM_ID = False +PY_EMBED_INGEST_SKIP_IF_OUTPUT_EXISTS = True + ### # File I/O diff --git a/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_Blocking.conf b/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_Blocking.conf index cc69d67d28..c9dc1283f0 100644 --- a/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_Blocking.conf +++ b/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_Blocking.conf @@ -37,7 +37,7 @@ VALID_INCREMENT = 86400 LEAD_SEQ = 0 # Only Process DJF -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:0229" ### @@ -192,7 +192,7 @@ VALID_END = 2017022800 VALID_INCREMENT = 86400 # Add the first/last 2 days to the skip times to compute the running mean -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,1202,0227,0228,0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,1202,0227,0228,0229" OBS_PCP_COMBINE_RUN = TRUE OBS_PCP_COMBINE_METHOD = DERIVE @@ -235,7 +235,7 @@ VALID_INCREMENT = 86400 LEAD_SEQ = 0 # Add the first/last 2 days to the skip times to compute the running mean -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,1202,0227,0228,0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,1202,0227,0228,0229" OBS_PCP_COMBINE_RUN = True @@ -259,7 +259,7 @@ OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_ANOM_INPUT_TEMPLATE} [create_cbl_filelist] # Skip the days on the edges that are not available due to the running mean -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,0229" # Find the files for each lead time USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_LEAD @@ -373,7 +373,7 @@ IBL_PLOT_FCST_LABEL = GEFS LEAD_SEQ = 24 # Skip the days on the edges that are not available due to the running mean -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,0229" # Run the user script once for each lead USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_LEAD diff --git a/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_Blocking/ci_overrides.conf b/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_Blocking/ci_overrides.conf new file mode 100644 index 0000000000..ce4c429054 --- /dev/null +++ b/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_Blocking/ci_overrides.conf @@ -0,0 +1,5 @@ +[config] + +# do not exit when warnings are logged because they are expected + +EXIT_IF_WARN = False diff --git a/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_WeatherRegime.conf b/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_WeatherRegime.conf index 07ec4711c8..d8d0a1dfba 100644 --- a/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_WeatherRegime.conf +++ b/parm/use_cases/model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_WeatherRegime.conf @@ -39,7 +39,7 @@ VALID_INCREMENT = 86400 LEAD_SEQ = 0 # Only Process DJF -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,0229" ### diff --git a/parm/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_Blocking.conf b/parm/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_Blocking.conf index c949f8c777..18b42a686a 100644 --- a/parm/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_Blocking.conf +++ b/parm/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_Blocking.conf @@ -38,7 +38,7 @@ VALID_INCREMENT = 86400 LEAD_SEQ = 0 # Only Process DJF -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:0229" # Run the obs data # A variable set to be used in the pre-processing steps @@ -121,7 +121,7 @@ OBS_PCP_COMBINE_OUTPUT_TEMPLATE = Z500_daily_{valid?fmt=%Y%m%d?shift=-64800}_NH. [running_mean_obs] # Add the first/last 2 days to the skip times to compute the running mean -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,1202,1203,1204,0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,1202,1203,1204,0229" OBS_PCP_COMBINE_RUN = {OBS_RUN} @@ -156,7 +156,7 @@ OBS_PCP_COMBINE_OUTPUT_TEMPLATE = Z500_5daymean_{valid?fmt=%Y%m%d?shift=-172800} [anomaly_obs] # Add the first/last 2 days to the skip times to compute the running mean -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,1202,0227,0228,0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,1202,0227,0228,0229" OBS_PCP_COMBINE_RUN = {OBS_RUN} @@ -247,7 +247,7 @@ OBS_BLOCKING_PLOT_OUTPUT_NAME = ERA_Block_Freq_DJF [script_blocking] # Skip the days on the edges that are not available due to the running mean -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,1202,0227,0228,0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:1201,1202,0227,0228,0229" # Run the user script once per lead USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_LEAD diff --git a/parm/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.conf b/parm/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.conf index b228043c9f..a461347e25 100644 --- a/parm/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.conf +++ b/parm/use_cases/model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.conf @@ -37,7 +37,7 @@ VALID_INCREMENT = 86400 LEAD_SEQ = 0 # Only Process DJF -SKIP_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:0229" +SKIP_VALID_TIMES = "%m:begin_end_incr(3,11,1)", "%m%d:0229" USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_LEAD diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/PointStat_fcstWRF_obsMADIS_hurricane_matthew.conf b/parm/use_cases/model_applications/tc_and_extra_tc/PointStat_fcstWRF_obsMADIS_hurricane_matthew.conf index d9347f8eb8..39faf37e3d 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/PointStat_fcstWRF_obsMADIS_hurricane_matthew.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/PointStat_fcstWRF_obsMADIS_hurricane_matthew.conf @@ -39,6 +39,9 @@ LEAD_SEQ = begin_end_incr(0,48,1) MADIS2NC_ALLOW_MISSING_INPUTS = True MADIS2NC_MISSING_INPUT_THRESH = 0.875 +# skip known missing times to prevent warnings in example use case +MADIS2NC_SKIP_VALID_TIMES = "%Y%m%d%H:2016100603,2016100621,2016100715,2016100719,2016100720,2016100721" + [config] ### diff --git a/pyproject.toml b/pyproject.toml index 30e5629b73..061e947592 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,3 +45,18 @@ dependencies = {file = ["requirements.txt"]} [tool.setuptools-git-versioning] enabled = true version_file = "metplus/VERSION" + +[tool.pytest.ini_options] +markers = [ + "run_metplus: custom marker for testing run_metplus.py script", + "util: custom marker for testing metplus/util logic", + "wrapper_a: custom marker for testing metplus/wrapper logic - A group", + "wrapper_b: custom marker for testing metplus/wrapper logic - B group", + "wrapper_c: custom marker for testing metplus/wrapper logic - C group", + "wrapper_d: custom marker for testing metplus/wrapper logic - D group", + "wrapper: custom marker for testing metplus/wrapper logic - all others", + "long: custom marker for tests that take a long time to run", + "plotting: custom marker for tests that involve plotting", + "diff: custom marker for diff util tests that require additional packages", + +] \ No newline at end of file