diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml new file mode 100644 index 00000000..d7506ae7 --- /dev/null +++ b/.github/workflows/python-lint.yml @@ -0,0 +1,32 @@ +name: Python Lint + +on: + pull_request: + paths: + - 'pre/remap_restart/**' + - '.github/workflows/python-lint.yml' + +jobs: + ruff: + name: ruff + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/ruff-action@v4.1.0 + with: + src: pre/remap_restart + + ty: + name: ty + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: astral-sh/setup-uv@v10.3.0 + with: + enable-cache: false + - name: Run ty + working-directory: pre/remap_restart + run: | + uv venv + uv pip install ty numpy netCDF4 scipy pyyaml ruamel.yaml questionary + uv run ty check --python .venv . diff --git a/.gitignore b/.gitignore index 64c70aec..28a241cd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,10 @@ *# .#* **/CVS/ + +# Python +__pycache__/ +*.py[cod] +.ruff_cache/ +.pytest_cache/ +.venv/ diff --git a/CHANGELOG.md b/CHANGELOG.md index d5d74c72..510c7ae2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,11 +24,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added plots for EDMF - Added new column and cloud variables and fixed bugs. +- Added `ruff.toml` for `pre/remap_restart` +- Added a GitHub Action (`python-lint.yml`) that runs `ruff` and `ty` on `pre/remap_restart` +- Added Python ignores (`__pycache__/`, `.ruff_cache/`, `.pytest_cache/`, `.venv/`) to the root `.gitignore` + ### Changed - Updated plot seasons in plot.rc +- Cleaned up `pre/remap_restart` Python code per ruff: explicit imports in place of star imports, context managers for file handling, `sys.exit()` in place of `exit()`, and other style fixes +- `remap_restarts` now makes v14 the default BCs version in the interactive questions; v12 remains in the main menu, and v11, v13 and v15_BETA remain under "Other" +- `remap_restarts` now removes its scratch directories (`InData/`, `OutData/`, `upper_data/`, `ana_data/`) after a successful remap +- Subprocess failures in `remap_restarts` now raise an error instead of being ignored +- `remap_params.yaml` is now written without line wrapping ### Fixed +- Fixed `shutil.remove` calls (now `os.remove`) in `remap_other_restarts.py` +- Fixed a `TypeError` in `remap_utils.py` when `bc_version` is missing +- Fixed a trailing space after `bc_base:` in the generated `remap_params.yaml` +- `get_resolutions` in `remap_utils.py` now raises a clear error if `agrid` or `ogrid` is missing + ### Removed ### Deprecated diff --git a/pre/remap_restart/.gitignore b/pre/remap_restart/.gitignore deleted file mode 100644 index a348e504..00000000 --- a/pre/remap_restart/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/__pycache__/ diff --git a/pre/remap_restart/remap_analysis.py b/pre/remap_restart/remap_analysis.py index 3e9b8f85..ecdb21cf 100755 --- a/pre/remap_restart/remap_analysis.py +++ b/pre/remap_restart/remap_analysis.py @@ -8,17 +8,17 @@ # source g5_modules.sh [bash] # source g5_modules [csh] # +import fnmatch +import glob import os -from datetime import datetime, timedelta -import subprocess import shlex import shutil -import glob -import fileinput -import ruamel.yaml +import subprocess +from datetime import datetime, timedelta + from remap_base import remap_base -from remap_utils import get_label, MERRA2_RST_BASE -import fnmatch +from remap_utils import MERRA2_RST_BASE, get_label + class analysis(remap_base): def __init__(self, **configs): @@ -56,11 +56,7 @@ def remap(self): os.chdir(tmpdir) yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh']) - yyyy_ = yyyymmddhh_[0:4] - mm_ = yyyymmddhh_[4:6] - dd_ = yyyymmddhh_[6:8] hh_ = yyyymmddhh_[8:10] - rst_time = datetime(year=int(yyyy_), month=int(mm_), day=int(dd_), hour = int(hh_)) expid_in = config['input']['shared']['expid'] expid_out = config['output']['shared']['expid'] if (expid_out) : @@ -83,13 +79,10 @@ def remap(self): f_tmp = tmpdir+'/'+out_name local_fs.append(f_tmp) shutil.copy(f,f_tmp) - if out_name.find('satbias') != -1 : - if (aqua): - f_ = open(f_tmp, 'w') - for line in fileinput.input(f): + if out_name.find('satbias') != -1 and aqua: + with open(f) as f_in, open(f_tmp, 'w') as f_: # This is unlikely to be the case with new files - f_.write(line.replace('airs281SUBSET_aqua', 'airs281_aqua ')) - f_.close() + f_.writelines(line.replace('airs281SUBSET_aqua', 'airs281_aqua ') for line in f_in) nlevel = config['output']['air']['nlevel'] agrid_out = config['output']['shared']['agrid'] @@ -102,7 +95,7 @@ def remap(self): shutil.move(f,f_orig) cmd = bindir + '/dyn2dyn.x ' + flags + ' -o ' + f + ' ' + f_orig print(cmd) - subprocess.call(shlex.split(cmd)) + subprocess.run(shlex.split(cmd), check=True) for f in local_fs: fname = os.path.basename(f) @@ -124,9 +117,11 @@ def remap(self): rstlcvOut = out_dir+'/'+expid_out+'rst.lcv.'+ymd_+'_'+hh_+'z.bin' + label cmd = bindir+'/mkdrstdate.x ' + ymd_ + ' ' + hms_ +' ' + rstlcvOut print(cmd) - subprocess.call(shlex.split(cmd)) + subprocess.run(shlex.split(cmd), check=True) print( "cd " + cwdir) os.chdir(cwdir) + print("remove " + tmpdir) + shutil.rmtree(tmpdir) if self.config['input']['shared']['MERRA-2']: self.remove_merra2() @@ -187,10 +182,10 @@ def copy_merra2(self): delt = timedelta(hours = h-3) new_time = rst_time + delt yyyy = "Y"+str(new_time.year) - mm = 'M%02d'%new_time.month - ymd = '%04d%02d%02d'%(new_time.year,new_time.month, new_time.day) - hh = '%02d'%h - newhh= '%02d'%new_time.hour + mm = f'M{new_time.month:02d}' + ymd = f'{new_time.year:04d}{new_time.month:02d}{new_time.day:02d}' + hh = f'{h:02d}' + newhh= f'{new_time.hour:02d}' m2_rst_dir = merra_2_rst_dir.replace('Y'+yyyy_,yyyy).replace('M'+mm_,mm) # bkg files for ftype in ['sfc', 'eta']: @@ -231,8 +226,8 @@ def copy_merra2(self): delt = timedelta(hours = 3) new_time = rst_time - delt yyyy = "Y"+str(new_time.year) - mm = 'M%02d'%new_time.month - ymdh = '%04d%02d%02d%02d'%(new_time.year, new_time.month, new_time.day, new_time.hour) + mm = f'M{new_time.month:02d}' + ymdh = f'{new_time.year:04d}{new_time.month:02d}{new_time.day:02d}{new_time.hour:02d}' m2_rst_dir = merra_2_rst_dir.replace('Y'+yyyy_,yyyy).replace('M'+mm_,mm) fname = expid+'.trak.GDA.rst.'+ymdh+'z.txt' f = m2_rst_dir+'/'+fname diff --git a/pre/remap_restart/remap_base.py b/pre/remap_restart/remap_base.py index df04c621..0e3559f9 100755 --- a/pre/remap_restart/remap_base.py +++ b/pre/remap_restart/remap_base.py @@ -4,12 +4,16 @@ # remap_base.py does [???] # import os -import ruamel.yaml +import shlex import shutil import subprocess -from remap_utils import * -class remap_base(object): +import netCDF4 as nc + +from remap_utils import GEOSIT_RST_BASE, get_config_from_file + + +class remap_base: def __init__(self, **configs): for key, value in configs.items(): if (key == 'params_file'): @@ -73,8 +77,7 @@ def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, s if diff != 0: return False # for catchment, even tile files are the same, if bc is different, it still need remap - if (catch) : - if in_bc_version != out_bc_version : return False + if catch and in_bc_version != out_bc_version : return False expid = config['output']['shared']['expid'] if (expid) : @@ -88,7 +91,7 @@ def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, s f = expid + os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_rst.'+suffix cmd = '/bin/cp ' + rst + ' ' + out_dir+'/'+f print('\n'+cmd) - subprocess.call(shlex.split(cmd)) + subprocess.run(shlex.split(cmd), check=True) return True diff --git a/pre/remap_restart/remap_bin2nc.py b/pre/remap_restart/remap_bin2nc.py index ec06d278..dfe79899 100755 --- a/pre/remap_restart/remap_bin2nc.py +++ b/pre/remap_restart/remap_bin2nc.py @@ -7,10 +7,11 @@ # remapped. The set of yaml config files "bin2nc_merra2_*.yaml" include hard-wired # variables names, long names, and tile/grid dimensions specific to MERRA-2 restarts. -from yaml import safe_load, load, dump -from netCDF4 import Dataset import numpy as np +from netCDF4 import Dataset from scipy.io import FortranFile +from yaml import dump, safe_load + def writeCVars(coordVars,dimensions): @@ -73,36 +74,36 @@ def defineDimVars(fid,dims): for d in dims: if d=="lon": newVar=fid.createVariable(d,'f8',d) - setattr(newVar,'units','degrees_east') - setattr(newVar,'long_name','Longitude') + newVar.units = 'degrees_east' + newVar.long_name = 'Longitude' coordVars.update([(d,newVar)]) if d=="lat": newVar=fid.createVariable(d,'f8',d) - setattr(newVar,'units','degrees_north') - setattr(newVar,'long_name','Latitude') + newVar.units = 'degrees_north' + newVar.long_name = 'Latitude' coordVars.update([(d,newVar)]) if d=="lev": newVar=fid.createVariable(d,'f8',d) - setattr(newVar,'units','layer') - setattr(newVar,'long_name','sigma_at_layer_midpoints') - setattr(newVar,'standard_name','atmosphere_hybrid_sigma_pressure_coordinate') - setattr(newVar,'coordinate','eta') - setattr(newVar,'positive','down') - setattr(newVar,'formulaTerms','ap: ak b: bk ps: ps p0: p00') + newVar.units = 'layer' + newVar.long_name = 'sigma_at_layer_midpoints' + newVar.standard_name = 'atmosphere_hybrid_sigma_pressure_coordinate' + newVar.coordinate = 'eta' + newVar.positive = 'down' + newVar.formulaTerms = 'ap: ak b: bk ps: ps p0: p00' coordVars.update([(d,newVar)]) if d=="edge": newVar=fid.createVariable(d,'f8',d) - setattr(newVar,'units','level') - setattr(newVar,'long_name','sigma_at_layer edge') - setattr(newVar,'standard_name','atmosphere_hybrid_sigma_pressure_coordinate') - setattr(newVar,'coordinate','eta') - setattr(newVar,'positive','down') - setattr(newVar,'formulaTerms','ap: ak b: bk ps: ps p0: p00') + newVar.units = 'level' + newVar.long_name = 'sigma_at_layer edge' + newVar.standard_name = 'atmosphere_hybrid_sigma_pressure_coordinate' + newVar.coordinate = 'eta' + newVar.positive = 'down' + newVar.formulaTerms = 'ap: ak b: bk ps: ps p0: p00' coordVars.update([(d,newVar)]) if d=="time": newVar=fid.createVariable(d,'f8',d) - setattr(newVar,'units','minutes since ') - setattr(newVar,'long_name','time') + newVar.units = 'minutes since ' + newVar.long_name = 'time' coordVars.update([(d,newVar)]) return coordVars @@ -114,9 +115,8 @@ def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=Fal bintype=np.float64 - f=open(yamlfile,'r') - - data=safe_load(f) + with open(yamlfile,'r') as f: + data=safe_load(f) dimensions = data["dimensions"] variables = data["variables"] @@ -142,8 +142,8 @@ def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=Fal units = v['units'] dims = v['dimension'] newVar = ncfid.createVariable(sname,'f4',dims) - setattr(newVar,'long_name',lname) - setattr(newVar,'units',units) + newVar.long_name = lname + newVar.units = units vars.update([(sname,newVar)]) writeCVars(cVars,dimensions) diff --git a/pre/remap_restart/remap_catchANDcn.py b/pre/remap_restart/remap_catchANDcn.py index 41aaaa6e..74a8cdaf 100755 --- a/pre/remap_restart/remap_catchANDcn.py +++ b/pre/remap_restart/remap_catchANDcn.py @@ -8,17 +8,50 @@ # source g5_modules.sh [bash] # source g5_modules [csh] # -import os -import sys -import subprocess -import shutil import glob -import ruamel.yaml -import shlex import mimetypes +import os +import shutil +import subprocess +import sys + import netCDF4 as nc +import questionary + from remap_base import remap_base -from remap_utils import * + +from remap_utils import ( #isort: skip + GEOS_SITE, + MERRA2_RST_BASE, + choices_bc_base, + choices_bc_ops, + choices_bc_all, + choices_ogrid_data, + config_to_yaml, + data_ocean_default, + get_account, + get_config_from_answers, + get_default_bc_base, + get_geomdir, + get_label, + get_landdir, + job_directive, + message_account, + message_agrid_new, + message_bc_base_new, + message_bc_ops_new, + message_bc_all_new, + message_datetime, + message_expid, + message_ogrid_new, + message_out_dir, + message_partition, + message_qos, + print_config, + remove_ogrid_comment, + validate_agrid, +) + class catchANDcn(remap_base): def __init__(self, **configs): @@ -202,9 +235,8 @@ def remap(self): script_name = './mk_catchANDcn.j' - catch_scrpt = open(script_name,'wt') - catch_scrpt.write(catch1script) - catch_scrpt.close() + with open(script_name,'wt') as catch_scrpt: + catch_scrpt.write(catch1script) interactive = None if GEOS_SITE == 'NAS': @@ -223,20 +255,23 @@ def remap(self): ntasks = int(ntasks) if (ntasks < NPE): - print("\nYou should have at least {NPE} cores. Now you only have {ntasks} cores ".format(NPE=NPE, ntasks=ntasks)) + print(f"\nYou should have at least {NPE} cores. Now you only have {ntasks} cores ") - subprocess.call(['chmod', '755', script_name]) + subprocess.run(['chmod', '755', script_name], check=True) print(script_name+ ' 1>' + log_name + ' 2>&1') - os.system(script_name + ' 1>' + log_name+ ' 2>&1') + with open(log_name, "w") as log_f: + subprocess.run([script_name], stdout=log_f, stderr=subprocess.STDOUT, check=True) elif GEOS_SITE == "NAS" : print('qsub -W block=true '+ script_name +'\n') - subprocess.call(['qsub', '-W','block=true', script_name]) + subprocess.run(['qsub', '-W','block=true', script_name], check=True) else: print('sbatch -W '+ script_name +'\n') - subprocess.call(['sbatch', '-W', script_name]) + subprocess.run(['sbatch', '-W', script_name], check=True) print( "cd " + cwdir) os.chdir(cwdir) + print('remove ' + InData_dir) + shutil.rmtree(InData_dir) if self.config['input']['shared']['MERRA-2']: self.remove_merra2() @@ -249,7 +284,6 @@ def copy_merra2(self): yyyymmddhh_ = str(self.config['input']['shared']['yyyymmddhh']) yyyy_ = yyyymmddhh_[0:4] mm_ = yyyymmddhh_[4:6] - dd_ = yyyymmddhh_[6:8] hh_ = yyyymmddhh_[8:10] suffix = yyyymmddhh_[0:8]+'_'+ hh_ + 'z.bin' @@ -286,9 +320,7 @@ def has_catch_rst(text): nonlocal catch_input_shared_rst_dir rst_dir = catch_input_shared_rst_dir + '/rs/ens0000/Y'+yyyy +'/M'+mm+'/' rst_files = glob.glob(rst_dir+'*catch*_internal_rst.'+yyyy+mm+dd+'_'+hh+'00') - if len(rst_files) !=1 : - return False - return True + return len(rst_files) == 1 questions =[ { @@ -336,9 +368,9 @@ def has_catch_rst(text): { "type": "select", "name": "output:shared:bc_version", - "message": message_bc_other_new, - "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other', + "message": message_bc_all_new, + "choices": choices_bc_all, + "when": lambda x: x["output:shared:bc_version"] == 'All', }, { diff --git a/pre/remap_restart/remap_command_line.py b/pre/remap_restart/remap_command_line.py index 0de66bae..b53529b7 100755 --- a/pre/remap_restart/remap_command_line.py +++ b/pre/remap_restart/remap_command_line.py @@ -4,15 +4,26 @@ # remap_command_line.py parses and converts the command-line arguments and converts the information # into matching "Answers" for the questionary (remap_questions.py) # +import argparse import os -import subprocess -import shlex + import ruamel.yaml -import shutil -import questionary -import glob -import argparse -from remap_utils import * + +from remap_utils import ( + choices_bc_all, + choices_catchmodel, + choices_ogrid_cmd, + choices_omodel, + choices_stretch, + fvcore_info, + get_account, + get_config_from_answers, + get_zoom, + init_geosit, + init_merra2, + wemin_default, +) + def parse_args(program_description): @@ -29,7 +40,7 @@ def parse_args(program_description): p_config.add_argument('-c', '--config_file', help='YAML config file') - p_landonly = p_sub.add_parser( + p_sub.add_parser( 'land_only', help = "Remap land (catch[cn]) restart only; global domain only; ens0000 only.\n", ) @@ -51,8 +62,8 @@ def parse_args(program_description): p_command.add_argument('-expid', help='Experiment ID of input restarts') p_command.add_argument('-newid', default="", help='Experiment ID for new restarts') - p_command.add_argument('-bcvin', help='Boundary conditions version of input restarts', choices= choices_bc_cmd) - p_command.add_argument('-bcvout', help='Boundary conditions version for new restarts', choices= choices_bc_cmd) + p_command.add_argument('-bcvin', help='Boundary conditions version of input restarts', choices= choices_bc_all) + p_command.add_argument('-bcvout', help='Boundary conditions version for new restarts', choices= choices_bc_all) p_command.add_argument('-in_wemin', help='Min. snow water equivalent param. used with input restarts') p_command.add_argument('-out_wemin', help='Min. snow water equivalent param. to be used with new restarts') diff --git a/pre/remap_restart/remap_other_restarts.py b/pre/remap_restart/remap_other_restarts.py index 8b2c7cb2..a5f5318c 100755 --- a/pre/remap_restart/remap_other_restarts.py +++ b/pre/remap_restart/remap_other_restarts.py @@ -9,25 +9,23 @@ # source g5_modules.sh [bash] # source g5_modules [csh] # +import glob import os -import subprocess as sp import shutil import subprocess -import glob -import ruamel.yaml -import shlex -import netCDF4 as nc +import subprocess as sp + from remap_base import remap_base +from remap_bin2nc import bin2nc from remap_utils import ( - get_label, + GEOS_SITE, + MERRA2_RST_BASE, get_geomdir, - get_zoom, + get_label, get_landdir, + get_zoom, job_directive, - GEOS_SITE, - MERRA2_RST_BASE, ) -from remap_bin2nc import bin2nc class other_restarts(remap_base): @@ -200,7 +198,7 @@ def remap(self): f = os.path.basename(rst) dest = InData_dir + "/" + f if os.path.exists(dest): - shutil.remove(dest) + os.remove(dest) print("\nCopy " + rst + " to " + dest) shutil.copy(rst, dest) if "saltwater_internal" in f: @@ -226,9 +224,9 @@ def remap(self): out_til = OutData_dir + "/" + os.path.basename(out_tile_file) if os.path.exists(in_til): - shutil.remove(in_til) + os.remove(in_til) if os.path.exists(out_til): - shutil.remove(out_til) + os.remove(out_til) print("\n Copy " + in_tile_file + " to " + in_til) shutil.copy(in_tile_file, in_til) print("\n Copy " + out_tile_file + " to " + out_til) @@ -478,15 +476,16 @@ def remap(self): if interactive: print("interactive mode\n") - subprocess.call(["chmod", "755", script_name]) + subprocess.run(["chmod", "755", script_name], check=True) print(script_name + " 1>" + log_name + " 2>&1") - os.system(script_name + " 1>" + log_name + " 2>&1") + with open(log_name, "w") as log_f: + subprocess.run([script_name], stdout=log_f, stderr=subprocess.STDOUT, check=True) elif GEOS_SITE == "NAS": print("qsub -W block=true " + script_name + "\n") - subprocess.call(["qsub", "-W", "block=true", script_name]) + subprocess.run(["qsub", "-W", "block=true", script_name], check=True) else: print("sbatch -W " + script_name + "\n") - subprocess.call(["sbatch", "-W", script_name]) + subprocess.run(["sbatch", "-W", script_name], check=True) suffix = "_rst." + suffix for out_rst in glob.glob("OutData/*_rst*"): @@ -499,6 +498,9 @@ def remap(self): shutil.move(out_rst, out_dir + "/" + filename) print("cd " + cwdir) os.chdir(cwdir) + for scratch in (InData_dir, OutData_dir): + print("remove " + scratch) + shutil.rmtree(scratch) if self.config["input"]["shared"]["MERRA-2"]: self.remove_merra2() @@ -541,7 +543,6 @@ def copy_merra2(self): yyyymmddhh_ = str(self.config["input"]["shared"]["yyyymmddhh"]) yyyy_ = yyyymmddhh_[0:4] mm_ = yyyymmddhh_[4:6] - dd_ = yyyymmddhh_[6:8] hh_ = yyyymmddhh_[8:10] suffix = yyyymmddhh_[0:8] + "_" + hh_ + "z.bin" diff --git a/pre/remap_restart/remap_questions.py b/pre/remap_restart/remap_questions.py index 2437548b..e01b2ff1 100755 --- a/pre/remap_restart/remap_questions.py +++ b/pre/remap_restart/remap_questions.py @@ -6,9 +6,56 @@ # # import os -import ruamel.yaml +import sys + import questionary -from remap_utils import * +import ruamel.yaml + +from remap_utils import ( #isort: skip + GEOS_SITE, + MERRA2_RST_BASE, + catch_model, + choices_bc_base, + choices_bc_ops, + choices_bc_all, + choices_ogrid_cpld, + choices_ogrid_data, + choices_omodel, + choices_res_SG001, + choices_res_SG002, + choices_stretch, + data_ocean_default, + fvcore_info, + get_account, + get_command_line_from_answers, + get_config_from_answers, + get_default_bc_base, + get_landdir, + get_zoom, + init_geosit, + init_merra2, + message_account, + message_agrid_in, + message_agrid_new, + message_bc_base_in, + message_bc_base_new, + message_bc_ops_in, + message_bc_ops_new, + message_bc_all_in, + message_bc_all_new, + message_expid, + message_ogrid_in, + message_out_dir, + message_partition, + message_qos, + message_reservation, + message_stretch, + remove_ogrid_comment, + show_wemin_default, + validate_agrid, + wemin_default, +) + def echo_level(x): if x["output:air:nlevel"] != str(x.get("input:air:nlevel")) : @@ -27,7 +74,7 @@ def echo_bcs(x,opt): x[opt+':shared:bc_base'] = base land_dir = get_landdir(base, bcv, agrid=agrid, ogrid=ogrid, omodel=omodel, stretch=stretch) if not os.path.isdir(land_dir): - exit("cannot find grid subdirectory for agrid=" + agrid + " and ogrid=" + ogrid + " under " + base+'/'+bcv+'/land/') + sys.exit("cannot find grid subdirectory for agrid=" + agrid + " and ogrid=" + ogrid + " under " + base+'/'+bcv+'/land/') print("\n Land BCs for " + opt + " restarts: " + land_dir ) return False @@ -40,23 +87,15 @@ def default_partition(x): def validate_merra2_time(text): if len(text) == 10 : hh = text[8:] - if hh in ['03','09','15','21']: - return True - else: - return False - else: - return False + return hh in ['03','09','15','21'] + return False def validate_geosit_time(text): if len(text) == 10 : dd = text[6:8] hh = text[8:] - if dd in ['14','28'] and hh in ['21']: - return True - else: - return False - else: - return False + return dd in ['14','28'] and hh in ['21'] + return False def SITE_MERRA2(x): if GEOS_SITE == "NAS": @@ -282,9 +321,9 @@ def ask_questions(): { "type": "select", "name": "input:shared:bc_version", - "message": message_bc_other_in, - "choices": choices_bc_other, - "when": lambda x: x["input:shared:bc_version"] == 'Other', + "message": message_bc_all_in, + "choices": choices_bc_all, + "when": lambda x: x["input:shared:bc_version"] == 'All', }, { @@ -292,7 +331,7 @@ def ask_questions(): "name": "output:shared:bc_version", "message": message_bc_ops_new, "choices": choices_bc_ops, - "default": "v12", + "default": "v14", "when": lambda x: x.get("input:shared:MERRA-2") or x.get("input:shared:GEOS-IT"), }, @@ -301,24 +340,16 @@ def ask_questions(): "name": "output:shared:bc_version", "message": "Select BCs version for new restarts:\n", "choices": choices_bc_ops, - "default": "v12", + "default": "v14", "when": lambda x: not x.get("input:shared:MERRA-2") and not x.get("input:shared:GEOS-IT"), }, { "type": "select", "name": "output:shared:bc_version", - "message": message_bc_other_new, - "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] not in ['v11', 'v12', 'v13', 'v14', 'v15_BETA'], - }, - - { - "type": "select", - "name": "output:shared:bc_version", - "message": "\nSelect BCs version for new restarts:\n", - "choices": choices_bc_other, - "when": lambda x: x["output:shared:bc_version"] == 'Other' and x["input:shared:bc_version"] in ['v11', 'v12', 'v13', 'v14', 'v15_BETA'], + "message": message_bc_all_new, + "choices": choices_bc_all, + "when": lambda x: x.get("output:shared:bc_version") == "All", }, { @@ -378,7 +409,7 @@ def ask_questions(): { "type": "confirm", "name": "output:air:agcm_import_rst", - "message": f'''Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs? + "message": '''Remap agcm_import_rst (a.k.a. IAU) file needed for REPLAY runs? (NOTE: Preferred method is to regenerate IAU file, but IF requested, remapping will be performed.)''', "default": False, diff --git a/pre/remap_restart/remap_restarts.py b/pre/remap_restart/remap_restarts.py index ac6c1ba2..2feda760 100755 --- a/pre/remap_restart/remap_restarts.py +++ b/pre/remap_restart/remap_restarts.py @@ -10,20 +10,25 @@ # source g5_modules [csh] # import sys -import argparse import textwrap -import ruamel.yaml -import questionary -from remap_utils import * -from remap_questions import * -from remap_command_line import * -from remap_upper import * -from remap_other_restarts import * -from remap_analysis import * -from remap_catchANDcn import * +import questionary -program_description = textwrap.dedent(f''' +from remap_analysis import analysis +from remap_catchANDcn import catchANDcn, remap_land_only +from remap_command_line import get_answers_from_command_line, parse_args +from remap_other_restarts import other_restarts +from remap_questions import ask_questions +from remap_upper import upperair +from remap_utils import ( + config_to_yaml, + get_command_line_from_answers, + get_config_from_answers, + print_config, + write_cmd, +) + +program_description = textwrap.dedent(''' USAGE: This script provides four options for remapping GEOS restart files: @@ -58,11 +63,9 @@ ''') def main(): - question_flag = False config = '' - yaml = ruamel.yaml.YAML() # Parse the command line arguments from parse_args() capturing the arguments and the rest - cmdl, extra_args = parse_args(program_description) + cmdl, _ = parse_args(program_description) answers = {} config_yaml ='' noprompt = False @@ -78,7 +81,6 @@ def main(): if (len(sys.argv) == 1 or answers) : if not answers: answers = ask_questions() - question_flag = True cmd = get_command_line_from_answers(answers) write_cmd(answers['output:shared:out_dir'], cmd) # just for debugging diff --git a/pre/remap_restart/remap_upper.py b/pre/remap_restart/remap_upper.py index e11f8909..b937bed7 100755 --- a/pre/remap_restart/remap_upper.py +++ b/pre/remap_restart/remap_upper.py @@ -8,16 +8,24 @@ # source g5_modules.sh [bash] # source g5_modules [csh] # +import glob import os -import ruamel.yaml -import subprocess import shlex import shutil -import glob +import subprocess +import sys + from remap_base import remap_base -from remap_utils import * from remap_bin2nc import bin2nc -import netCDF4 as nc +from remap_utils import ( + GEOS_SITE, + MERRA2_RST_BASE, + STRETCH_GRID, + get_label, + get_topodir, + job_directive, +) + class upperair(remap_base): def __init__(self, **configs): @@ -124,23 +132,23 @@ def remap(self): f = os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_restart_in' cmd = '/bin/ln -s ' + rst + ' ' + f print('\n'+cmd) - subprocess.call(shlex.split(cmd)) + subprocess.run(shlex.split(cmd), check=True) # link topo file cmd = '/bin/ln -s ' + topoin + ' .' print('\n'+cmd) - subprocess.call(shlex.split(cmd)) + subprocess.run(shlex.split(cmd), check=True) cmd = '/bin/ln -s ' + topoout + ' topo_dynave.data' print('\n'+cmd) - subprocess.call(shlex.split(cmd)) + subprocess.run(shlex.split(cmd), check=True) agrid = config['output']['shared']['agrid'] if agrid[0].upper() == 'C': imout = int(agrid[1:]) else: - exit("Only support cs grid so far") + sys.exit("Only support cs grid so far") if (imout <90): NPE = 12; nwrit = 1 @@ -205,7 +213,7 @@ def remap(self): target_lat = STRETCH_GRID['SG002'][1] target_lon = STRETCH_GRID['SG002'][2] else: - exit("This stretched grid option is not supported " + str(stretch)) + sys.exit("This stretched grid option is not supported " + str(stretch)) # note "reversed" order of args (relative to order in definition of STRETCH_GRID) @@ -222,7 +230,7 @@ def remap(self): target_lat = STRETCH_GRID['SG002'][1] target_lon = STRETCH_GRID['SG002'][2] else: - exit("This stretched grid option is not supported " + str(stretch)) + sys.exit("This stretched grid option is not supported " + str(stretch)) # note "reversed" order of args (relative to order in definition of STRETCH_GRID) @@ -360,9 +368,8 @@ def remap(self): script_name = './remap_upper.j' - upper = open(script_name,'wt') - upper.write(remap_upper_script) - upper.close() + with open(script_name,'wt') as upper: + upper.write(remap_upper_script) interactive = None if GEOS_SITE == 'NAS': @@ -380,17 +387,18 @@ def remap(self): ntasks = nnodes * ncpus ntasks = int(ntasks) if (ntasks < NPE ): - print("\nYou should have at least {NPE} cores. Now you only have {ntasks} cores ".format(NPE=NPE, ntasks=ntasks)) + print(f"\nYou should have at least {NPE} cores. Now you only have {ntasks} cores ") - subprocess.call(['chmod', '755', script_name]) + subprocess.run(['chmod', '755', script_name], check=True) print(script_name+ ' 1>' + log_name + ' 2>&1') - os.system(script_name + ' 1>' + log_name+ ' 2>&1') + with open(log_name, "w") as log_f: + subprocess.run([script_name], stdout=log_f, stderr=subprocess.STDOUT, check=True) elif GEOS_SITE == "NAS" : print('qsub -W block=true '+ script_name +'\n') - subprocess.call(['qsub', '-W','block=true', script_name]) + subprocess.run(['qsub', '-W','block=true', script_name], check=True) else: print('sbatch -W '+ script_name +'\n') - subprocess.call(['sbatch', '-W', script_name]) + subprocess.run(['sbatch', '-W', script_name], check=True) # # post process @@ -411,6 +419,8 @@ def remap(self): f.write(time) print('cd ' + cwdir) os.chdir(cwdir) + print('remove ' + out_dir + '/upper_data') + shutil.rmtree(out_dir + '/upper_data') if self.config['input']['shared']['MERRA-2']: self.remove_merra2() @@ -442,7 +452,6 @@ def copy_merra2(self): yyyymmddhh_ = str(self.config['input']['shared']['yyyymmddhh']) yyyy_ = yyyymmddhh_[0:4] mm_ = yyyymmddhh_[4:6] - dd_ = yyyymmddhh_[6:8] hh_ = yyyymmddhh_[8:10] suffix = yyyymmddhh_[0:8]+'_'+ hh_ + 'z.bin' diff --git a/pre/remap_restart/remap_utils.py b/pre/remap_restart/remap_utils.py index 8a427e22..d78538d0 100755 --- a/pre/remap_restart/remap_utils.py +++ b/pre/remap_restart/remap_utils.py @@ -3,17 +3,18 @@ # remap_restarts package: # remap_utils.py contains utility functions and definitions # +import glob import os +import re +import shlex +import shutil import subprocess -import ruamel.yaml +import sys from collections import OrderedDict -import shutil -import questionary -import glob -import shlex + import netCDF4 as nc -import linecache -import re +import questionary +import ruamel.yaml # shared global variables @@ -30,11 +31,9 @@ # define "choices", "message" strings, and "validate" lists that are used multiple times # (and related definitions, even if they are used just once). -choices_bc_ops = ['v12', 'NL3', 'ICA', 'Other'] - -choices_bc_other = ['v11', 'v12', 'v13', 'v14', 'v15_BETA'] +choices_bc_ops = ['v14', 'NL3', 'ICA', 'GM4', 'All'] -choices_bc_cmd = ['NL3', 'ICA', 'GM4', 'v11', 'v12', 'v13','v14', 'v15_BETA'] +choices_bc_all = ['v14', 'v13', 'v12', 'v11', 'NL3', 'ICA', 'GM4'] choices_omodel = ['data', 'MOM5', 'MOM6'] @@ -53,7 +52,7 @@ # the following needs more cleanup; e.g., first define list of SGxxx names and parameters (i.e., STRETCH_GRID), # then assemble message_stretch and choices_stretch using this definition -message_stretch = f'''\n +message_stretch = '''\n Select parameters of stretched cubed-sphere grid for new restarts: Name Stretch_Factor Focus_Lat Focus_Lon @@ -81,34 +80,35 @@ message_bc_base_in = "BCs base directory for input restarts: \n" message_bc_base_new = "BCs base directory for new restarts: \n" -message_bc_ops = f'''\n +message_bc_ops = '''\n BCs version | ADAS tags | GCM tags typically used with BCs version -----------------|----------------------|----------------------------------------- - v12: v12 | future | 12.0 ... present - NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... 11.7 + GM4: Ganymed-4 | 5_12_4 (MERRA-2) | MERRA-2 ICA: Icarus | 5_17_0 ... 5_24_0_p1 | Icarus, Jason ... 10.18 + NL3: Icarus-NLv3 | 5_25_1 ... present | Icarus_NL, 10.19 ... 11.9 + v14: v14 | future | 11.11 ... present ---------------------------------------------------------------------------------- - Other: Additional choices used in model or DAS development.\n\n''' + All: All choices, incl. older BCs used only in model or DAS development.\n\n''' message_bc_ops_in = ("Select boundary conditions (BCs) version of input restarts:\n" + message_bc_ops) message_bc_ops_new = ("Select boundary conditions (BCs) version for new restarts:\n" + message_bc_ops) -# v12 used to be in the "other" section but is now the default, but we preserve this information -# in a comment since v14 will probably soon be the new default and we'll want to move v12 back to "other". -#v12: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n - -message_bc_other = f'''\n - v11: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2\n - v12: NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix \n - v13: As in v12 + mean land elevation fix + MOM6 v2 (OM4) ocean-seaice bathymetry \n - v14: As in v13 but with GPM 2.0 peat \n - v15_BETA: As in v14 but with v2 topography for atmosphere \n\n''' +message_bc_all = '''\n + BCs version | GCM tags | Description + ------------|----------------------------|------------------------------------------------------------ + GM4 | MERRA-2 | See Mahanama et al. 2015, GMAO TM Vol. 39 (Section 1: "last decade") + ICA | Icarus, Jason ... 10.18 | As in GM4 but with minor bug fixes + NL3 | Icarus_NL, 10.19 ... 11.9 | See Mahanama et al. 2015, GMAO TM Vol. 39 (Section 1: "updates") + v11 | testing only | NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + v12 | 11.10 | NL3 + JPL veg height + PEATMAP + MODIS snow alb v2 + Argentina peatland fix + v13 | testing only | As in v12 + mean land elevation fix + MOM6 v2 (OM4) bathymetry + v14 | 11.11 ... | As in v13 but with GPM 2.0 peat \n\n''' -message_bc_other_in = ("Select BCs version of input restarts:\n" + message_bc_other) -message_bc_other_new = ("Select BCs version for new restarts:\n" + message_bc_other) +message_bc_all_in = ("Select BCs version of input restarts:\n" + message_bc_all) +message_bc_all_new = ("Select BCs version for new restarts:\n" + message_bc_all) -message_agrid_list = f''' +message_agrid_list = ''' C12 C180 C1120 C24 C360 C1440 C48 C720 C2880 @@ -166,7 +166,7 @@ def init_merra2(x): yyyymm = int(x.get('input:shared:yyyymmddhh')[0:6]) if yyyymm < 197901 : - exit("Error. MERRA-2 data < 1979 not available\n") + sys.exit("Error. MERRA-2 data < 1979 not available\n") elif (yyyymm < 199201): expid = "d5124_m2_jan79" elif (yyyymm < 200106): @@ -202,7 +202,7 @@ def init_geosit(x): yyyymm = int(x.get('input:shared:yyyymmddhh')[0:6]) if yyyymm < 199701: - exit("Error. GEOS-IT data < 1997 not available\n") + sys.exit("Error. GEOS-IT data < 1997 not available\n") elif 199612 <= yyyymm <= 200712: expid = "d5294_geosit_jan98" elif 200801 <= yyyymm < 201801: @@ -210,7 +210,7 @@ def init_geosit(x): elif yyyymm >= 201801: expid = "d5294_geosit_jan18" # For any date starting from 201801 and beyond else: - exit("Error. GEOS-IT data not available for this date\n") + sys.exit("Error. GEOS-IT data not available for this date\n") x['input:shared:expid'] = expid x['input:shared:omodel'] = 'data' @@ -228,7 +228,7 @@ def init_geosit(x): def fvcore_info(x): - if 'input:shared:agrid' in x.keys(): + if 'input:shared:agrid' in x: return True rst_dir = x.get('input:shared:rst_dir') if not rst_dir : return False @@ -258,7 +258,7 @@ def fvcore_info(x): ymdh = fvrst.variables['time'].units.split('since ')[1].split(":")[0].replace('-','').replace(' ', "") if (lat != lon*6) : print("(N_lon, N_lat) = ", lon, lat) - exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') + sys.exit('This is not a cubed-sphere grid fvcore restart. Please contact SI team') x['input:shared:yyyymmddhh'] = ymdh x['input:shared:agrid'] = "C"+str(lon) print("The input fvcore restart has atm grid: " + "C" + str(lon) + '\n') @@ -283,12 +283,10 @@ def fvcore_info(x): elif f_ == STRETCH_GRID['SG002'] : x['input:shared:stretch'] = 'SG002' else: - exit("This stretched cubed-sphere grid is not supported ") + sys.exit("This stretched cubed-sphere grid is not supported ") return True def catch_model(x): - ymdh = x['input:shared:yyyymmddhh'] - time = ymdh[0:8] + '_'+ymdh[8:10] rst_dir = x.get('input:shared:rst_dir') if not rst_dir : return False @@ -367,24 +365,23 @@ def get_zoom(x): zoom_ = '2' return zoom_ agrid = None - if 'input:shared:agrid' in x.keys(): + if 'input:shared:agrid' in x: agrid = x.get('input:shared:agrid') - elif 'input' in x.keys(): + elif 'input' in x: agrid = x['input']['shared']['agrid'] - if agrid: - if (agrid[0].upper() == 'C'): # for cube-sphere: agrid = C90, C180, C1440, ... - lat = int(agrid[1:]) - zoom = lat /90.0 - if zoom < 1 : zoom = 1 - if zoom > 8 : zoom = 8 - zoom_= str(int(zoom)) + if agrid and agrid[0].upper() == 'C': # for cube-sphere: agrid = C90, C180, C1440, ... + lat = int(agrid[1:]) + zoom = lat /90.0 + zoom = max(zoom, 1) + zoom = min(zoom, 8) + zoom_= str(int(zoom)) return zoom_ def get_account(): cmd = 'id -gn' p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - (accounts, err) = p.communicate() - p_status = p.wait() + (accounts, _) = p.communicate() + p.wait() accounts = accounts.decode().split() return accounts[0] @@ -400,6 +397,7 @@ def config_to_yaml(config, yaml_file, noprompt = False): shutil.move(yaml_file, new_name) break yaml = ruamel.yaml.YAML() + yaml.width = 4096 out_dir = os.path.dirname(yaml_file) if not os.path.exists(out_dir) : os.mkdir(out_dir) with open(yaml_file, "w") as f: @@ -555,10 +553,10 @@ def get_command_line_from_answers(answers): def flatten_nested(nested_dict, result=None, prefix=''): if result is None: - result = dict() + result = {} for k, v in nested_dict.items(): - new_k = ':'.join((prefix, k)) if prefix else k - if not (isinstance(v, dict) or isinstance(v, OrderedDict)): + new_k = f'{prefix}:{k}' if prefix else k + if not (isinstance(v, (dict, OrderedDict))): result.update({new_k: v}) else: flatten_nested(v, result, new_k) @@ -597,13 +595,13 @@ def get_config_from_answers(answers, config_tpl = False): config[keys[0]][keys[1]][keys[2]] = value # Do the inputs have a split saltwater? - input_bc_version = config['input']['shared'].get('bc_version') + input_bc_version = config['input']['shared'].get('bc_version', '') config['input']['surface']['split_saltwater'] = True if 'Ganymed' in input_bc_version or 'GM4' in input_bc_version: config['input']['surface']['split_saltwater'] = False # Do the outputs need a split saltwater? - output_bc_version = config['output']['shared'].get('bc_version') + output_bc_version = config['output']['shared'].get('bc_version', '') config['output']['surface']['split_saltwater'] = True if 'Ganymed' in output_bc_version or 'GM4' in output_bc_version: config['output']['surface']['split_saltwater'] = False @@ -612,11 +610,13 @@ def get_config_from_answers(answers, config_tpl = False): def get_resolutions(agrid=None, ogrid=None, omodel=None, stretch=None, grid=None): if grid is not None : return grid + if agrid is None or ogrid is None: + raise ValueError("get_resolutions needs agrid and ogrid unless grid is given") aname = '' oname = '' if (agrid[0].upper() == 'C'): n = int(agrid[1:]) - aname ='CF{:04d}x6C'.format(n) + aname =f'CF{n:04d}x6C' if (ogrid[0].upper() == 'C'): oname = aname @@ -625,11 +625,11 @@ def get_resolutions(agrid=None, ogrid=None, omodel=None, stretch=None, grid=None s0 = int(xy[0]) s1 = int(xy[1]) if omodel == 'data': - oname = 'DE{:04d}xPE{:04d}'.format(s0,s1) + oname = f'DE{s0:04d}xPE{s1:04d}' if omodel == 'MOM5': - oname = 'M5TP{:04d}x{:04d}'.format(s0,s1) + oname = f'M5TP{s0:04d}x{s1:04d}' if omodel == 'MOM6': - oname = 'M6TP{:04d}x{:04d}'.format(s0,s1) + oname = f'M6TP{s0:04d}x{s1:04d}' if stretch: aname = aname + '-' + stretch @@ -640,8 +640,8 @@ def get_resolutions(agrid=None, ogrid=None, omodel=None, stretch=None, grid=None def get_default_bc_base(): cmd = 'uname -n' p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - (node, err) = p.communicate() - p_status = p.wait() + (node, _) = p.communicate() + p.wait() node = node.decode().split() node0 = node[0] discover = ['dirac', 'borg','warp', 'discover'] diff --git a/pre/remap_restart/ruff.toml b/pre/remap_restart/ruff.toml new file mode 100644 index 00000000..526dc971 --- /dev/null +++ b/pre/remap_restart/ruff.toml @@ -0,0 +1,16 @@ +target-version = "py312" +line-length = 120 + +[lint] +select = ["E", "F", "W", "B", "UP", "SIM", "I"] +ignore = [ + "E501", # long lines + "E701", # multiple statements (colon) + "E702", # multiple statements (semicolon) + "W291", # trailing whitespace + "W293", # whitespace on blank lines + "UP015", # redundant open modes + "SIM105", # contextlib.suppress + "SIM108", # ternary + "B905", # zip without strict +] diff --git a/pre/remap_restart/tests/test_remap_restarts.py b/pre/remap_restart/tests/test_remap_restarts.py index 56c1fd36..16acd1c2 100755 --- a/pre/remap_restart/tests/test_remap_restarts.py +++ b/pre/remap_restart/tests/test_remap_restarts.py @@ -5,19 +5,21 @@ # Newer GEOS code should load a module with GEOSpyD Python3 if not run: # module load python/GEOSpyD/Min4.10.3_py3.9 # -import os -import sys, getopt -import ruamel.yaml -import questionary import glob +import os +import shlex +import subprocess import subprocess as sp -import remap_restarts +import sys + +import ruamel.yaml + +from remap_analysis import analysis +from remap_catchANDcn import catchANDcn +from remap_other_restarts import other_restarts +from remap_upper import upperair +from remap_utils import GEOS_SITE, get_account -from remap_utils import * -from remap_upper import * -from remap_other_restarts import * -from remap_analysis import * -from remap_catchANDcn import * def compare(base, result): #1) comparing nc4 @@ -35,8 +37,8 @@ def compare(base, result): cmd = NCCMP + ' -dmgs '+ b + ' ' + r print(cmd) p = sp.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - (out, err) = p.communicate() - rc = p.wait() + (out, _) = p.communicate() + p.wait() out = out.decode().split() if "identical." in out : print('identical') @@ -61,7 +63,7 @@ def test_remap(config): if GEOS_SITE != "NCCS" : print("Test baseline data are only available at NCCS. Please run tests on Discover.") - exit() + sys.exit() yaml = ruamel.yaml.YAML() stream ='' cases_yaml = 'test_remap_cases.yaml'