Ruff cleanup of remap_restart, bug fixes, and v14 as default BCs

.github/workflows/python-lint.yml

@@ -0,0 +1,32 @@
+name: Python Lint
+
+on:
+  pull_request:
+    paths:
+      - 'pre/remap_restart/**'
+      - '.github/workflows/python-lint.yml'
+
+jobs:
+  ruff:
+    name: ruff
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/ruff-action@v4.1.0
+        with:
+          src: pre/remap_restart
+
+  ty:
+    name: ty
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v10.3.0
+        with:
+          enable-cache: false
+      - name: Run ty
+        working-directory: pre/remap_restart
+        run: |
+          uv venv
+          uv pip install ty numpy netCDF4 scipy pyyaml ruamel.yaml questionary
+          uv run ty check --python .venv .

.gitignore

@@ -5,3 +5,10 @@
 *#
 .#*
 **/CVS/
+
+# Python
+__pycache__/
+*.py[cod]
+.ruff_cache/
+.pytest_cache/
+.venv/

CHANGELOG.md

@@ -11,11 +11,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added plots for EDMF
 - Added new column and cloud variables and fixed bugs.
 
+- Added `ruff.toml` for `pre/remap_restart`
+- Added a GitHub Action (`python-lint.yml`) that runs `ruff` and `ty` on `pre/remap_restart`
+- Added Python ignores (`__pycache__/`, `.ruff_cache/`, `.pytest_cache/`, `.venv/`) to the root `.gitignore`
+
 ### Changed
 - Updated plot seasons in plot.rc
+- Cleaned up `pre/remap_restart` Python code per ruff: explicit imports in place of star imports, context managers for file handling, `sys.exit()` in place of `exit()`, and other style fixes
+- `remap_restarts` now makes v14 the default BCs version in the interactive questions; v12 remains in the main menu, and v11, v13 and v15_BETA remain under "Other"
+- `remap_restarts` now removes its scratch directories (`InData/`, `OutData/`, `upper_data/`, `ana_data/`) after a successful remap
+- Subprocess failures in `remap_restarts` now raise an error instead of being ignored
+- `remap_params.yaml` is now written without line wrapping
 
 ### Fixed
 
+- Fixed `shutil.remove` calls (now `os.remove`) in `remap_other_restarts.py`
+- Fixed a `TypeError` in `remap_utils.py` when `bc_version` is missing
+- Fixed a trailing space after `bc_base:` in the generated `remap_params.yaml`
+- `get_resolutions` in `remap_utils.py` now raises a clear error if `agrid` or `ogrid` is missing
+
 ### Removed
 
 ### Deprecated

pre/remap_restart/remap_analysis.py

@@ -8,17 +8,17 @@
 #   source g5_modules.sh    [bash]
 #   source g5_modules       [csh]
 #
+import fnmatch
+import glob
 import os
-from datetime import datetime, timedelta
-import subprocess
 import shlex
 import shutil
-import glob
-import fileinput
-import ruamel.yaml
+import subprocess
+from datetime import datetime, timedelta
+
 from remap_base import remap_base
-from remap_utils import get_label, MERRA2_RST_BASE
-import fnmatch
+from remap_utils import MERRA2_RST_BASE, get_label
+
 
 class analysis(remap_base):
   def __init__(self, **configs):
@@ -56,11 +56,7 @@ def remap(self):
      os.chdir(tmpdir)
 
      yyyymmddhh_ = str(config['input']['shared']['yyyymmddhh'])
-     yyyy_ = yyyymmddhh_[0:4]
-     mm_   = yyyymmddhh_[4:6]
-     dd_   = yyyymmddhh_[6:8]
      hh_   = yyyymmddhh_[8:10]
-     rst_time = datetime(year=int(yyyy_), month=int(mm_), day=int(dd_), hour = int(hh_))
      expid_in  = config['input']['shared']['expid']
      expid_out = config['output']['shared']['expid']
      if (expid_out) :
@@ -83,13 +79,10 @@ def remap(self):
        f_tmp = tmpdir+'/'+out_name
        local_fs.append(f_tmp)
        shutil.copy(f,f_tmp)
-       if out_name.find('satbias') != -1 :
-         if (aqua):
-            f_ = open(f_tmp, 'w')
-            for line in fileinput.input(f):
+       if out_name.find('satbias') != -1 and aqua:
+            with open(f) as f_in, open(f_tmp, 'w') as f_:
                # This is unlikely to be the case with new files
-               f_.write(line.replace('airs281SUBSET_aqua', 'airs281_aqua      '))
-            f_.close()
+               f_.writelines(line.replace('airs281SUBSET_aqua', 'airs281_aqua      ') for line in f_in)
 
      nlevel    = config['output']['air']['nlevel']
      agrid_out = config['output']['shared']['agrid']
@@ -102,7 +95,7 @@ def remap(self):
           shutil.move(f,f_orig)
           cmd = bindir + '/dyn2dyn.x ' + flags + ' -o ' + f + ' ' + f_orig
           print(cmd)
-          subprocess.call(shlex.split(cmd))
+          subprocess.run(shlex.split(cmd), check=True)
 
      for f in local_fs:
        fname = os.path.basename(f)
@@ -124,9 +117,11 @@ def remap(self):
        rstlcvOut = out_dir+'/'+expid_out+'rst.lcv.'+ymd_+'_'+hh_+'z.bin' + label
        cmd = bindir+'/mkdrstdate.x ' + ymd_ + ' ' + hms_ +' ' + rstlcvOut
        print(cmd)
-       subprocess.call(shlex.split(cmd))
+       subprocess.run(shlex.split(cmd), check=True)
      print( "cd " + cwdir)
      os.chdir(cwdir)
+     print("remove " + tmpdir)
+     shutil.rmtree(tmpdir)
 
      if self.config['input']['shared']['MERRA-2']:
         self.remove_merra2()
@@ -187,10 +182,10 @@ def copy_merra2(self):
        delt = timedelta(hours = h-3)
        new_time = rst_time + delt
        yyyy = "Y"+str(new_time.year)
-       mm   = 'M%02d'%new_time.month
-       ymd  = '%04d%02d%02d'%(new_time.year,new_time.month, new_time.day)
-       hh   = '%02d'%h
-       newhh= '%02d'%new_time.hour
+       mm   = f'M{new_time.month:02d}'
+       ymd  = f'{new_time.year:04d}{new_time.month:02d}{new_time.day:02d}'
+       hh   = f'{h:02d}'
+       newhh= f'{new_time.hour:02d}'
        m2_rst_dir = merra_2_rst_dir.replace('Y'+yyyy_,yyyy).replace('M'+mm_,mm)
        # bkg files
        for ftype in ['sfc', 'eta']:
@@ -231,8 +226,8 @@ def copy_merra2(self):
     delt = timedelta(hours = 3)
     new_time = rst_time - delt
     yyyy = "Y"+str(new_time.year)
-    mm   = 'M%02d'%new_time.month
-    ymdh = '%04d%02d%02d%02d'%(new_time.year, new_time.month, new_time.day, new_time.hour)
+    mm   = f'M{new_time.month:02d}'
+    ymdh = f'{new_time.year:04d}{new_time.month:02d}{new_time.day:02d}{new_time.hour:02d}'
     m2_rst_dir = merra_2_rst_dir.replace('Y'+yyyy_,yyyy).replace('M'+mm_,mm)
     fname = expid+'.trak.GDA.rst.'+ymdh+'z.txt'
     f = m2_rst_dir+'/'+fname

pre/remap_restart/remap_base.py

@@ -4,12 +4,16 @@
 #  remap_base.py does [???]
 #
 import os
-import ruamel.yaml
+import shlex
 import shutil
 import subprocess
-from remap_utils import *
 
-class remap_base(object):
+import netCDF4 as nc
+
+from remap_utils import GEOSIT_RST_BASE, get_config_from_file
+
+
+class remap_base:
   def __init__(self, **configs):
      for key, value in configs.items():
         if (key == 'params_file'):
@@ -73,8 +77,7 @@ def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, s
        if diff != 0: return False
 
        # for catchment, even tile files are the same, if bc is different, it still need remap
-       if (catch) :
-          if in_bc_version != out_bc_version : return False
+       if catch and in_bc_version != out_bc_version : return False
 
        expid = config['output']['shared']['expid']
        if (expid) :
@@ -88,7 +91,7 @@ def copy_without_remap(self, restarts_in, compared_file_in, compared_file_out, s
          f = expid + os.path.basename(rst).split('_rst')[0].split('.')[-1]+'_rst.'+suffix
          cmd = '/bin/cp ' + rst + ' ' + out_dir+'/'+f
          print('\n'+cmd)
-         subprocess.call(shlex.split(cmd))
+         subprocess.run(shlex.split(cmd), check=True)
 
        return True
 

pre/remap_restart/remap_bin2nc.py

@@ -7,10 +7,11 @@
 # remapped.  The set of yaml config files "bin2nc_merra2_*.yaml" include hard-wired
 # variables names, long names, and tile/grid dimensions specific to MERRA-2 restarts.
 
-from yaml import safe_load, load, dump
-from netCDF4 import Dataset
 import numpy as np
+from netCDF4 import Dataset
 from scipy.io import FortranFile
+from yaml import dump, safe_load
+
 
 def writeCVars(coordVars,dimensions):
 
@@ -73,36 +74,36 @@ def defineDimVars(fid,dims):
    for d in dims:
        if d=="lon":
           newVar=fid.createVariable(d,'f8',d)
-          setattr(newVar,'units','degrees_east')
-          setattr(newVar,'long_name','Longitude')
+          newVar.units = 'degrees_east'
+          newVar.long_name = 'Longitude'
           coordVars.update([(d,newVar)])
        if d=="lat":
           newVar=fid.createVariable(d,'f8',d)
-          setattr(newVar,'units','degrees_north')
-          setattr(newVar,'long_name','Latitude')
+          newVar.units = 'degrees_north'
+          newVar.long_name = 'Latitude'
           coordVars.update([(d,newVar)])
        if d=="lev":
           newVar=fid.createVariable(d,'f8',d)
-          setattr(newVar,'units','layer')
-          setattr(newVar,'long_name','sigma_at_layer_midpoints')
-          setattr(newVar,'standard_name','atmosphere_hybrid_sigma_pressure_coordinate')
-          setattr(newVar,'coordinate','eta')
-          setattr(newVar,'positive','down')
-          setattr(newVar,'formulaTerms','ap: ak b: bk ps: ps p0: p00')
+          newVar.units = 'layer'
+          newVar.long_name = 'sigma_at_layer_midpoints'
+          newVar.standard_name = 'atmosphere_hybrid_sigma_pressure_coordinate'
+          newVar.coordinate = 'eta'
+          newVar.positive = 'down'
+          newVar.formulaTerms = 'ap: ak b: bk ps: ps p0: p00'
           coordVars.update([(d,newVar)])
        if d=="edge":
           newVar=fid.createVariable(d,'f8',d)
-          setattr(newVar,'units','level')
-          setattr(newVar,'long_name','sigma_at_layer edge')
-          setattr(newVar,'standard_name','atmosphere_hybrid_sigma_pressure_coordinate')
-          setattr(newVar,'coordinate','eta')
-          setattr(newVar,'positive','down')
-          setattr(newVar,'formulaTerms','ap: ak b: bk ps: ps p0: p00')
+          newVar.units = 'level'
+          newVar.long_name = 'sigma_at_layer edge'
+          newVar.standard_name = 'atmosphere_hybrid_sigma_pressure_coordinate'
+          newVar.coordinate = 'eta'
+          newVar.positive = 'down'
+          newVar.formulaTerms = 'ap: ak b: bk ps: ps p0: p00'
           coordVars.update([(d,newVar)])
        if d=="time":
           newVar=fid.createVariable(d,'f8',d)
-          setattr(newVar,'units','minutes since ')
-          setattr(newVar,'long_name','time')
+          newVar.units = 'minutes since '
+          newVar.long_name = 'time'
           coordVars.update([(d,newVar)])
 
    return coordVars
@@ -114,9 +115,8 @@ def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=Fal
      bintype=np.float64
 
 
-  f=open(yamlfile,'r')
-
-  data=safe_load(f)
+  with open(yamlfile,'r') as f:
+     data=safe_load(f)
 
   dimensions = data["dimensions"]
   variables = data["variables"]
@@ -142,8 +142,8 @@ def bin2nc(binfile, ncfile, yamlfile, isDouble=False, hasHeader=False, debug=Fal
       units = v['units']
       dims = v['dimension']
       newVar = ncfid.createVariable(sname,'f4',dims)
-      setattr(newVar,'long_name',lname)
-      setattr(newVar,'units',units)
+      newVar.long_name = lname
+      newVar.units = units
       vars.update([(sname,newVar)])
 
   writeCVars(cVars,dimensions)