diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/AGCM_c360.rc b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/AGCM_c360.rc index 74354241f..9db0dbaf0 100755 --- a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/AGCM_c360.rc +++ b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/AGCM_c360.rc @@ -2,11 +2,11 @@ # Atmospheric Model Configuration Parameters # ------------------------------------------ - NX: 24 - NY: 144 + NX: 60 + NY: 120 IOSERVER_NODES: 4 -NUM_BACKEND_PES: 6 +NUM_BACKEND_PES: 24 AGCM_IM: 360 AGCM_JM: 2160 @@ -254,17 +254,13 @@ RECORD_REF_TIME: >>>SWELL_FC6H_H<<< 120000 # ASSIMILATION_CYCLE: nnnnnn # (Default: 21600) # IAU_DIGITAL_FILTER: YES or NO # (Default: YES ) -# Typical MERRA-2 Regular REPLAY Configuration -# -------------------------------------------- - REPLAY_ANA_EXPID: >>>SWELL_FP_EXP<<< - REPLAY_ANA_LOCATION: >>>SWELL_SCRATCHDIR<<< +# Regular REPLAY Configuration (source set via met_replay_exp / met_replay_root) +# -------------------------------------------------------------------------------- + REPLAY_ANA_EXPID: >>>SWELL_REPLAY_EXPID<<< + REPLAY_ANA_LOCATION: >>>SWELL_REPLAY_LOCATION<<< REPLAY_MODE: Regular - REPLAY_FILE: >>>SWELL_FP_EXP<<<.ana.eta.%y4%m2%d2_%h200z.nc4 - - #REPLAY_ANA_EXPID: d5294_geosit_jan18 - #REPLAY_ANA_LOCATION: /home/dao_ops/d5294_geosit_jan18/run/.../archive - #REPLAY_MODE: Regular - #REPLAY_FILE: ana/Y%y4/M%m2/d5294_geosit_jan18.bkg.eta.%y4%m2%d2_%h200z.nc4 + REPLAY_FILE: >>>SWELL_REPLAY_FILE<<< + REPLAY_FILE09: >>>SWELL_REPLAY_FILE09<<< # 4DIAU (recreate analysis tendency on the fly) # --------------------------------------------- diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/AGCM_c90.rc b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/AGCM_c90.rc index 22c427565..128ccc06d 100644 --- a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/AGCM_c90.rc +++ b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/AGCM_c90.rc @@ -6,6 +6,7 @@ NY: 72 IOSERVER_NODES: 0 +NUM_BACKEND_PES: 8 AGCM_IM: 90 @@ -76,9 +77,9 @@ BERES_FILE_NAME: ExtData/g5gcm/gwd/newmfspectra40_dc25.nc # Ocean Model Configuration Parameters # ------------------------------------ OGCM.GRID_TYPE: LatLon - OGCM.GRIDNAME: PE360x180-DE - OGCM.IM_WORLD: 360 - OGCM.JM_WORLD: 180 + OGCM.GRIDNAME: PE2880x1440-DE + OGCM.IM_WORLD: 2880 + OGCM.JM_WORLD: 1440 OGCM.LM: 34 OGCM.POLE: 'PE' @@ -252,17 +253,13 @@ RECORD_REF_TIME: >>>SWELL_FC6H_H<<< 120000 # ASSIMILATION_CYCLE: nnnnnn # (Default: 21600) # IAU_DIGITAL_FILTER: YES or NO # (Default: YES ) -# Typical MERRA-2 Regular REPLAY Configuration -# -------------------------------------------- - REPLAY_ANA_EXPID: >>>SWELL_FP_EXP<<< - REPLAY_ANA_LOCATION: >>>SWELL_SCRATCHDIR<<< +# Regular REPLAY Configuration (source set via met_replay_exp / met_replay_root) +# -------------------------------------------------------------------------------- + REPLAY_ANA_EXPID: >>>SWELL_REPLAY_EXPID<<< + REPLAY_ANA_LOCATION: >>>SWELL_REPLAY_LOCATION<<< REPLAY_MODE: Regular - REPLAY_FILE: >>>SWELL_FP_EXP<<<.ana.eta.%y4%m2%d2_%h200z.nc4 - - #REPLAY_ANA_EXPID: d5294_geosit_jan18 - #REPLAY_ANA_LOCATION: /home/dao_ops/d5294_geosit_jan18/run/.../archive - #REPLAY_MODE: Regular - #REPLAY_FILE: ana/Y%y4/M%m2/d5294_geosit_jan18.bkg.eta.%y4%m2%d2_%h200z.nc4 + REPLAY_FILE: >>>SWELL_REPLAY_FILE<<< + REPLAY_FILE09: >>>SWELL_REPLAY_FILE09<<< # 4DIAU (recreate analysis tendency on the fly) # --------------------------------------------- diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/CAP_c90.rc b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/CAP_c90.rc index c74c6c77b..14e816fc1 100644 --- a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/CAP_c90.rc +++ b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/CAP_c90.rc @@ -13,7 +13,7 @@ END_DATE: 29990302 210000 JOB_SGMT: >>>SWELL_FC_LENGTH<<< # number of times you run the job/seg NUM_SGMT: 1 -HEARTBEAT_DT: 900 +HEARTBEAT_DT: 300 USE_SHMEM: 0 diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/GEOSCHEMchem_AnaSettings_CO.rc b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/GEOSCHEMchem_AnaSettings_CO.rc deleted file mode 100755 index c49ef338c..000000000 --- a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/GEOSCHEMchem_AnaSettings_CO.rc +++ /dev/null @@ -1,19 +0,0 @@ -SpeciesName: CO -Active: 0 -AnalysisFreq: 1 -AnalysisHour: 0 -AnalysisMinute: 0 -ForwardLooking: 0 -FileTemplate: >>>SWELL_RUNDIR<<>>SWELL_RUNDIR<<>>SWELL_RUNDIR<<>>SWELL_GEOSCF_FORECAST_EXPID<<< EXPDSC: JEDI -EXPSRC: GEOSCFv2/rc1/t13 - +EXPSRC: GEOSgcm-v10.23.0+GEOSChem-14.0.0 COLLECTIONS: '>>>SWELL_GEOSCF_FORECAST_COLLECTION<<<' -# 'geosgcm_prog' -# 'prog.eta' :: ################################################## @@ -14,128 +11,22 @@ COLLECTIONS: '>>>SWELL_GEOSCF_FORECAST_COLLECTION<<<' # list of COLLECTIONS for scripting purposes. # ################################################## -GRID_LABELS: PE90x540-CF - PE360x2160-CF - PC360x181-DC +GRID_LABELS: :: -PE360x2160-CF.GRID_TYPE: Cubed-Sphere -PE360x2160-CF.IM_WORLD: 360 -PE360x2160-CF.JM_WORLD: 2160 -PE360x2160-CF.POLE: PC -PE360x2160-CF.DATELINE: DC -PE360x2160-CF.LM: 72 - -PE90x540-CF.GRID_TYPE: Cubed-Sphere -PE90x540-CF.IM_WORLD: 90 -PE90x540-CF.JM_WORLD: 540 -PE90x540-CF.POLE: PC -PE90x540-CF.DATELINE: DC -PE90x540-CF.LM: 72 - -PC360x181-DC.GRID_TYPE: LatLon -PC360x181-DC.IM_WORLD: 360 -PC360x181-DC.JM_WORLD: 181 -PC360x181-DC.POLE: PC -PC360x181-DC.DATELINE: DC -PC360x181-DC.LM: 72 - - >>>SWELL_GEOSCF_FORECAST_COLLECTION<<<.template: '>>>SWELL_GEOSCF_FORECAST_TEMPLATE<<<', >>>SWELL_GEOSCF_FORECAST_COLLECTION<<<.archive: '%c/Y%y4', >>>SWELL_GEOSCF_FORECAST_COLLECTION<<<.format: 'CFIO', >>>SWELL_GEOSCF_FORECAST_COLLECTION<<<.mode: 'instantaneous' , - #>>>SWELL_GEOSCF_FORECAST_COLLECTION<<<.frequency: 010000, >>>SWELL_GEOSCF_FORECAST_COLLECTION<<<.frequency: >>>SWELL_FC_OUTPUT_FREQ<<<, >>>SWELL_GEOSCF_FORECAST_COLLECTION<<<.ref_time: 000000, - >>>SWELL_GEOSCF_FORECAST_COLLECTION<<<.grid_label: >>>SWELL_GEOSCF_JEDI_GRID<<< >>>SWELL_GEOSCF_FORECAST_COLLECTION<<<.fields: 'T' , 'DYN' , - 'U_DGRID;V_DGRID' , 'DYN' , 'ud;vd' , - 'U;V' , 'DYN' , 'ua;va' , 'DELP' , 'DYN' , 'PS' , 'DYN' , 'Q' , 'MOIST' , 'SPHU' , - 'QITOT' , 'AGCM' , 'qi' , - 'QLTOT' , 'AGCM' , 'ql' , 'PHIS' , 'AGCM' , 'phis' , 'SpeciesConc_O3' , 'GEOSCHEMCHEM' , 'O3' , 'SpeciesConc_NO' , 'GEOSCHEMCHEM' , 'NO' , 'SpeciesConc_NO2' , 'GEOSCHEMCHEM' , 'NO2' , 'SpeciesConc_CO' , 'GEOSCHEMCHEM' , 'CO' , - 'EMIS_CO' , 'GEOSCHEMCHEM' , - 'EMIS_CO_IND' , 'GEOSCHEMCHEM' , - 'EMIS_CO_POW' , 'GEOSCHEMCHEM' , - 'EMIS_CO_TRA' , 'GEOSCHEMCHEM' , - 'EMIS_CO_RES' , 'GEOSCHEMCHEM' , - 'EMIS_NO' , 'GEOSCHEMCHEM' , - 'EMIS_NO_IND' , 'GEOSCHEMCHEM' , - 'EMIS_NO_POW' , 'GEOSCHEMCHEM' , - 'EMIS_NO_TRA' , 'GEOSCHEMCHEM' , - 'EMIS_NO_RES' , 'GEOSCHEMCHEM' , - 'EMIS_NO2' , 'GEOSCHEMCHEM' , - 'EMIS_NO2_IND' , 'GEOSCHEMCHEM' , - 'EMIS_NO2_POW' , 'GEOSCHEMCHEM' , - 'EMIS_NO2_TRA' , 'GEOSCHEMCHEM' , - 'EMIS_NO2_RES' , 'GEOSCHEMCHEM' , - 'Met_PBLTHICK' , 'GEOSCHEMCHEM' , - :: - - - - - - geosgcm_prog.template: '%y4%m2%d2_%h2%n2z.nc4' , - geosgcm_prog.archive: '%c/Y%y4' , - geosgcm_prog.format: 'CFIO' , - geosgcm_prog.deflate: 1 , - geosgcm_prog.grid_label: PE90x540-CF , - geosgcm_prog.frequency: 060000 , - geosgcm_prog.vscale: 100.0 , - geosgcm_prog.vunit: 'hPa' , - geosgcm_prog.vvars: 'log(PLE)' , 'DYN' , - geosgcm_prog.levels: 1000 975 950 925 900 875 850 825 800 775 750 725 700 650 600 550 500 450 400 350 300 250 200 150 100 70 50 40 30 20 10 7 5 4 3 2 1 0.7 0.5 0.4 0.3 0.2 0.1 0.07 0.05 0.04 0.03 0.02 , - geosgcm_prog.fields: 'PHIS' , 'AGCM' , - 'SLP' , 'DYN' , - 'U;V' , 'DYN' , - 'T' , 'DYN' , - 'PS' , 'DYN' , - 'ZLE' , 'DYN' , 'H' , - 'OMEGA' , 'DYN' , - 'Q' , 'MOIST' , 'QV' , - 'QITOT' , 'AGCM' , 'QI' , - 'QLTOT' , 'AGCM' , 'QL' , - 'RH2' , 'MOIST' , 'RH' , - 'O3' , 'CHEMISTRY' , - :: - - - - prog.eta.template: '%y4%m2%d2_%h2%n2z.nc4' , - prog.eta.archive: '%c/Y%y4' , - prog.eta.format: 'CFIO' , - prog.eta.deflate: 1 , - prog.eta.grid_label: PC360x181-DC , - prog.eta.regrid_method: 'BILINEAR', - prog.eta.mode: 'instantaneous' , - prog.eta.frequency: 060000 , - prog.eta.duration: 060000 , - prog.eta.ref_time: 000000 , - prog.eta.fields: 'PHIS' , 'AGCM' , 'phis' , - 'SGH' , 'AGCM' , 'hs_stdv' , - 'TS' , 'SURFACE' , 'ts' , - 'FRLAND' , 'SURFACE' , 'frland' , - 'FRLANDICE' , 'SURFACE' , 'frlandice' , - 'FRLAKE' , 'SURFACE' , 'frlake' , - 'FROCEAN' , 'SURFACE' , 'frocean' , - 'FRACI' , 'SURFACE' , 'frseaice' , - 'PS' , 'DYN' , 'ps' , - 'DELP' , 'DYN' , 'delp' , - 'U;V' , 'DYN' , 'u;v' , - 'TV' , 'DYN' , 'tv' , - 'Q' , 'MOIST' , 'sphu' , - 'O3PPMV' , 'CHEMISTRY' , 'ozone' , - 'QITOT' , 'AGCM' , 'qitot' , - 'QLTOT' , 'AGCM' , 'qltot' , - 'QRTOT' , 'MOIST' , 'qrtot' , - 'QSTOT' , 'MOIST' , 'qstot' , - :: + :: \ No newline at end of file diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/fvcore_layout.rc b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/fvcore_layout.rc new file mode 100644 index 000000000..30dca5b84 --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/fvcore_layout.rc @@ -0,0 +1,74 @@ +&fv_core_nml + k_split = 2 + Make_NH = .F. + hydrostatic = .T. + do_sat_adj = .F. + z_tracer = .T. + +/ + +&main_nml +/ + +&test_case_nml + test_case = 13 +/ + +&fms_io_nml +/ + +&fms_nml + print_memory_usage=.true. + domains_stack_size = 24000000 + clock_grain='MODULE', + clock_flags='DETAILED', +/ + +&gfdl_cloud_microphysics_nml + sedi_transport = .true. + do_sedi_heat = .false. + rad_snow = .true. + rad_graupel = .true. + rad_rain = .true. + const_vi = .F. + const_vs = .F. + const_vg = .F. + const_vr = .F. + vi_max = 1. + vs_max = 2. + vg_max = 12. + vr_max = 12. + qi_lim = 1. + prog_ccn = .false. + do_qa = .true. + fast_sat_adj = .true. + tau_l2v = 1200. + tau_v2l = 150. + tau_g2v = 900. + rthresh = 7.e-6 ! This is a key parameter for cloud water + dw_land = 0.16 + dw_ocean = 0.10 + ql_gen = 1.0e-3 + ql_mlt = 1.0e-3 + qi0_crt = 8.0E-5 + qs0_crt = 1.0e-3 + tau_i2s = 1000. + c_psaci = 0.05 + c_pgacs = 0.01 + rh_inc = 0.30 + rh_inr = 0.30 + rh_ins = 0.30 + ccn_l = 300. + ccn_o = 100. + c_paut = 1.0 + c_cracw = 0.8 + use_ppm = .false. + use_ccn = .true. + mono_prof = .true. + z_slope_liq = .true. + z_slope_ice = .true. + de_ice = .false. + fix_negative = .true. + icloud_f = 1 + mp_time = 150. +/ diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/gcm_run_geoscf_c90.j b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/gcm_run_geoscf.j similarity index 76% rename from src/swell/configuration/jedi/interfaces/geos_cf/namelists/gcm_run_geoscf_c90.j rename to src/swell/configuration/jedi/interfaces/geos_cf/namelists/gcm_run_geoscf.j index 263ecd2ea..76c54c29e 100644 --- a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/gcm_run_geoscf_c90.j +++ b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/gcm_run_geoscf.j @@ -4,7 +4,7 @@ # Batch Parameters for Run Job ####################################################################### #SBATCH --time=00:40:00 -#SBATCH --nodes=8 --ntasks-per-node=108 +#SBATCH --nodes=>>>SWELL_SBATCH_NODES<<< --ntasks-per-node=108 #SBATCH --job-name=CFv2rc1t13 #SBATCH --output=CFv2_gcm_%j #SBATCH --constraint=mil @@ -44,9 +44,7 @@ echo VERSION: $GCMVER ####################################################################### -setenv EXPID GCv14.0_GCMv1.17_c90b -setenv EXPDIR >>>SWELL_GEOSRUN<<< -setenv HOMDIR $EXPDIR +setenv EXPID ForecastCF_c360_swell # Run GSI? set RUN_GSI = 0 @@ -78,14 +76,12 @@ set OGCM_JM = `grep '^\s*OGCM\.JM_WORLD:' $CYCLEDIR/AGCM.rc | cut -d: -f2` # Calculate number of cores/nodes for IOSERVER # -------------------------------------------- -set USE_IOSERVER = 0 -set AGCM_IOS_NODES = `grep '^\s*IOSERVER_NODES:' $CYCLEDIR/AGCM.rc | cut -d: -f2` +set USE_IOSERVER = 1 +set NUM_OSERVER_NODES = `grep '^\s*IOSERVER_NODES:' $CYCLEDIR/AGCM.rc | cut -d: -f2` +set NUM_BACKEND_PES = `grep '^\s*NUM_BACKEND_PES:' $CYCLEDIR/AGCM.rc | cut -d: -f2` -if ($USE_IOSERVER == 0) then - set IOS_NODES = 0 -else - set IOS_NODES = $AGCM_IOS_NODES -endif +if ( "$NUM_BACKEND_PES" == "" ) set NUM_BACKEND_PES = 2 +if ( $NUM_OSERVER_NODES == 0 ) set USE_IOSERVER = 0 # Check for Over-Specification of CPU Resources # --------------------------------------------- @@ -99,43 +95,47 @@ endif @ MODEL_NPES = $NX * $NY +set NCPUS_PER_NODE = 108 +set NUM_MODEL_NODES=`echo "scale=1;($MODEL_NPES / $NCPUS_PER_NODE)" | bc | awk 'function ceil(x, y){y=int(x); return(x>y?y+1:y)} {print ceil($1)}'` + if ( $NCPUS != NULL ) then if ( $USE_IOSERVER == 1 ) then - set NCPUS_PER_NODE = 40 + @ TOTAL_NODES = $NUM_MODEL_NODES + $NUM_OSERVER_NODES - @ NODES = `echo "( ($MODEL_NPES + $NCPUS_PER_NODE) + ($AGCM_IOS_NODES * $NCPUS_PER_NODE) - 1)/$NCPUS_PER_NODE" | bc` - @ NPES = $NODES * $NCPUS_PER_NODE + @ TOTAL_PES = $TOTAL_NODES * $NCPUS_PER_NODE - if( $NPES > $NCPUS ) then + if( $TOTAL_PES > $NCPUS ) then echo "CPU Resources are Over-Specified" echo "--------------------------------" echo "Allotted NCPUs: $NCPUS" - echo "Requested NCPUs: $NPES" + echo "Requested NCPUs: $TOTAL_PES" echo "" echo "Specified NX: $NX" echo "Specified NY: $NY" echo "" - echo "Specified IOSERVER_NODES: $AGCM_IOS_NODES" + echo "Specified model nodes: $NUM_MODEL_NODES" + echo "Specified oserver nodes: $NUM_OSERVER_NODES" echo "Specified cores per node: $NCPUS_PER_NODE" - echo "Exit with NPES > NCPUS 1" exit endif else - @ NPES = $MODEL_NPES + @ TOTAL_PES = $MODEL_NPES - if( $NPES > $NCPUS ) then + if( $TOTAL_PES > $NCPUS ) then echo "CPU Resources are Over-Specified" echo "--------------------------------" echo "Allotted NCPUs: $NCPUS" - echo "Requested NCPUs: $NPES" + echo "Requested NCPUs: $TOTAL_PES" echo "" echo "Specified NX: $NX" echo "Specified NY: $NY" - echo "Exit with NPES > NCPUS 2" + echo "" + echo "Specified model nodes: $NUM_MODEL_NODES" + echo "Specified cores per node: $NCPUS_PER_NODE" exit endif @@ -144,7 +144,7 @@ if ( $NCPUS != NULL ) then else # This is for the desktop path - @ NPES = $MODEL_NPES + @ TOTAL_PES = $MODEL_NPES endif @@ -210,10 +210,10 @@ set USE_SHMEM = `grep '^\s*USE_SHMEM:' CAP.rc | cut -d: -f2` # Link Boundary Datasets ####################################################################### echo " Link Boundary Datasets" -setenv BCSDIR /discover/nobackup/ltakacs/bcs/Icarus-NLv3/Icarus-NLv3_Reynolds +setenv BCSDIR /discover/nobackup/ltakacs/bcs/Icarus-NLv3/Icarus-NLv3_Ostia setenv SSTDIR /discover/nobackup/projects/gmao/share/gmao_ops/fvInput/g5gcm/bcs/realtime/OSTIA_REYNOLDS/2880x1440/ setenv CHMDIR /discover/nobackup/projects/gmao/share/gmao_ops/fvInput_nc3 -setenv BCRSLV CF0090x6C_DE0360xPE0180 +setenv BCRSLV >>>SWELL_BCRSLV<<< setenv DATELINE DC setenv EMISSIONS OPS_EMISSIONS @@ -278,9 +278,9 @@ endif _EOF_ - echo "/bin/ln -sf $SSTDIR/dataoceanfile_OSTIA_REYNOLDS_SST.2880x1440.2023.data sst.data" >> $FILE - echo "/bin/ln -sf $SSTDIR/dataoceanfile_OSTIA_REYNOLDS_ICE.2880x1440.2023.data fraci.data" >> $FILE - echo "/bin/ln -sf $SSTDIR/SEAWIFS_KPAR_mon_clim.2880x1440 SEAWIFS_KPAR_mon_clim.data" >> $FILE + echo "/bin/ln -sf $SSTDIR"'/dataoceanfile_OSTIA_REYNOLDS_SST.2880x1440.$YEAR.data sst.data' >> $FILE + echo "/bin/ln -sf $SSTDIR"'/dataoceanfile_OSTIA_REYNOLDS_ICE.2880x1440.$YEAR.data fraci.data' >> $FILE + echo "/bin/ln -sf $SSTDIR"'/SEAWIFS_KPAR_mon_clim.2880x1440 SEAWIFS_KPAR_mon_clim.data' >> $FILE chmod +x linkbcs # Done in prep_forecast @@ -292,55 +292,8 @@ chmod +x linkbcs echo " Get Executable and RESTARTS" cp $GEOSBIN/GEOSgcm.x . -set rst_files = `grep "RESTART_FILE" AGCM.rc | grep -v VEGDYN | grep -v "#" | cut -d ":" -f1 | cut -d "_" -f1-2` -set rst_file_names = `grep "RESTART_FILE" AGCM.rc | grep -v VEGDYN | grep -v "#" | cut -d ":" -f2` - -set chk_files = `grep "CHECKPOINT_FILE" AGCM.rc | grep -v "#" | cut -d ":" -f1 | cut -d "_" -f1-2` -set chk_file_names = `grep "CHECKPOINT_FILE" AGCM.rc | grep -v "#" | cut -d ":" -f2` - -set monthly_chk_names = `cat $CYCLEDIR/HISTORY.rc | grep -v '^[\t ]*#' | sed -n 's/\([^\t ]\+\).monthly:[\t ]*1.*/\1/p' | sed 's/$/_rst/' ` - -# Remove possible bootstrap parameters (+/-) -# ------------------------------------------ -set dummy = `echo $rst_file_names` -set rst_file_names = '' -foreach rst ( $dummy ) - set length = `echo $rst | awk '{print length($0)}'` - set bit = `echo $rst | cut -c1` - if( "$bit" == "+" | \ - "$bit" == "-" ) set rst = `echo $rst | cut -c2-$length` - set rst_file_names = `echo $rst_file_names $rst` -end - -# Copy Restarts to Scratch Directory -# ---------------------------------- -# Rsts moved to CYCLEDIR/scratch by getRSTGEOSCF -#foreach rst ( $rst_file_names $monthly_chk_names ) -# if(-e $CYCLEDIR/$rst ) cp $CYCLEDIR/$rst . & -#end -#wait - -# If any restart is binary, set NUM_READERS to 1 so that -# +-style bootstrapping of missing files can occur in -# MAPL. pbinary cannot do this, but pnc4 can. -# ------------------------------------------------------ -# maybe delete -set found_binary = 0 - -foreach rst ( $rst_file_names ) - if (-e $rst) then - set rst_type = `/usr/bin/file -Lb --mime-type $rst` - if ( $rst_type =~ "application/octet-stream" ) then - set found_binary = 1 - endif - endif -end - -if ($found_binary == 1) then - /bin/mv AGCM.rc AGCM.tmp - cat AGCM.tmp | sed -e "/^NUM_READERS/ s/\([0-9]\+\)/1/g" > AGCM.rc - /bin/rm AGCM.tmp -endif +# Restarts are staged into $CYCLEDIR/scratch by getRSTGEOSCF ahead of time, +# so no optional/bootstrap (+/-) restart handling is needed here. ################################################################## ###### @@ -485,7 +438,7 @@ endif # Link Boundary Conditions for Appropriate Date # --------------------------------------------- -setenv YEAR 2023 #$yearc +setenv YEAR $yearc ./linkbcs if (! -e tile.bin) then @@ -499,56 +452,14 @@ endif #ln -sf $SSTDIR/dataoceanfile_MERRA2_ICE.${OGCM_IM}x${OGCM_JM}.${yy}.data fraci.data ####################################################################### -# Split Saltwater Restart if detected +# Check for Split Saltwater Restarts ####################################################################### if ( (-e $SCRDIR/openwater_internal_rst) && (-e $SCRDIR/seaicethermo_internal_rst)) then echo "Saltwater internal state is already split, good to go!" else - if ( ( ( -e $SCRDIR/saltwater_internal_rst ) || ( -e $EXPDIR/saltwater_internal_rst) ) && ( $counter == 1 ) ) then - - echo "Found Saltwater internal state. Splitting..." - - # If saltwater_internal_rst is in EXPDIR move to SCRDIR - # ----------------------------------------------------- - if ( -e $EXPDIR/saltwater_internal_rst ) /bin/mv $EXPDIR/saltwater_internal_rst $SCRDIR - - # The splitter script requires an OutData directory - # ------------------------------------------------- - if (! -d OutData ) mkdir -p OutData - - # Run the script - # -------------- - $RUN_CMD 1 $GEOSBIN/SaltIntSplitter tile.data $SCRDIR/saltwater_internal_rst - - # Move restarts - # ------------- - /bin/mv OutData/openwater_internal_rst OutData/seaicethermo_internal_rst . - - # Remove OutData - # -------------- - /bin/rmdir OutData - - # Make decorated copies for restarts tarball - # ------------------------------------------ - cp openwater_internal_rst $EXPID.openwater_internal_rst.${edate}.${GCMVER}.${BCTAG}_${BCRSLV} - cp seaicethermo_internal_rst $EXPID.seaicethermo_internal_rst.${edate}.${GCMVER}.${BCTAG}_${BCRSLV} - - # Inject decorated copies into restarts tarball - # --------------------------------------------- - tar rf $CYCLEDIR/restarts/restarts.${edate}.tar $EXPID.*.${edate}.${GCMVER}.${BCTAG}_${BCRSLV} - - # Remove the decorated restarts - # ----------------------------- - /bin/rm $EXPID.*.${edate}.${GCMVER}.${BCTAG}_${BCRSLV} - - # Remove the saltwater internal restart - # ------------------------------------- - /bin/rm $SCRDIR/saltwater_internal_rst - else - echo "Neither saltwater_internal_rst, nor openwater_internal_rst and seaicethermo_internal_rst were found. Abort!" - exit 6 - endif + echo "openwater_internal_rst and seaicethermo_internal_rst were not found. Abort!" + exit 6 endif # Test Openwater Restart for Number of tiles correctness @@ -597,17 +508,13 @@ endif # Environment variables for MPI, etc # ---------------------------------- -# potential ewok problem... + +# OFI Provider Selection and Tuning +setenv I_MPI_FABRICS shm:ofi # Use shared memory and OFI +setenv I_MPI_OFI_PROVIDER psm3 # Specify the PSM3 OFI provider setenv I_MPI_ADJUST_ALLREDUCE 12 setenv I_MPI_ADJUST_GATHERV 3 -# This flag prints out the Intel MPI state. Uncomment if needed -#setenv I_MPI_DEBUG 9 -setenv I_MPI_SHM_HEAP_VSIZE 512 -setenv PSM2_MEMORY large -setenv I_MPI_EXTRA_FILESYSTEM 1 -setenv I_MPI_EXTRA_FILESYSTEM_FORCE gpfs - # Run bundleParser.py #--------------------- @@ -652,40 +559,14 @@ setenv OMP_NUM_THREADS 1 if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null if( $USE_IOSERVER == 1 ) then - set IOSERVER_OPTIONS = "--npes_model $MODEL_NPES --nodes_output_server $IOS_NODES" - - # Per SI Team, the multigroup server should always be used - # The ideal number of backend PEs is based on the number of HISTORY - # collections and number of IO nodes - - # First we figure out the number of collections in the HISTORY.rc (this is not perfect, but is close to right) - set NUM_HIST_COLS = `cat HISTORY.rc | sed -n '/^COLLECTIONS:/,/^ *::$/{p;/^ *::$/q}' | grep -v '^ *#' | wc -l` - - # Protect against divide by zero - if ($IOS_NODES == 0) then - echo "Something is wrong. IOSERVER asked for, but zero IO nodes provided" - exit 3 - endif - - # Now we divide that number of collections by the ioserver nodes - echo "NUM_BACKEND_PES = $NUM_BACKEND_PES" - set NUM_BACKEND_PES = `echo "scale=1;(($NUM_HIST_COLS - 1) / $IOS_NODES)" | bc | awk '{print int($1 + 0.5)}'` - - # Finally multigroup requires at least two backend pes - if ($NUM_BACKEND_PES < 2) set NUM_BACKEND_PES = 2 - - echo "IOSERVER_EXTRA = $IOSERVER_EXTRA" - set IOSERVER_EXTRA = "--oserver_type multigroup --npes_backend_pernode $NUM_BACKEND_PES" + set IOSERVER_OPTIONS = "--npes_model $MODEL_NPES --nodes_output_server $NUM_OSERVER_NODES" + set IOSERVER_EXTRA = "--oserver_type multigroup --npes_backend_pernode $NUM_BACKEND_PES" else set IOSERVER_OPTIONS = "" - set IOSERVER_EXTRA = "" + set IOSERVER_EXTRA = "" endif - - #set +e - #$RUN_CMD $NPES ./GEOSgcm.x $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml' - $RUN_CMD $NPES ./GEOSgcm.x --logging_config 'logging.yaml' - #exit_code=$? - #set -e + + $RUN_CMD $TOTAL_PES ./GEOSgcm.x $IOSERVER_OPTIONS $IOSERVER_EXTRA --logging_config 'logging.yaml' if( $USE_SHMEM == 1 ) $GEOSBIN/RmShmKeys_sshmpi.csh >& /dev/null diff --git a/src/swell/configuration/jedi/interfaces/geos_cf/namelists/geoschem_analysis/co_analysis.yaml b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/geoschem_analysis/co_analysis.yaml new file mode 100644 index 000000000..879baffa4 --- /dev/null +++ b/src/swell/configuration/jedi/interfaces/geos_cf/namelists/geoschem_analysis/co_analysis.yaml @@ -0,0 +1,21 @@ +### Species fragment assembled into geoschem_analysis.yml by prep_forecast_cf.py. +### 'Active' is set programmatically based on the analysis_variables config question. + +SpeciesName: CO +AnalysisFreq: 6 +AnalysisHour: 0 +AnalysisMinute: 0 +ForwardLooking: false +FileTemplate: '>>>SWELL_RUNDIR<<>>SWELL_RUNDIR<<>>SWELL_RUNDIR<< None: self.resolution = self.config.horizontal_resolution() self.an_vars_long = self.config.analysis_variables() - self.geos_cf_run_dir = self.config.geos_cf_run_dir() + self.geos_cf_rc_dir = self.config.geos_cf_rc_dir() self.geos_cf_install_dir = self.config.geos_cf_install_dir() self.namelists_dir = os.path.join(self.experiment_config_path(), 'jedi', 'interfaces', 'geos_cf', 'namelists') - self.fp_exp = self.config.geosfp_exp() - self.fp_loc = self.config.geosfp_path() + self.met_replay_exp = self.config.met_replay_exp() + self.met_replay_root = self.config.met_replay_root() # Derive window times # ------------------- @@ -70,25 +71,39 @@ def execute(self) -> None: self.parse_wlen = isodate.parse_duration(self.window_length) self.parse_fclen = isodate.parse_duration(self.forecast_length) - # Determine analysis variables for GEOS-CF (NO2 and CO) - # ------------------------------------------------------- - an_vars_long_tg = ['volume_mixing_ratio_of_no2', - 'volume_mixing_ratio_of_co', - 'volume_mixing_ratio_of_o3'] + # Determine active GEOS-Chem composition species for this experiment: every + # '_analysis.yaml' file under namelists/geoschem_analysis/ is a candidate, + # and it is active if 'volume_mixing_ratio_of_' is in analysis_variables. + # ------------------------------------------------------------------------------------- + geoschem_analysis_dir = os.path.join(self.namelists_dir, 'geoschem_analysis') + self.geoschem_species_files = sorted( + f for f in os.listdir(geoschem_analysis_dir) if f.endswith('_analysis.yaml') + ) + self.an_vars_compo = [] - for an_var in an_vars_long_tg: - if an_var in self.an_vars_long: - self.an_vars_compo.append(an_var.split('_')[-1].upper()) + for fname in self.geoschem_species_files: + species = fname[:-len('_analysis.yaml')] + if f'volume_mixing_ratio_of_{species}' in self.an_vars_long: + self.an_vars_compo.append(species.upper()) # Section 2: Create GEOS-CF increment files # ------------------------------------------ self.logger.info('Creating GEOS-CF increment files') self.create_geos_cf_increments() - # Section 3: Get GEOS FP analysis files for replay - # ----------------------------------------------- - self.logger.info('Fetching GEOS FP analysis files for replay') - self.get_geosfp_replay_files() + # Section 3: Get meteorology replay analysis files + # ------------------------------------------------- + self.logger.info('Fetching meteorology replay analysis files') + self.get_met_replay_files() + + # Meteorology REPLAY settings for AGCM.rc: files are staged into scratch_dir + # under their original name, so location/file are the same for any source + # (GEOS FP, GEOS-IT, ...). + # --------------------------------------------------------------------------- + self.replay_expid = self.met_replay_exp + self.replay_location = self.scratch_dir + self.replay_file = f'{self.met_replay_exp}.ana.eta.%y4%m2%d2_%h200z.nc4' + self.replay_file09 = self.replay_file # Section 4: Copy and update GEOS-CF namelist files # --------------------------------------------------- @@ -141,8 +156,39 @@ def create_geos_cf_increments(self) -> None: # ---------------------------------------------------------------------------------------------- - def get_geosfp_replay_files(self) -> None: - """Fetch GEOS FP analysis files needed for replay over the forecast length.""" + def write_geoschem_analysis_yml(self, analysis_dst: str) -> None: + """Assemble geoschem_analysis.yml from one YAML fragment per species. + """ + + species_dir = os.path.join(self.namelists_dir, 'geoschem_analysis') + + species_config = {} + for i, fname in enumerate(self.geoschem_species_files, start=1): + with open(os.path.join(species_dir, fname), 'r') as f: + species_def = yaml.safe_load(f) + species_def['Active'] = species_def['SpeciesName'] in self.an_vars_compo + species_config[f'Spc{i:03d}'] = species_def + + analysis_config = { + 'general': { + 'runphase': 2, + 'nspecies': len(self.geoschem_species_files), + }, + 'species': species_config, + } + + with open(analysis_dst, 'w') as f: + yaml.dump(analysis_config, f, sort_keys=False) + + # ---------------------------------------------------------------------------------------------- + + def get_met_replay_files(self) -> None: + """Fetch meteorology replay analysis files needed over the forecast length. + + Works for any replay source (GEOS FP, GEOS-IT, ...) since they share the same + archive layout: //run/.../archive/ana/Y%y4/M%m2/ + .ana.eta.%y4%m2%d2_%h200z.nc4 + """ n_win_step = math.ceil(self.parse_fclen / self.parse_wlen) for wstep in range(n_win_step): @@ -151,12 +197,12 @@ def get_geosfp_replay_files(self) -> None: anMM = fc_date.strftime('%m') anDD = fc_date.strftime('%d') anHH = fc_date.strftime('%H') - date_path = f'Y{anYYYY}/M{anMM}' - fp_path = f'{self.fp_loc}/{self.fp_exp}/ana/{date_path}' - fp_file = (f'{self.fp_exp}.ana.eta.' - f'{anYYYY}{anMM}{anDD}_{anHH}00z.nc4') - shutil.copy(f'{fp_path}/{fp_file}', self.scratch_dir) + replay_file = f'{self.met_replay_exp}.ana.eta.{anYYYY}{anMM}{anDD}_{anHH}00z.nc4' + replay_src = os.path.join(self.met_replay_root, self.met_replay_exp, 'run', '...', + 'archive', 'ana', f'Y{anYYYY}', f'M{anMM}', replay_file) + + shutil.copy(replay_src, self.scratch_dir) # ---------------------------------------------------------------------------------------------- @@ -167,14 +213,13 @@ def prepare_namelists(self) -> None: namelists_dir = self.namelists_dir resolution = self.resolution - fp_exp = self.fp_exp parse_wbegin = self.parse_wbegin parse_wend = self.parse_wend parse_fclen = self.parse_fclen - # Copy all static files in RC/ in GEOS-CF run directory to scratch - # ---------------------------------------------------------------- - src_rc = os.path.join(self.geos_cf_run_dir, 'RC') + # Copy all static files in GEOS-CF RC directory to scratch + # --------------------------------------------------------- + src_rc = self.geos_cf_rc_dir self.logger.info(f'Copy files from {src_rc} to {scratch_dir}') for item in os.listdir(src_rc): @@ -189,7 +234,7 @@ def prepare_namelists(self) -> None: # Copy template namelist files # -------------------------- for fname in ['logging.yaml', 'GEOSCHEMchem_ExtData.yaml', - 'HEMCO_Config.rc', 'geoschem_config.yml']: + 'HEMCO_Config.rc', 'geoschem_config.yml', 'fvcore_layout.rc']: src = os.path.join(namelists_dir, fname) self.logger.info(f'Copy {src} to {scratch_dir}') @@ -203,8 +248,10 @@ def prepare_namelists(self) -> None: # Update AGCM.rc placeholders # --------------------------- agcm_rc = os.path.join(scratch_dir, 'AGCM.rc') - self.replace_string(agcm_rc, '>>>SWELL_FP_EXP<<<', fp_exp) - self.replace_string(agcm_rc, '>>>SWELL_SCRATCHDIR<<<', scratch_dir) + self.replace_string(agcm_rc, '>>>SWELL_REPLAY_EXPID<<<', self.replay_expid) + self.replace_string(agcm_rc, '>>>SWELL_REPLAY_LOCATION<<<', self.replay_location) + self.replace_string(agcm_rc, '>>>SWELL_REPLAY_FILE09<<<', self.replay_file09) + self.replace_string(agcm_rc, '>>>SWELL_REPLAY_FILE<<<', self.replay_file) weYYYY = parse_wend.strftime('%Y') weMM = parse_wend.strftime('%m') @@ -226,18 +273,6 @@ def prepare_namelists(self) -> None: self.replace_string(history_dst, '>>>SWELL_GEOSCF_FORECAST_TEMPLATE<<<', history['template']) - if resolution == 'c90': - grid_label = 'PE90x540-CF' - elif resolution == 'c360': - grid_label = 'PE360x2160-CF' - else: - raise ValueError( - f'Unsupported horizontal resolution for ' - f'HISTORY.rc grid label: {resolution}' - ) - - self.replace_string(history_dst, '>>>SWELL_GEOSCF_JEDI_GRID<<<', grid_label) - freq_dur = isodate.parse_duration(self.forecast_output_frequency) freq_total_secs = int(freq_dur.total_seconds()) freq_hh = freq_total_secs // 3600 @@ -254,20 +289,11 @@ def prepare_namelists(self) -> None: shutil.copy(gridcomp_src, gridcomp_dst) self.replace_string(gridcomp_dst, '>>>SWELL_NUM_AN_VARS<<<', str(num_an_vars)) - for index, an_var in enumerate(self.an_vars_compo): - self.replace_string(gridcomp_dst, - f'#Analysis_Settings_Spec00{index + 1}:', - f'Analysis_Settings_Spec00{index + 1}: ' - f'GEOSCHEMchem_AnaSettings_{an_var}.rc') - - # Update GEOSCHEMchem_AnaSettings_.rc files placeholders - # ----------------------------------------------------------- - for an_var in self.an_vars_compo: - ana_src = os.path.join(namelists_dir, f'GEOSCHEMchem_AnaSettings_{an_var}.rc') - ana_dst = os.path.join(scratch_dir, f'GEOSCHEMchem_AnaSettings_{an_var}.rc') - if os.path.exists(ana_src): - shutil.copy(ana_src, ana_dst) - self.replace_string(ana_dst, '>>>SWELL_RUNDIR<<<', scratch_dir) + # Assemble geoschem_analysis.yml from per-species YAML fragments + # ----------------------------------------------------------------------------- + analysis_dst = os.path.join(scratch_dir, 'geoschem_analysis.yml') + self.write_geoschem_analysis_yml(analysis_dst) + self.replace_string(analysis_dst, '>>>SWELL_RUNDIR<<<', scratch_dir) # Write cap_restart with window begin date # ----------------------------------------- @@ -279,14 +305,24 @@ def prepare_namelists(self) -> None: # Copy and configure gcm_run.j # ----------------------------- - gcm_run_src = os.path.join(namelists_dir, f'gcm_run_geoscf_{resolution}.j') + if resolution == 'c90': + sbatch_nodes = '8' + bcrslv = 'CF0090x6C_DE2880xPE1440' + elif resolution == 'c360': + sbatch_nodes = '32' + bcrslv = 'CF0360x6C_DE2880xPE1440' + else: + raise ValueError( + f'Unsupported horizontal resolution for gcm_run_geoscf.j: {resolution}' + ) + + gcm_run_src = os.path.join(namelists_dir, 'gcm_run_geoscf.j') gcm_run_dst = os.path.join(scratch_dir, 'gcm_run_geoscf.j') shutil.copy(gcm_run_src, gcm_run_dst) self.replace_string(gcm_run_dst, '>>>SWELL_CYCLEDIR<<<', scratch_dir) self.replace_string(gcm_run_dst, '>>>SWELL_GEOSINSTALL<<<', self.geos_cf_install_dir) - - # used in handling Saltwater Restart only - self.replace_string(gcm_run_dst, '>>>SWELL_GEOSRUN<<<', self.geos_cf_run_dir) + self.replace_string(gcm_run_dst, '>>>SWELL_SBATCH_NODES<<<', sbatch_nodes) + self.replace_string(gcm_run_dst, '>>>SWELL_BCRSLV<<<', bcrslv) os.chmod(gcm_run_dst, 0o755) # Update CAP.rc with forecast length diff --git a/src/swell/tasks/task_questions.py b/src/swell/tasks/task_questions.py index 5474c5b89..5d9e6587b 100644 --- a/src/swell/tasks/task_questions.py +++ b/src/swell/tasks/task_questions.py @@ -624,12 +624,12 @@ class TaskQuestions(QuestionContainer, Enum): qd.forecast_length(), qd.forecast_output_frequency(), qd.geos_cf_install_dir(), - qd.geos_cf_run_dir(), - qd.geosfp_exp(), - qd.geosfp_path(), + qd.geos_cf_rc_dir(), qd.horizontal_resolution(), qd.iau(), qd.inc_template(), + qd.met_replay_exp(), + qd.met_replay_root(), qd.window_length(), qd.rst_experiment() ] diff --git a/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml b/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml index b36cfcc5a..d607c3fa1 100644 --- a/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml +++ b/src/swell/test/jedi_configs/jedi_3dvar_cf_config.yaml @@ -71,42 +71,6 @@ cost function: observations: get values: observers: - - obs space: - name: tempo_no2_tropo - obsdatain: - engine: - type: H5File - obsfile: cycle_dir/tempo_no2_tropo.20230805T150000Z.nc4 - obsdataout: - engine: - type: H5File - allow overwrite: true - obsfile: cycle_dir/experiment_id.tempo_no2_tropo.20230805T150000Z.nc4 - simulated variables: - - nitrogendioxideColumn - observed variables: - - nitrogendioxideColumn - monitoring only: false - obs filters: - - filter: RejectList - where: - - variable: - name: MetaData/cloudAmount - minvalue: 0.15 - - filter: RejectList - where: - - variable: - name: MetaData/solarZenithAngle - minvalue: 70 - obs operator: - name: ColumnRetrieval - nlayers_retrieval: 72 - tracer variables: - - volume_mixing_ratio_of_no2 - isApriori: false - isAveragingKernel: true - stretchVertices: topbottom - observation_name: tempo_no2_tropo - obs space: name: tropomi_s5p_no2_tropo obsdatain: diff --git a/src/swell/utilities/question_defaults.py b/src/swell/utilities/question_defaults.py index 1ed626a62..08821f035 100644 --- a/src/swell/utilities/question_defaults.py +++ b/src/swell/utilities/question_defaults.py @@ -835,40 +835,43 @@ class geos_cf_install_dir(TaskQuestion): # -------------------------------------------------------------------------------------------------- @dataclass - class geos_cf_run_dir(TaskQuestion): + class geos_cf_rc_dir(TaskQuestion): default_value: str = "defer_to_platform" - question_name: str = "geos_cf_run_dir" + question_name: str = "geos_cf_rc_dir" ask_question: bool = True models: List[str] = mutable_field([ "geos_cf" ]) - prompt: str = "What is the path to the GEOS-CF model run directory?" + prompt: str = "What is the path to the GEOS-CF RC directory?" widget_type: WType = WType.STRING # -------------------------------------------------------------------------------------------------- @dataclass - class geosfp_exp(TaskQuestion): - default_value: str = "f5295_fp" - question_name: str = "geosfp_exp" + class met_replay_exp(TaskQuestion): + default_value: str = "d5294_geosit_jan18" + question_name: str = "met_replay_exp" ask_question: bool = True models: List[str] = mutable_field([ "geos_cf" ]) - prompt: str = "What is the GEOS FP experiment ID used for IAU analysis files?" + prompt: str = ("What is the meteorology replay experiment ID (e.g. a GEOS FP " + "experiment such as f5295_fp, or a GEOS-IT experiment such as " + "d5294_geosit_jan18)?") widget_type: WType = WType.STRING # -------------------------------------------------------------------------------------------------- @dataclass - class geosfp_path(TaskQuestion): - default_value: str = "defer_to_platform" - question_name: str = "geosfp_path" + class met_replay_root(TaskQuestion): + default_value: str = "/home/dao_ops" + question_name: str = "met_replay_root" ask_question: bool = True models: List[str] = mutable_field([ "geos_cf" ]) - prompt: str = "What is the path to the GEOS FP archive?" + prompt: str = ("What is the root directory containing the meteorology replay " + "experiment (parent of /run/.../archive)?") widget_type: WType = WType.STRING # --------------------------------------------------------------------------------------------------