diff --git a/.github/workflows/build_mpas.yml b/.github/workflows/build_mpas.yml index bb4b2bb08c..8937629905 100644 --- a/.github/workflows/build_mpas.yml +++ b/.github/workflows/build_mpas.yml @@ -11,7 +11,7 @@ jobs: fail-fast: false # Disable fail-fast matrix: fortran-compiler: [gfortran-12] - build-system: [make]###, cmake] + build-system: [make] ###, cmake] # Environmental variables env: @@ -37,7 +37,7 @@ jobs: - name: Install NetCDF library run: | sudo apt-get update - sudo apt-get install libnetcdff-dev liblapack-dev libblas-dev + sudo apt-get install libnetcdff-dev - name: Cache openmpi id: cache-openmpi @@ -138,18 +138,6 @@ jobs: echo "LD_LIBRARY_PATH=${PNETCDF}/bin:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=${PNETCDF}/bin:$PATH" >> $GITHUB_ENV - - name: Setup NetCDF - run: | - NETCDF_DIR=/home/runner/NetCDF - mkdir -p ${NETCDF_DIR}/lib ${NETCDF_DIR}/include - for f in /usr/include/netcdf*.h /usr/include/netcdf*.inc /usr/include/netcdf*.mod; do - [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/include/ || true - done - for f in /usr/lib/x86_64-linux-gnu/libnetcdf*.so; do - [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/lib/ || true - done - echo "NETCDF=${NETCDF_DIR}" >> $GITHUB_ENV - # - name: Setup environment # run: | # echo "FC=mpif90" >> $GITHUB_ENV diff --git a/.github/workflows/build_mpas_intel.yml b/.github/workflows/build_mpas_intel.yml index 0e68fd5555..49868e02bc 100644 --- a/.github/workflows/build_mpas_intel.yml +++ b/.github/workflows/build_mpas_intel.yml @@ -41,7 +41,6 @@ jobs: install-mpi: true install-oneapi: false mpi-wrapper-setup: classic - install-mkl: true - name: Cache Intel id: cache-INTEL @@ -50,18 +49,6 @@ jobs: path: /opt/intel/oneapi/mpi/2021.10.0 key: cache-Intel-MPI-${{matrix.fortran-compiler}}-${{matrix.build-system}}-key - - name: Install NetCDF-C library - run: | - sudo apt-get update - sudo apt-get install libnetcdf-dev - - - name: Cache NetCDF-Fortran - id: cache-netcdf-fortran - uses: actions/cache@v4 - with: - path: /home/runner/NetCDF - key: cache-NetCDF-Fortran-${{matrix.fortran-compiler}}-${{matrix.build-system}}-key - - name: Cache GNU autotools id: cache-AUTOTOOLS uses: actions/cache@v5 @@ -137,23 +124,6 @@ jobs: echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=${PNETCDF}/bin:$PATH" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=${PNETCDF}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - echo "NETCDF=/home/runner/NetCDF" >> $GITHUB_ENV - - - name: Build NetCDF-Fortran - if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' - run: | - NETCDF_F_VERSION=4.6.1 - wget -q https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v${NETCDF_F_VERSION}.tar.gz - tar -xf v${NETCDF_F_VERSION}.tar.gz - cd netcdf-fortran-${NETCDF_F_VERSION} - autoreconf -i - FC=mpiifort CC=mpiicc \ - CPPFLAGS="-I$(nc-config --includedir)" \ - LDFLAGS="-L$(nc-config --libdir)" \ - ./configure --prefix=/home/runner/NetCDF - make -j 8 install - NCLIBDIR=$(nc-config --libdir) - ln -sf ${NCLIBDIR}/libnetcdf.so /home/runner/NetCDF/lib/libnetcdf.so - name: Build MPAS Standalone (make) if: matrix.build-system == 'make' diff --git a/.github/workflows/build_mpas_intel_stoch.yml b/.github/workflows/build_mpas_intel_stoch.yml new file mode 100644 index 0000000000..03ca42a64c --- /dev/null +++ b/.github/workflows/build_mpas_intel_stoch.yml @@ -0,0 +1,172 @@ +name: Build (Intel) MPAS Stochastic + +on: [pull_request,workflow_dispatch] + +jobs: + build_mpas_intel: + + # The type of runner that the job will run on + runs-on: ubuntu-latest + strategy: + fail-fast: false # Disable fail-fast + matrix: + fortran-compiler: [intel] + build-system: [make, cmake] + + # Environmental variables + env: + driver_ROOT: /home/runner/work + AUTOCONF_VERSION: 2.71 + AUTOMAKE_VERSION: 1.17 + M4_VERSION: 1.4.19 + LIBTOOL_VERSION: 2.5.4 + PNETCDF: /home/runner/PnetCDF + AUTOTOOLS: /home/runner/AUTOTOOLS + OMPI: /opt/intel/oneapi/mpi/2021.10.0 + + # Workflow steps + steps: + + - name: Checkout code (into ${driver_ROOT}) + uses: actions/checkout@v6 + + - name: Initialize submodules + run: | + git submodule update --init --recursive + echo "ls -l src/core_atmosphere/physics/physics_mmm" + + - name: "Install Intel compilers & MPI" + uses: NOAA-EMC/ci-install-intel-toolkit@ufs-community-mpas-ci-v2.0 + with: + install-mpi: true + install-oneapi: false + mpi-wrapper-setup: classic + install-mkl: true + + - name: Cache Intel + id: cache-INTEL + uses: actions/cache@v5 + with: + path: /opt/intel/oneapi/mpi/2021.10.0 + key: cache-Intel-MPI-${{matrix.fortran-compiler}}-${{matrix.build-system}}-key + + - name: Install NetCDF-C library + run: | + sudo apt-get update + sudo apt-get install libnetcdf-dev + + - name: Cache NetCDF-Fortran + id: cache-netcdf-fortran + uses: actions/cache@v4 + with: + path: /home/runner/NetCDF + key: cache-NetCDF-Fortran-${{matrix.fortran-compiler}}-${{matrix.build-system}}-key + + - name: Cache GNU autotools + id: cache-AUTOTOOLS + uses: actions/cache@v5 + with: + path: /home/runner/AUTOTOOLS + key: cache-AUTOTOOLS-${{matrix.fortran-compiler}}-${{matrix.build-system}}-key + + - name: Build GNU autotools + if: steps.cache-AUTOTOOLS.outputs.cache-hit != 'true' + run: | + wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz + gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - + cd m4-${M4_VERSION} + ./configure --prefix=${AUTOTOOLS} --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + + wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz + gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - + cd autoconf-${AUTOCONF_VERSION} + ./configure --prefix=${AUTOTOOLS} --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + + wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz + gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - + cd automake-${AUTOMAKE_VERSION} + ./configure --prefix=${AUTOTOOLS} --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + + wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz + gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - + cd libtool-${LIBTOOL_VERSION} + ./configure --prefix=${AUTOTOOLS} --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=${AUTOTOOLS}/bin:$PATH" >> $GITHUB_ENV + + - name: Cache PnetCDF + id: cache-pnetcdf + uses: actions/cache@v5 + with: + path: /home/runner/PnetCDF + key: cache-PnetCDF-${{matrix.fortran-compiler}}-${{matrix.build-system}}-key + + - name: Install PnetCDF + if: steps.cache-pnetcdf.outputs.cache-hit != 'true' + run: | + set -x + echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=${AUTOTOOLS}/bin:$PATH" >> $GITHUB_ENV + m4 --version + autoconf --version + automake --version + libtool --version + echo "Install PnetCDF on ${PNETCDF}" + rm -rf PnetCDF ; mkdir PnetCDF ; cd PnetCDF + git clone -q https://github.com/Parallel-NetCDF/PnetCDF.git + cd PnetCDF + autoreconf -i + ./configure --prefix=${PNETCDF} --with-mpi=${OMPI} + make -j 8 install + echo "PATH=${PNETCDF}/bin:$PATH" >> $GITHUB_ENV + + - name: Setup environment + run: | + echo "FC=mpiifort" >> $GITHUB_ENV + echo "CC=mpiicc" >> $GITHUB_ENV + echo "PNETCDF=${PNETCDF}" >> $GITHUB_ENV + echo "PATH=${AUTOTOOLS}/bin:$PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=${PNETCDF}/bin:$PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${PNETCDF}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "NETCDF=/home/runner/NetCDF" >> $GITHUB_ENV + + - name: Build NetCDF-Fortran + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + run: | + NETCDF_F_VERSION=4.6.1 + wget -q https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v${NETCDF_F_VERSION}.tar.gz + tar -xf v${NETCDF_F_VERSION}.tar.gz + cd netcdf-fortran-${NETCDF_F_VERSION} + autoreconf -i + FC=mpiifort CC=mpiicc \ + CPPFLAGS="-I$(nc-config --includedir)" \ + LDFLAGS="-L$(nc-config --libdir)" \ + ./configure --prefix=/home/runner/NetCDF + make -j 8 install + NCLIBDIR=$(nc-config --libdir) + ln -sf ${NCLIBDIR}/libnetcdf.so /home/runner/NetCDF/lib/libnetcdf.so + + - name: Build MPAS Standalone (make) + if: matrix.build-system == 'make' + run: | + cd ${driver_ROOT}/MPAS-Model/MPAS-Model + make intel-mpi CORE=init_atmosphere + make intel-mpi CORE=atmosphere STOCHASTIC_PHYSICS=true AUTOCLEAN=true + + - name: Build MPAS Standalone (cmake) + if: matrix.build-system == 'cmake' + run: | + cd ${driver_ROOT}/MPAS-Model/MPAS-Model + mkdir build + cd build + cmake -DMPAS_DOUBLE_PRECISION=OFF -DSTOCHASTIC_PHYSICS=ON -DMPAS_CORES="init_atmosphere;atmosphere" .. + make -j8 diff --git a/.github/workflows/build_mpas_stoch.yml b/.github/workflows/build_mpas_stoch.yml new file mode 100644 index 0000000000..e4687537a1 --- /dev/null +++ b/.github/workflows/build_mpas_stoch.yml @@ -0,0 +1,187 @@ +name: Build (GNU) MPAS Stochastic + +on: [pull_request,workflow_dispatch] + +jobs: + build_mpas_GNU: + + # The type of runner that the job will run on + runs-on: ubuntu-22.04 + strategy: + fail-fast: false # Disable fail-fast + matrix: + fortran-compiler: [gfortran-12] + build-system: [make] ###, cmake] + + # Environmental variables + env: + driver_ROOT: /home/runner/work + AUTOCONF_VERSION: 2.71 + AUTOMAKE_VERSION: 1.17 + M4_VERSION: 1.4.19 + LIBTOOL_VERSION: 2.5.4 + PNETCDF: /home/runner/PnetCDF + AUTOTOOLS: /home/runner/AUTOTOOLS + OMPI: /home/runner/ompi-4.1.6 + + # Workflow steps + steps: + + - name: Checkout code (into ${driver_ROOT}) + uses: actions/checkout@v6 + + - name: Initialize submodules + run: | + git submodule update --init --recursive + + - name: Install NetCDF library + run: | + sudo apt-get update + sudo apt-get install libnetcdff-dev liblapack-dev libblas-dev + + - name: Cache openmpi + id: cache-openmpi + uses: actions/cache@v5 + with: + path: /home/runner/ompi-4.1.6 + key: cache-openmpi-${{matrix.fortran-compiler}}-key + + - name: Install openmpi + if: steps.cache-openmpi.outputs.cache-hit != 'true' + run: | + wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz + tar -xvf v4.1.6.tar.gz + cd ompi-4.1.6 + ./autogen.pl + ./configure --prefix=${OMPI} + make -j4 + make install + + - name: Add (OMPI) to Paths + run: | + echo "LD_LIBRARY_PATH=${OMPI}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=${OMPI}/bin:$PATH" >> $GITHUB_ENV + + - name: Cache GNU autotools + id: cache-AUTOTOOLS + uses: actions/cache@v5 + with: + path: /home/runner/AUTOTOOLS + key: cache-AUTOTOOLS-${{matrix.fortran-compiler}}-key + + - name: Build GNU autotools + if: steps.cache-AUTOTOOLS.outputs.cache-hit != 'true' + run: | + wget -q https://ftp.gnu.org/gnu/m4/m4-${M4_VERSION}.tar.gz + gzip -dc m4-${M4_VERSION}.tar.gz | tar -xf - + cd m4-${M4_VERSION} + ./configure --prefix=${AUTOTOOLS} --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + + wget -q https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.gz + gzip -dc autoconf-${AUTOCONF_VERSION}.tar.gz | tar -xf - + cd autoconf-${AUTOCONF_VERSION} + ./configure --prefix=${AUTOTOOLS} --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + + wget -q https://ftp.gnu.org/gnu/automake/automake-${AUTOMAKE_VERSION}.tar.gz + gzip -dc automake-${AUTOMAKE_VERSION}.tar.gz | tar -xf - + cd automake-${AUTOMAKE_VERSION} + ./configure --prefix=${AUTOTOOLS} --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + + wget -q https://ftp.gnu.org/gnu/libtool/libtool-${LIBTOOL_VERSION}.tar.gz + gzip -dc libtool-${LIBTOOL_VERSION}.tar.gz | tar -xf - + cd libtool-${LIBTOOL_VERSION} + ./configure --prefix=${AUTOTOOLS} --silent + make -s -j 8 install > qout 2>&1 + make -s -j 8 distclean >> qout 2>&1 + echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=${AUTOTOOLS}/bin:$PATH" >> $GITHUB_ENV + + - name: Add (autotools) to Paths + run: | + echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=${AUTOTOOLS}/bin:$PATH" >> $GITHUB_ENV + + - name: Cache PnetCDF + id: cache-pnetcdf + uses: actions/cache@v5 + with: + path: /home/runner/PnetCDF + key: cache-PnetCDF-${{matrix.fortran-compiler}}-key + + - name: Install PnetCDF + if: steps.cache-PnetCDF.outputs.cache-hit != 'true' + run: | + set -x + #echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + #echo "PATH=${AUTOTOOLS}/bin:$PATH" >> $GITHUB_ENV + m4 --version + autoconf --version + automake --version + libtool --version + echo "Install PnetCDF on ${PNETCDF}" + rm -rf PnetCDF ; mkdir PnetCDF ; cd PnetCDF + git clone -q https://github.com/Parallel-NetCDF/PnetCDF.git + cd PnetCDF + autoreconf -i + ./configure --prefix=${PNETCDF} --with-mpi=${OMPI} + make -j 8 install + + - name: Add (PNETCDF) to paths. + run: | + echo "PNETCDF=${PNETCDF}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${PNETCDF}/bin:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "PATH=${PNETCDF}/bin:$PATH" >> $GITHUB_ENV + + - name: Setup NetCDF + run: | + NETCDF_DIR=/home/runner/NetCDF + mkdir -p ${NETCDF_DIR}/lib ${NETCDF_DIR}/include + for f in /usr/include/netcdf*.h /usr/include/netcdf*.inc /usr/include/netcdf*.mod; do + [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/include/ || true + done + for f in /usr/lib/x86_64-linux-gnu/libnetcdf*.so; do + [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/lib/ || true + done + echo "NETCDF=${NETCDF_DIR}" >> $GITHUB_ENV + +# - name: Setup environment +# run: | +# echo "FC=mpif90" >> $GITHUB_ENV +# echo "CC=mpicc" >> $GITHUB_ENV +# echo "PNETCDF=${PNETCDF}" >> $GITHUB_ENV +# echo "PATH=${AUTOTOOLS}/bin:$PATH" >> $GITHUB_ENV +# echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV +# echo "PATH=${OMPI}/bin:$PATH" >> $GITHUB_ENV +# echo "LD_LIBRARY_PATH=${OMPI}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV +# echo "PATH=${PNETCDF}/bin:$PATH" >> $GITHUB_ENV +# echo "LD_LIBRARY_PATH=${PNETCDF}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Display environment + run: | + echo "FC: " ${FC} + echo "CC: " ${CC} + echo "PATH: " ${PATH} + echo "LD_LIBRARY_PATH: " ${LD_LIBRARY_PATH} + echo "PNETCDF: " ${PNETCDF} + + - name: Build MPAS Standalone (make) + if: matrix.build-system == 'make' + run: | + cd ${driver_ROOT}/MPAS-Model/MPAS-Model + make gfortran CORE=init_atmosphere + make gfortran CORE=atmosphere STOCHASTIC_PHYSICS=true AUTOCLEAN=true + + - name: Build MPAS Standalone (cmake) + if: matrix.build-system == 'cmake' + run: | + cd ${driver_ROOT}/MPAS-Model/MPAS-Model + mkdir build + cd build + cmake -DMPAS_DOUBLE_PRECISION=OFF -DSTOCHASTIC_PHYSICS=ON -DMPAS_CORES="init_atmosphere;atmosphere" .. + make -j8 diff --git a/.github/workflows/run_mpas.yml b/.github/workflows/run_mpas.yml index 6d20db8f93..1c9c108feb 100644 --- a/.github/workflows/run_mpas.yml +++ b/.github/workflows/run_mpas.yml @@ -98,23 +98,6 @@ jobs: echo "FC=mpiifx" >> $GITHUB_ENV echo "MPICC=/opt/intel/oneapi/mpi/latest/bin/mpiicx" >> $GITHUB_ENV - - name: Install LAPACK/BLAS - run: | - apt-get update -y - apt-get install -y liblapack-dev libblas-dev libnetcdff-dev - - - name: Setup NetCDF - run: | - NETCDF_DIR=/home/runner/NetCDF - mkdir -p ${NETCDF_DIR}/lib ${NETCDF_DIR}/include - for f in /usr/include/netcdf*.h /usr/include/netcdf*.inc /usr/include/netcdf*.mod; do - [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/include/ || true - done - for f in /usr/lib/x86_64-linux-gnu/libnetcdf*.so; do - [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/lib/ || true - done - echo "NETCDF=${NETCDF_DIR}" >> $GITHUB_ENV - - name: Setup PnetCDF run: | echo "PNETCDF=/opt/pnetcdf" >> $GITHUB_ENV diff --git a/.github/workflows/run_mpas_hrrr.yml b/.github/workflows/run_mpas_hrrr.yml index 995700ba80..643174cec5 100644 --- a/.github/workflows/run_mpas_hrrr.yml +++ b/.github/workflows/run_mpas_hrrr.yml @@ -79,23 +79,6 @@ jobs: echo "CXX=mpicxx" >> $GITHUB_ENV echo "FC=mpif90" >> $GITHUB_ENV - - name: Install LAPACK/BLAS - run: | - apt-get update -y - apt-get install -y liblapack-dev libblas-dev libnetcdff-dev - - - name: Setup NetCDF - run: | - NETCDF_DIR=/home/runner/NetCDF - mkdir -p ${NETCDF_DIR}/lib ${NETCDF_DIR}/include - for f in /usr/include/netcdf*.h /usr/include/netcdf*.inc /usr/include/netcdf*.mod; do - [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/include/ || true - done - for f in /usr/lib/x86_64-linux-gnu/libnetcdf*.so; do - [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/lib/ || true - done - echo "NETCDF=${NETCDF_DIR}" >> $GITHUB_ENV - - name: Setup PnetCDF run: | echo "PNETCDF=/opt/pnetcdf" >> $GITHUB_ENV diff --git a/.github/workflows/run_mpas_stoch.yml b/.github/workflows/run_mpas_stoch.yml index 738a4da4df..b622343b44 100644 --- a/.github/workflows/run_mpas_stoch.yml +++ b/.github/workflows/run_mpas_stoch.yml @@ -220,7 +220,7 @@ jobs: if: steps.cache-feature.outputs.cache-hit != 'true' run: | cd ${SRCDIR_FT} - make ${{matrix.bld_target}} CORE=atmosphere ${{matrix.build-type == 'Debug' && 'DEBUG=true' || ''}} + make ${{matrix.bld_target}} CORE=atmosphere STOCHASTIC_PHYSICS=true ${{matrix.build-type == 'Debug' && 'DEBUG=true' || ''}} ########################################################################################## # Download any data/files needed for MPAS runs. diff --git a/CMakeLists.txt b/CMakeLists.txt index bd15c2b3ee..8adf7cb774 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ if(MPAS_CORES MATCHES " ") #Convert strings separated with spaces to CMake list set(MPAS_CORES ${MPAS_CORES} CACHE STRING "MPAS cores to build. Options: ${MPAS_ALL_CORES}" FORCE) endif() option(DO_PHYSICS "Use built-in physics schemes." TRUE) +option(STOCHASTIC_PHYSICS "Build with stochastic physics support." FALSE) option(MPAS_DOUBLE_PRECISION "Use double precision 64-bit Floating point." TRUE) option(MPAS_PROFILE "Enable GPTL profiling" OFF) option(MPAS_OPENMP "Enable OpenMP" OFF) @@ -67,10 +68,12 @@ if(CMAKE_Fortran_COMPILER_ID MATCHES NVHPC) endif() find_package(MPI REQUIRED COMPONENTS Fortran) # -# Always set the MPAS_USE_MPI_F08 macro, which entails using "use mpi_f08" -# instead of the older "use mpi" in Fortran source code. +# The MPAS_USE_MPI_F08 macro (use mpi_f08, type(MPI_Comm) communicators) is +# required only when building stochastic_physics, which uses mpi_f08. # -add_compile_definitions(MPAS_USE_MPI_F08) +if(STOCHASTIC_PHYSICS) + add_compile_definitions(MPAS_USE_MPI_F08) +endif() # # When building stochastic_physics under the UFS, by default we use the # "esmf_dgemm" Fortran matrix multiplication subroutine that the UFS host diff --git a/Makefile b/Makefile index 27b65194b1..55a3c80923 100644 --- a/Makefile +++ b/Makefile @@ -375,7 +375,7 @@ ifort_icx: # BUILDTARGET Intel Fortran, C, and C++ compiler suite "DEBUG = $(DEBUG)" \ "USE_PAPI = $(USE_PAPI)" \ "OPENMP = $(OPENMP)" \ - "LAPACK_LIBS = -lmkl_intel_lp64 -lmkl_core -lmkl_sequential" \ + "LAPACK_LIBS = $(if $(filter true,$(STOCHASTIC_PHYSICS)),-lmkl_intel_lp64 -lmkl_core -lmkl_sequential)" \ "CPPFLAGS = $(MODEL_FORMULATION) -D_MPI" ) jet_ifort: @@ -482,7 +482,7 @@ intel-mpi: # BUILDTARGET Intel compiler suite with Intel MPI library "DEBUG = $(DEBUG)" \ "USE_PAPI = $(USE_PAPI)" \ "OPENMP = $(OPENMP)" \ - "LAPACK_LIBS = -lmkl_intel_lp64 -lmkl_core -lmkl_sequential" \ + "LAPACK_LIBS = $(if $(filter true,$(STOCHASTIC_PHYSICS)),-lmkl_intel_lp64 -lmkl_core -lmkl_sequential)" \ "CPPFLAGS = $(MODEL_FORMULATION) -D_MPI" ) intel-llvm-mpi: # BUILDTARGET Intel LLVM compiler suite with Intel MPI library @@ -537,7 +537,7 @@ intel-mpi-ursa: # usra "DEBUG = $(DEBUG)" \ "USE_PAPI = $(USE_PAPI)" \ "OPENMP = $(OPENMP)" \ - "LAPACK_LIBS = -lmkl_intel_lp64 -lmkl_core -lmkl_sequential" \ + "LAPACK_LIBS = $(if $(filter true,$(STOCHASTIC_PHYSICS)),-lmkl_intel_lp64 -lmkl_core -lmkl_sequential)" \ "CPPFLAGS = $(MODEL_FORMULATION) -D_MPI" ) gfortran: # BUILDTARGET GNU Fortran, C, and C++ compilers diff --git a/README.md b/README.md index 5e4edc8edc..6896b7af00 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -MPAS-v8.4.1-1.6 +MPAS-v8.4.1-1.7 ==== The Model for Prediction Across Scales (MPAS) is a collaborative project for diff --git a/build.sh b/build.sh index 7413883eef..b1fc0a6d95 100755 --- a/build.sh +++ b/build.sh @@ -8,9 +8,10 @@ COMPILER="${COMPILER:-intel}" usage() { set +x echo - echo "Usage: $0 -j -h" + echo "Usage: $0 [-j ] [-s] [-c ] [-f] [-h]" echo echo " -j number of build jobs DEFAULT: 8" + echo " -s build with stochastic_physics" echo " -c additional CMAKE options" echo " -f force a clean build" echo " -h display this message and quit" @@ -18,11 +19,15 @@ usage() { exit 1 } -while getopts "c:j:hf" opt; do +STOCHASTIC="" +while getopts "c:j:hfs" opt; do case ${opt} in c) CMAKE_OPTS=${OPTARG} ;; + s) + STOCHASTIC="-DSTOCHASTIC_PHYSICS=ON" + ;; j) BUILD_JOBS=${OPTARG} ;; @@ -63,5 +68,5 @@ cd ${BUILD_DIR} || exit 1 BUILD_JOBS=${BUILD_JOBS:-8} CMAKE_OPTS+=' -DMPAS_DOUBLE_PRECISION=OFF' -cmake ${CMAKE_OPTS} -DMPAS_CORES="init_atmosphere;atmosphere" .. +cmake ${CMAKE_OPTS} ${STOCHASTIC} -DMPAS_CORES="init_atmosphere;atmosphere" .. make -j $BUILD_JOBS diff --git a/src/core_atmosphere/CMakeLists.txt b/src/core_atmosphere/CMakeLists.txt index d7d461f35c..065a97eed8 100644 --- a/src/core_atmosphere/CMakeLists.txt +++ b/src/core_atmosphere/CMakeLists.txt @@ -142,7 +142,8 @@ list(TRANSFORM ATMOSPHERE_CORE_PHYSICS_SMOKE_SOURCES PREPEND physics/physics_noa set(ATMOSPHERE_CORE_PHYSICS_MMM_DIR ${CMAKE_CURRENT_SOURCE_DIR}/physics/physics_mmm) -# build stochastic_physics as a library +# build stochastic_physics as a library (optional) +if (${STOCHASTIC_PHYSICS}) set(ATMOSPHERE_CORE_STOCHASTIC_PHYSICS_SOURCES kinddef.F90 mpi_wrapper.F90 @@ -174,6 +175,7 @@ target_link_libraries(stochastic_physics PUBLIC MPI::MPI_Fortran framework ${BLA install(TARGETS stochastic_physics EXPORT ${PROJECT_NAME}ExportsCore ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) +endif() if(NOT EXISTS ${ATMOSPHERE_CORE_PHYSICS_MMM_DIR}) set(PHYSICS_MMM_REPO_URL "https://github.com/NCAR/MMM-physics") @@ -573,8 +575,13 @@ set(CORE_ATMOSPHERE_COMPILE_DEFINITIONS if (${DO_PHYSICS}) list(APPEND CORE_ATMOSPHERE_COMPILE_DEFINITIONS DO_PHYSICS) endif () +if (${STOCHASTIC_PHYSICS}) + list(APPEND CORE_ATMOSPHERE_COMPILE_DEFINITIONS STOCHASTIC_PHYSICS) +endif() target_compile_definitions(core_atmosphere PRIVATE ${CORE_ATMOSPHERE_COMPILE_DEFINITIONS}) -target_link_libraries(core_atmosphere PUBLIC stochastic_physics) +if (${STOCHASTIC_PHYSICS}) + target_link_libraries(core_atmosphere PUBLIC stochastic_physics) +endif() set_MPAS_DEBUG_flag(core_atmosphere) mpas_core_target(CORE atmosphere TARGET core_atmosphere INCLUDES ${ATMOSPHERE_CORE_INCLUDES}) diff --git a/src/core_atmosphere/Makefile b/src/core_atmosphere/Makefile index c8b6100802..65c3af2a38 100644 --- a/src/core_atmosphere/Makefile +++ b/src/core_atmosphere/Makefile @@ -21,6 +21,10 @@ ifdef CHEMISTRY CHEM_OBJS = libchem/*.o endif +ifeq "$(STOCHASTIC_PHYSICS)" "true" + STOCH = true +endif + OBJS = mpas_atm_core.o \ mpas_atm_core_interface.o \ mpas_atm_dimensions.o \ @@ -66,8 +70,12 @@ post_build: physcore: mpas_atm_dimensions.o ( cd physics; $(MAKE) all ) +ifdef STOCH ( cd stochastic_physics; $(MAKE) -f Makefile all ) - ( mkdir libphys; cd libphys; ar -x ../physics/libphys.a; ar -x ../stochastic_physics/libstochphys.a ) + ( mkdir libphys; cd libphys; ar -x ../physics/libphys.a; ar -x ../stochastic_physics/libstochphys.a ) +else + ( mkdir libphys; cd libphys; ar -x ../physics/libphys.a ) +endif ( cd ../..; ln -sf ./src/core_atmosphere/physics/physics_wrf/files/*TBL .) ( cd ../..; ln -sf ./src/core_atmosphere/physics/physics_wrf/files/*DATA* .) ( cd ../..; ln -sf ./src/core_atmosphere/physics/physics_noahmp/parameters/*TBL .) @@ -103,7 +111,9 @@ clean: @# the latter for MPAS. The default for "make" is to use "makefile", but @# here, we want to use the one for MPAS (with an upper case "M"). Indicate @# that using the -f flag. +ifneq ($(wildcard stochastic_physics/Makefile),) ( cd stochastic_physics; $(MAKE) -f Makefile clean ) +endif ( cd diagnostics; $(MAKE) clean ) ( cd utils; $(MAKE) clean ) ( cd ../..; rm -f *TBL ) @@ -121,7 +131,15 @@ clean: $(RM) $@ $*.mod ifeq "$(GEN_F90)" "true" $(CPP) $(CPPFLAGS) $(PHYSICS) $(CHEMISTRY) $(CPPINCLUDES) -I./inc -I../framework $< > $*.f90 +ifdef STOCH $(FC) $(FFLAGS) -c $*.f90 $(FCINCLUDES) -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP $(MPAS_ESMF_INC) -I./chemistry -I./stochastic_physics else + $(FC) $(FFLAGS) -c $*.f90 $(FCINCLUDES) -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP $(MPAS_ESMF_INC) -I./chemistry +endif +else +ifdef STOCH $(FC) $(CPPFLAGS) $(PHYSICS) $(CHEMISTRY) $(FFLAGS) -c $*.F $(CPPINCLUDES) $(FCINCLUDES) -I./inc -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP $(MPAS_ESMF_INC) -I./chemistry -I./stochastic_physics +else + $(FC) $(CPPFLAGS) $(PHYSICS) $(CHEMISTRY) $(FFLAGS) -c $*.F $(CPPINCLUDES) $(FCINCLUDES) -I./inc -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP $(MPAS_ESMF_INC) -I./chemistry +endif endif diff --git a/src/core_atmosphere/Registry.xml b/src/core_atmosphere/Registry.xml index bc964fd3d2..7f0ef79425 100644 --- a/src/core_atmosphere/Registry.xml +++ b/src/core_atmosphere/Registry.xml @@ -1072,7 +1072,9 @@ +#ifdef STOCHASTIC_PHYSICS +#endif @@ -4415,8 +4417,10 @@ description="Total cell-centered meridional wind tendency from physics" persistence="scratch" /> +#ifdef STOCHASTIC_PHYSICS +#endif #ifdef DO_PHYSICS @@ -5135,8 +5139,10 @@ #include "physics/Registry_noahmp.xml" #include "physics/Registry_tempo.xml" #include "physics/registry.chemistry.xml" +#ifdef STOCHASTIC_PHYSICS #include "stochastic_physics/Registry_stoch_physics.xml" #endif +#endif #ifdef MPAS_UFS_DYCORE #include "physics/Registry_noahmp.xml" diff --git a/src/core_atmosphere/build_options.mk b/src/core_atmosphere/build_options.mk index 3b5a873451..ff55930e74 100644 --- a/src/core_atmosphere/build_options.mk +++ b/src/core_atmosphere/build_options.mk @@ -6,5 +6,9 @@ FCINCLUDES += -I$(PWD)/src/core_atmosphere/physics/physics_noahmp/drivers/mpas \ -I$(PWD)/src/core_atmosphere/physics/physics_noahmp/utility \ -I$(PWD)/src/core_atmosphere/physics/physics_noahmp/src +ifeq "$(STOCHASTIC_PHYSICS)" "true" +override CPPFLAGS += -DSTOCHASTIC_PHYSICS +endif + report_builds: @echo "CORE=atmosphere" diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index f8707cd166..178e5868fa 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -32,7 +32,9 @@ module atm_time_integration use mpas_atmphys_interface, only: pre_microphysics, post_microphysics use mpas_atmphys_todynamics use mpas_atmphys_utilities +#ifdef STOCHASTIC_PHYSICS use mpas_stochastic_physics, only : stochastic_physics_pattern_apply, dosppt +#endif #endif use mpas_atm_boundaries, only : nSpecZone, nRelaxZone, nBdyZone, mpas_atm_get_bdy_state, mpas_atm_get_bdy_tend ! regional_MPAS addition @@ -2145,10 +2147,12 @@ subroutine atm_srk3(domain, dt, itimestep, exchange_halo_group) rk_step = 1 dynamics_substep = 1 +#ifdef STOCHASTIC_PHYSICS ! apply random perturbation pattern to physics process (hydrostatic) tendencies if (dosppt(domain)) then call stochastic_physics_pattern_apply(domain, 'phys') endif +#endif call physics_get_tend( block, mesh, state, diag, tend, tend_physics, & block % configs, rk_step, dynamics_substep, & @@ -2157,10 +2161,12 @@ subroutine atm_srk3(domain, dt, itimestep, exchange_halo_group) call mpas_timer_stop('physics_get_tend') call post_physics_get_tend(block % configs, state, diag, tend) +#ifdef STOCHASTIC_PHYSICS ! apply random perturbation pattern to the accumulated (non-hydrostatic) physics tendencies if (dosppt(domain)) then call stochastic_physics_pattern_apply(domain, 'prog') endif +#endif #else #ifndef MPAS_CAM_DYCORE diff --git a/src/core_atmosphere/mpas_atm_core.F b/src/core_atmosphere/mpas_atm_core.F index c989492536..8166e089de 100644 --- a/src/core_atmosphere/mpas_atm_core.F +++ b/src/core_atmosphere/mpas_atm_core.F @@ -39,7 +39,9 @@ function atm_core_init(domain, startTimeStamp) result(ierr) #ifdef DO_CHEMISTRY use mpas_atm_chemistry, only: chemistry_init #endif +#ifdef STOCHASTIC_PHYSICS use mpas_stochastic_physics, only : stochastic_physics_pattern_init, dosppt +#endif implicit none @@ -301,11 +303,13 @@ function atm_core_init(domain, startTimeStamp) result(ierr) call chemistry_init(domain % blocklist % configs, domain % blocklist % dimensions) #endif +#ifdef STOCHASTIC_PHYSICS ! ! init stochastic pattern generation if (dosppt(domain)) then call stochastic_physics_pattern_init (domain) endif +#endif end function atm_core_init @@ -1059,7 +1063,9 @@ subroutine atm_do_timestep(domain, dt, itimestep) use mpas_atm_chemistry, only: chemistry_step #endif use mpas_atm_halos, only: exchange_halo_group +#ifdef STOCHASTIC_PHYSICS use mpas_stochastic_physics, only : stochastic_physics_pattern_adv, dosppt +#endif implicit none @@ -1096,9 +1102,11 @@ subroutine atm_do_timestep(domain, dt, itimestep) call chemistry_step() #endif +#ifdef STOCHASTIC_PHYSICS if (dosppt(domain)) then call stochastic_physics_pattern_adv(domain, itimestep) endif +#endif call atm_timestep(domain, dt, currTime, itimestep, exchange_halo_group) end subroutine atm_do_timestep