Merge branch 'develop' into ocnmrg_tests

Fixes #944.

.github/workflows/macos_clang.yaml

@@ -47,7 +47,7 @@ jobs:
         run: |
           git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
-          sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[apple-clang@14\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
+          sed "s/\[oneapi, gcc@10:10, apple-clang@14\]/\[apple-clang@14\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
           spack env create ufs_utils-env spack_ci.yaml
           spack env activate ufs_utils-env
           spack external find

.github/workflows/ubuntu_gcc.yaml

@@ -47,7 +47,7 @@ jobs:
         run: |
           git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
-          sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[gcc@10:10\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
+          sed "s/\[oneapi, gcc@10:10, apple-clang@14\]/\[gcc@10:10\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
           spack env create ufs_utils-env spack_ci.yaml
           spack env activate ufs_utils-env
           sudo apt install cmake

.github/workflows/ubuntu_intel.yaml

@@ -7,14 +7,14 @@ defaults:
   run:
     shell: bash -leo pipefail {0}
 
-# Set I_MPI_CC/F90 so Intel MPI wrapper uses icc/ifort instead of gcc/gfortran
+# Set I_MPI_CC/F90 so IntelLLVM is used.
 env:
   cache_key: intel
-  CC: icc
-  FC: ifort
-  CXX: icpc
-  I_MPI_CC: icc
-  I_MPI_F90: ifort
+  CC: mpiicc
+  FC: mpiifort
+  CXX: mpiicpc
+  I_MPI_CC: icx
+  I_MPI_F90: ifx
 
 # The jobs are split into:
 # 1. a dependency build step (setup), and
@@ -59,7 +59,7 @@ jobs:
         run: |
           git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
-          sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[intel\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
+          sed "s/\[oneapi, gcc@10:10, apple-clang@14\]/\[oneapi\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
           spack env create ufs_utils-env spack_ci.yaml
           spack env activate ufs_utils-env
           spack compiler find
@@ -99,8 +99,8 @@ jobs:
         run: |
           source spack/share/spack/setup-env.sh
           spack env activate ufs_utils-env
-          export CC=mpiicc
-          export FC=mpiifort
+          export CC=mpiicx
+          export FC=mpiifx
           cd ufs_utils
           mkdir -p build && cd build
           cmake -DCMAKE_INSTALL_PREFIX=../install ..

CMakeLists.txt

@@ -26,7 +26,7 @@ option(CHGRES "Enable building chgres_cube.fd" ON)
 option(CHGRES_ALL "Build chgres with all input data options." OFF)
 option(OROG_MASK_TOOLS "Enable building orog_mask_tools.fd" ON)
 # OROG_MASK_TOOLS must be ON for OROG_NETCDF_TOOLS to build.
-option(OROG_NETCDF_TOOLS "Enable building orog_netcdf_tools.fd" OFF)
+option(OROG_NETCDF_TOOLS "Enable building orog_netcdf_tools.fd" ON)
 option(SFC_CLIMO_GEN "Enable building sfc_climo_gen.fd" ON)
 option(VCOORD_GEN "Enable building vcoord_gen.fd" ON)
 option(FVCOMTOOLS "Enable building fvcom_tools.fd" ON)
@@ -67,10 +67,15 @@ if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$")
 endif()
 
 # Set compiler flags.
-if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$")
+if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -traceback")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -fp-model precise")
   set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check all -ftrapuv")
+# On some machines (Hera), using -check all results in an error
+# at the linking step. The uninit check is the culprit.
+  if(CMAKE_Fortran_COMPILER_ID MATCHES "^(IntelLLVM)$")
+    set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -check nouninit")
+  endif()
   if(APPLE)
     # The linker on macOS does not include `common symbols` by default.
     # Passing the -c flag includes them and fixes an error with undefined symbols.
@@ -86,10 +91,14 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$")
   set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -ggdb -Wall -fcheck=all")
 endif()
 
-if(CMAKE_C_COMPILER_ID MATCHES "^(Intel)$")
+if(CMAKE_C_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -traceback")
-  set(CMAKE_C_FLAGS_RELEASE "-O2")
   set(CMAKE_C_FLAGS_DEBUG "-O0")
+  if(CMAKE_C_COMPILER_ID MATCHES "^(Intel)$")
+    set(CMAKE_C_FLAGS_RELEASE "-O2")
+  else()
+    set(CMAKE_C_FLAGS_RELEASE "-O1")
+  endif()
 endif()
 
 # Find packages.

build_all.sh

@@ -19,7 +19,8 @@ source "${DIR_ROOT}/sorc/machine-setup.sh"
 
 # User Options
 target=${target:-"NULL"}
-compiler=${compiler:-"intel"}
+compiler=${compiler:-"intelllvm"} # If IntelLLVM is not available on the machine, will
+                                  # default to Intel classic.
 
 if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
   unset -f module
@@ -29,6 +30,12 @@ if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
 else
   set +x
   module use "${DIR_ROOT}/modulefiles"
+  if [[ "$compiler" == "intelllvm" ]]; then
+    if [[ ! -f ${DIR_ROOT}/modulefiles/build.$target.$compiler.lua ]];then
+      echo "IntelLLVM not available. Will use Intel Classic."
+      compiler=intel
+    fi
+  fi
   module load "build.$target.$compiler" > /dev/null
   module list
   set -x

ci/spack.yaml

@@ -2,7 +2,7 @@
 spack:
   packages:
     all:
-      compiler: [intel, gcc@10:10, apple-clang@14]
+      compiler: [oneapi, gcc@10:10, apple-clang@14]
   specs:
   - [email protected]
   - [email protected]
@@ -12,8 +12,7 @@ spack:
   - [email protected]
   - [email protected]
   - [email protected]
-  - [email protected]
-  - [email protected]
+  - [email protected]
   - [email protected]
   - [email protected]+fortran~pnetcdf
   - [email protected]

driver_scripts/driver_grid.hera.sh

@@ -64,7 +64,7 @@
 
 set -x
 
-compiler=${compiler:-"intel"}
+compiler=${compiler:-"intelllvm"}
 source ../sorc/machine-setup.sh > /dev/null 2>&1
 module use ../modulefiles
 module load build.$target.$compiler

driver_scripts/driver_grid.hercules.sh

@@ -64,7 +64,7 @@ set -x
 
 source ../sorc/machine-setup.sh > /dev/null 2>&1
 module use ../modulefiles
-module load build.$target.intel
+module load build.$target.intelllvm
 module list
 
 #-----------------------------------------------------------------------

driver_scripts/driver_grid.orion.sh

@@ -64,7 +64,7 @@ set -x
 
 source ../sorc/machine-setup.sh > /dev/null 2>&1
 module use ../modulefiles
-module load build.$target.intel
+module load build.$target.intelllvm
 module list
 
 #-----------------------------------------------------------------------

fix/link_fixdirs.sh

@@ -9,15 +9,15 @@ set -ex
 #               'nco' (copies data).
 #
 #  $machine - is the machine. Choices are:
-#             'wcoss2', 'hera', 'jet', 'orion', 'hercules', 's4', 'gaea'
+#             'wcoss2', 'hera', 'jet', 'orion', 'hercules', 's4', 'gaeac5', 'gaeac6'
 
 RUN_ENVIR=${1}
 machine=${2}
 
 if [ $# -lt 2 ]; then
     set +x
     echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine'
-    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 |  hera  | jet | orion | hercules | s4 | gaea )'
+    echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 |  hera  | jet | orion | hercules | s4 | gaeac5 | gaeac6 )'
     exit 1
 fi
 
@@ -28,10 +28,10 @@ if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then
     exit 1
 fi
 
-if [ $machine != wcoss2 -a $machine != hera -a $machine != jet -a $machine != orion -a $machine != s4 -a $machine != hercules -a $machine != gaea ]; then
+if [ $machine != wcoss2 -a $machine != hera -a $machine != jet -a $machine != orion -a $machine != s4 -a $machine != hercules -a $machine != gaeac5 -a $machine != gaeac6 ]; then
     set +x
     echo '***ERROR*** unsupported machine'
-    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | hera | jet | orion | hercules | s4 | gaea )'
+    echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( wcoss2 | hera | jet | orion | hercules | s4 | gaeac5 | gaeac6 )'
     exit 1
 fi
 
@@ -54,8 +54,10 @@ elif [ $machine = "wcoss2" ]; then
     FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix"
 elif [ $machine = "s4" ]; then
     FIX_DIR="/data/prod/glopara/fix"
-elif [ $machine = "gaea" ]; then
+elif [ $machine = "gaeac5" ]; then
     FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix"
+elif [ $machine = "gaeac6" ]; then
+    FIX_DIR="/gpfs/f6/bil-fire8/world-shared/global/glopara/data/fix"
 fi
 
 am_ver=${am_ver:-20220805}

modulefiles/build.container.intel.lua

@@ -0,0 +1,55 @@
+help([[
+Load environment to compile UFS_UTILS in a container using Intel
+]])
+
+prepend_path("MODULEPATH", "/opt/spack-stack/spack-stack-1.8.0/envs/unified-env/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.12.1"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.27.9"
+load(pathJoin("cmake", cmake_ver))
+
+bacio_ver=os.getenv("bacio_ver") or "2.4.1"
+load(pathJoin("bacio", bacio_ver))
+
+g2_ver=os.getenv("g2_ver") or "3.5.1"
+load(pathJoin("g2", g2_ver))
+
+ip_ver=os.getenv("ip_ver") or "5.0.0"
+load(pathJoin("ip", ip_ver))
+
+nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
+load(pathJoin("nemsio", nemsio_ver))
+
+sp_ver=os.getenv("sp_ver") or "2.5.0"
+load(pathJoin("sp", sp_ver))
+
+w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
+load(pathJoin("w3emc", w3emc_ver))
+
+sigio_ver=os.getenv("sigio_ver") or "2.3.2"
+load(pathJoin("sigio", sigio_ver))
+
+png_ver=os.getenv("png_ver") or "1.6.37"
+load(pathJoin("libpng", png_ver))
+
+netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
+load(pathJoin("netcdf-c", netcdf_c_ver))
+
+netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+esmf_ver=os.getenv("esmf_ver") or "8.6.1"
+load(pathJoin("esmf", esmf_ver))
+
+nco_ver=os.getenv("nco_ver") or "5.1.6"
+load(pathJoin("nco", nco_ver))
+
+whatis("Description: UFS_UTILS build environment")

modulefiles/build.gaea.intel.lua → modulefiles/build.gaeac5.intel.lua

@@ -7,10 +7,10 @@ load("hsi")
 
 prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
 
-stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
-stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25"
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.28"
 load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"

modulefiles/build.gaeaC6.intel.lua → modulefiles/build.gaeac6.intel.lua

@@ -1,17 +1,17 @@
 help([[
-Load environment to compile UFS_UTILS on GAEA C6 using Intel
+Load environment to compile UFS_UTILS on Gaea C6 using Intel
 ]])
 
-prepend_path("MODULEPATH", "/autofs/ncrc-svm1_proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env-c6/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/sw/rdtn/modulefiles")
+load("hsi")
+
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
-stack_mpich_ver=os.getenv("stack_mpich_ver") or "8.1.29"
-load(pathJoin("stack-cray-mpich", stack_mpich_ver))
-
-craype_ver=os.getenv("craype_ver") or "2.7.30"
-load(pathJoin("craype", craype_ver))
+stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29"
+load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
@@ -34,17 +34,18 @@ load(pathJoin("sp", sp_ver))
 w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
 load(pathJoin("w3emc", w3emc_ver))
 
+-- Uncomment when CHGRES_ALL is ON
+--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
+--load(pathJoin("sfcio", sfcio_ver))
+
 sigio_ver=os.getenv("sigio_ver") or "2.3.2"
 load(pathJoin("sigio", sigio_ver))
 
-sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
-load(pathJoin("sfcio", sfcio_ver))
-
 zlib_ver=os.getenv("zlib_ver") or "1.2.13"
 load(pathJoin("zlib", zlib_ver))
 
-libpng_ver=os.getenv("libpng_ver") or "1.6.37"
-load(pathJoin("libpng", libpng_ver))
+png_ver=os.getenv("png_ver") or "1.6.37"
+load(pathJoin("libpng", png_ver))
 
 netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
 load(pathJoin("netcdf-c", netcdf_c_ver))
@@ -55,10 +56,11 @@ load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
 nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
 load(pathJoin("nccmp", nccmp_ver))
 
-esmf_ver=os.getenv("esmf_ver") or "8.5.0"
+esmf_ver=os.getenv("esmf_ver") or "8.6.0"
 load(pathJoin("esmf", esmf_ver))
 
 nco_ver=os.getenv("nco_ver") or "5.0.6"
 load(pathJoin("nco", nco_ver))
 
 whatis("Description: UFS_UTILS build environment")
+