Port to Intel LLVM on Hera, Orion and Hercules

.github/workflows/macos_clang.yaml

@@ -47,7 +47,7 @@ jobs:
         run: |
           git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
-          sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[apple-clang@14\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
+          sed "s/\[oneapi, gcc@10:10, apple-clang@14\]/\[apple-clang@14\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
           spack env create ufs_utils-env spack_ci.yaml
           spack env activate ufs_utils-env
           spack external find

.github/workflows/ubuntu_gcc.yaml

@@ -47,7 +47,7 @@ jobs:
         run: |
           git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
-          sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[gcc@10:10\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
+          sed "s/\[oneapi, gcc@10:10, apple-clang@14\]/\[gcc@10:10\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
           spack env create ufs_utils-env spack_ci.yaml
           spack env activate ufs_utils-env
           sudo apt install cmake

.github/workflows/ubuntu_intel.yaml

@@ -7,14 +7,14 @@ defaults:
   run:
     shell: bash -leo pipefail {0}
 
-# Set I_MPI_CC/F90 so Intel MPI wrapper uses icc/ifort instead of gcc/gfortran
+# Set I_MPI_CC/F90 so IntelLLVM is used.
 env:
   cache_key: intel
-  CC: icc
-  FC: ifort
-  CXX: icpc
-  I_MPI_CC: icc
-  I_MPI_F90: ifort
+  CC: mpiicc
+  FC: mpiifort
+  CXX: mpiicpc
+  I_MPI_CC: icx
+  I_MPI_F90: ifx
 
 # The jobs are split into:
 # 1. a dependency build step (setup), and
@@ -59,7 +59,7 @@ jobs:
         run: |
           git clone -c feature.manyFiles=true https://github.com/JCSDA/spack.git
           source spack/share/spack/setup-env.sh
-          sed "s/\[intel, gcc@10:10, apple-clang@14\]/\[intel\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
+          sed "s/\[oneapi, gcc@10:10, apple-clang@14\]/\[oneapi\]/g" ufs_utils/ci/spack.yaml > spack_ci.yaml
           spack env create ufs_utils-env spack_ci.yaml
           spack env activate ufs_utils-env
           spack compiler find
@@ -99,8 +99,8 @@ jobs:
         run: |
           source spack/share/spack/setup-env.sh
           spack env activate ufs_utils-env
-          export CC=mpiicc
-          export FC=mpiifort
+          export CC=mpiicx
+          export FC=mpiifx
           cd ufs_utils
           mkdir -p build && cd build
           cmake -DCMAKE_INSTALL_PREFIX=../install ..

CMakeLists.txt

@@ -26,7 +26,7 @@ option(CHGRES "Enable building chgres_cube.fd" ON)
 option(CHGRES_ALL "Build chgres with all input data options." OFF)
 option(OROG_MASK_TOOLS "Enable building orog_mask_tools.fd" ON)
 # OROG_MASK_TOOLS must be ON for OROG_NETCDF_TOOLS to build.
-option(OROG_NETCDF_TOOLS "Enable building orog_netcdf_tools.fd" OFF)
+option(OROG_NETCDF_TOOLS "Enable building orog_netcdf_tools.fd" ON)
 option(SFC_CLIMO_GEN "Enable building sfc_climo_gen.fd" ON)
 option(VCOORD_GEN "Enable building vcoord_gen.fd" ON)
 option(FVCOMTOOLS "Enable building fvcom_tools.fd" ON)
@@ -67,10 +67,15 @@ if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$")
 endif()
 
 # Set compiler flags.
-if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$")
+if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -traceback")
   set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -fp-model precise")
   set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check all -ftrapuv")
+# On some machines (Hera), using -check all results in an error
+# at the linking step. The uninit check is the culprit.
+  if(CMAKE_Fortran_COMPILER_ID MATCHES "^(IntelLLVM)$")
+    set(CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -check nouninit")
+  endif()
   if(APPLE)
     # The linker on macOS does not include `common symbols` by default.
     # Passing the -c flag includes them and fixes an error with undefined symbols.
@@ -86,10 +91,14 @@ elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$")
   set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -ggdb -Wall -fcheck=all")
 endif()
 
-if(CMAKE_C_COMPILER_ID MATCHES "^(Intel)$")
+if(CMAKE_C_COMPILER_ID MATCHES "^(Intel|IntelLLVM)$")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -traceback")
-  set(CMAKE_C_FLAGS_RELEASE "-O2")
   set(CMAKE_C_FLAGS_DEBUG "-O0")
+  if(CMAKE_C_COMPILER_ID MATCHES "^(Intel)$")
+    set(CMAKE_C_FLAGS_RELEASE "-O2")
+  else()
+    set(CMAKE_C_FLAGS_RELEASE "-O1")
+  endif()
 endif()
 
 # Find packages.

build_all.sh

@@ -19,7 +19,8 @@ source "${DIR_ROOT}/sorc/machine-setup.sh"
 
 # User Options
 target=${target:-"NULL"}
-compiler=${compiler:-"intel"}
+compiler=${compiler:-"intelllvm"} # If IntelLLVM is not available on the machine, will
+                                  # default to Intel classic.
 
 if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
   unset -f module
@@ -29,6 +30,12 @@ if [[ "$target" == "linux.*" || "$target" == "macosx.*" ]]; then
 else
   set +x
   module use "${DIR_ROOT}/modulefiles"
+  if [[ "$compiler" == "intelllvm" ]]; then
+    if [[ ! -f ${DIR_ROOT}/modulefiles/build.$target.$compiler.lua ]];then
+      echo "IntelLLVM not available. Will use Intel Classic."
+      compiler=intel
+    fi
+  fi
   module load "build.$target.$compiler" > /dev/null
   module list
   set -x

ci/spack.yaml

@@ -2,7 +2,7 @@
 spack:
   packages:
     all:
-      compiler: [intel, gcc@10:10, apple-clang@14]
+      compiler: [oneapi, gcc@10:10, apple-clang@14]
   specs:
   - [email protected]
   - [email protected]
@@ -12,8 +12,7 @@ spack:
   - [email protected]
   - [email protected]
   - [email protected]
-  - [email protected]
-  - [email protected]
+  - [email protected]
   - [email protected]
   - [email protected]+fortran~pnetcdf
   - [email protected]

driver_scripts/driver_grid.hera.sh

@@ -64,7 +64,7 @@
 
 set -x
 
-compiler=${compiler:-"intel"}
+compiler=${compiler:-"intelllvm"}
 source ../sorc/machine-setup.sh > /dev/null 2>&1
 module use ../modulefiles
 module load build.$target.$compiler

driver_scripts/driver_grid.hercules.sh

@@ -64,7 +64,7 @@ set -x
 
 source ../sorc/machine-setup.sh > /dev/null 2>&1
 module use ../modulefiles
-module load build.$target.intel
+module load build.$target.intelllvm
 module list
 
 #-----------------------------------------------------------------------

driver_scripts/driver_grid.orion.sh

@@ -64,7 +64,7 @@ set -x
 
 source ../sorc/machine-setup.sh > /dev/null 2>&1
 module use ../modulefiles
-module load build.$target.intel
+module load build.$target.intelllvm
 module list
 
 #-----------------------------------------------------------------------

modulefiles/build.hera.intelllvm.lua

@@ -0,0 +1,74 @@
+help([[
+Load environment to compile UFS_UTILS on Hera using Intel
+]])
+
+hpss_ver=os.getenv("hpss_ver") or ""
+load(pathJoin("hpss", hpss_ver))
+
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+load("gnu")
+load("intel/2024.2.1")
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+bacio_ver=os.getenv("bacio_ver") or "2.4.1"
+load(pathJoin("bacio", bacio_ver))
+
+g2_ver=os.getenv("g2_ver") or "3.4.5"
+load(pathJoin("g2", g2_ver))
+
+ip_ver=os.getenv("ip_ver") or "4.3.0"
+load(pathJoin("ip", ip_ver))
+
+nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
+load(pathJoin("nemsio", nemsio_ver))
+
+sp_ver=os.getenv("sp_ver") or "2.5.0"
+load(pathJoin("sp", sp_ver))
+
+w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
+load(pathJoin("w3emc", w3emc_ver))
+
+-- Uncomment when CHGRES_ALL is ON
+--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
+--load(pathJoin("sfcio", sfcio_ver))
+
+sigio_ver=os.getenv("sigio_ver") or "2.3.2"
+load(pathJoin("sigio", sigio_ver))
+
+zlib_ver=os.getenv("zlib_ver") or "1.2.13"
+load(pathJoin("zlib", zlib_ver))
+
+png_ver=os.getenv("png_ver") or "1.6.37"
+load(pathJoin("libpng", png_ver))
+
+netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
+load(pathJoin("netcdf-c", netcdf_c_ver))
+
+netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+esmf_ver=os.getenv("esmf_ver") or "8.6.0"
+load(pathJoin("esmf", esmf_ver))
+
+nco_ver=os.getenv("nco_ver") or "5.0.6"
+load(pathJoin("nco", nco_ver))
+
+setenv("I_MPI_CC", "icx")
+setenv("I_MPI_FC", "ifx")
+
+setenv("CC", "mpicc")
+setenv("FC", "mpifc")
+
+whatis("Description: UFS_UTILS build environment")

modulefiles/build.hercules.intel.lua

@@ -1,5 +1,5 @@
 help([[
-Load environment to compile UFS_UTILS on Hercules using Intel
+Load environment to compile UFS_UTILS on Hercules using Intel Classic
 ]])
 
 prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")

modulefiles/build.hercules.intelllvm.lua

@@ -0,0 +1,68 @@
+help([[
+Load environment to compile UFS_UTILS on Hercules using IntelLLVM
+]])
+
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+bacio_ver=os.getenv("bacio_ver") or "2.4.1"
+load(pathJoin("bacio", bacio_ver))
+
+g2_ver=os.getenv("g2_ver") or "3.4.5"
+load(pathJoin("g2", g2_ver))
+
+ip_ver=os.getenv("ip_ver") or "4.3.0"
+load(pathJoin("ip", ip_ver))
+
+nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
+load(pathJoin("nemsio", nemsio_ver))
+
+sp_ver=os.getenv("sp_ver") or "2.5.0"
+load(pathJoin("sp", sp_ver))
+
+w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
+load(pathJoin("w3emc", w3emc_ver))
+
+-- Uncomment when CHGRES_ALL is ON
+--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
+--load(pathJoin("sfcio", sfcio_ver))
+
+sigio_ver=os.getenv("sigio_ver") or "2.3.2"
+load(pathJoin("sigio", sigio_ver))
+
+zlib_ver=os.getenv("zlib_ver") or "1.2.13"
+load(pathJoin("zlib", zlib_ver))
+
+png_ver=os.getenv("png_ver") or "1.6.37"
+load(pathJoin("libpng", png_ver))
+
+netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
+load(pathJoin("netcdf-c", netcdf_c_ver))
+
+netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+esmf_ver=os.getenv("esmf_ver") or "8.6.0"
+load(pathJoin("esmf", esmf_ver))
+
+nco_ver=os.getenv("nco_ver") or "5.0.6"
+load(pathJoin("nco", nco_ver))
+
+setenv("I_MPI_CC", "icx")
+setenv("I_MPI_F90", "ifx")
+
+setenv("CC", "mpiicc")
+setenv("FC", "mpiifort")
+
+whatis("Description: UFS_UTILS build environment")

modulefiles/build.orion.intel.lua

@@ -1,5 +1,5 @@
 help([[
-Load environment to compile UFS_UTILS on Orion using Intel
+Load environment to compile UFS_UTILS on Orion using Intel classic
 ]])
 
 prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core")

modulefiles/build.orion.intelllvm.lua

@@ -0,0 +1,68 @@
+help([[
+Load environment to compile UFS_UTILS on Orion using IntelLLVM
+]])
+
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+bacio_ver=os.getenv("bacio_ver") or "2.4.1"
+load(pathJoin("bacio", bacio_ver))
+
+g2_ver=os.getenv("g2_ver") or "3.4.5"
+load(pathJoin("g2", g2_ver))
+
+ip_ver=os.getenv("ip_ver") or "4.3.0"
+load(pathJoin("ip", ip_ver))
+
+nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
+load(pathJoin("nemsio", nemsio_ver))
+
+sp_ver=os.getenv("sp_ver") or "2.5.0"
+load(pathJoin("sp", sp_ver))
+
+w3emc_ver=os.getenv("w3emc_ver") or "2.10.0"
+load(pathJoin("w3emc", w3emc_ver))
+
+-- Uncomment when CHGRES_ALL is ON
+--sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
+--load(pathJoin("sfcio", sfcio_ver))
+
+sigio_ver=os.getenv("sigio_ver") or "2.3.2"
+load(pathJoin("sigio", sigio_ver))
+
+zlib_ver=os.getenv("zlib_ver") or "1.2.13"
+load(pathJoin("zlib", zlib_ver))
+
+png_ver=os.getenv("png_ver") or "1.6.37"
+load(pathJoin("libpng", png_ver))
+
+netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2"
+load(pathJoin("netcdf-c", netcdf_c_ver))
+
+netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1"
+load(pathJoin("netcdf-fortran", netcdf_fortran_ver))
+
+nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1"
+load(pathJoin("nccmp", nccmp_ver))
+
+esmf_ver=os.getenv("esmf_ver") or "8.6.0"
+load(pathJoin("esmf", esmf_ver))
+
+nco_ver=os.getenv("nco_ver") or "5.0.6"
+load(pathJoin("nco", nco_ver))
+
+setenv("I_MPI_CC", "icx")
+setenv("I_MPI_F90", "ifx")
+
+setenv("CC", "mpiicc")
+setenv("FC", "mpiifort")
+
+whatis("Description: UFS_UTILS build environment")