update dependencies

.buildkite/JuliaProject.toml

@@ -1,22 +1,15 @@
 [extras]
-CUDA_Driver_jll = "4ee394cb-3365-5eb0-8335-949819d2adfc"
 CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
-HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59"
+CUDA_Driver_jll = "4ee394cb-3365-5eb0-8335-949819d2adfc"
 MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 
 [preferences.CUDA_Driver_jll]
 compat = false
 
 [preferences.CUDA_Runtime_jll]
-version = "local"
-
-[preferences.HDF5_jll]
-libhdf5_path = "libhdf5"
-libhdf5_hl_path = "libhdf5_hl"
+version = "12.2"
+local = "true"
 
 [preferences.MPIPreferences]
 _format = "1.0"
-abi = "OpenMPI"
-binary = "system"
-libmpi = "libmpi"
-mpiexec = "mpiexec"
+binary = "MPItrampoline_jll"

.buildkite/longruns/pipeline.yml

@@ -1,7 +1,7 @@
 agents:
   queue: central
   slurm_time: 24:00:00
-  modules: julia/1.9.4 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
+  modules: julia/1.9.4 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 nsight-systems/2023.3.1
 
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
@@ -14,7 +14,11 @@ env:
   CONFIG_PATH: "config/longrun_configs"
   PERF_CONFIG_PATH: "config/perf_configs"
 
-  LD_LIBRARY_PATH: "/central/scratch/esm/slurm-buildkite/climaatmos-ci/depot/cpu/artifacts/e9feeabfa989549ac1dac61e4df5f2849149ad6d/lib/:${LD_LIBRARY_PATH}"
+  JULIA_NVTX_CALLBACKS: gc
+  JULIA_CUDA_MEMORY_POOL: none
+  JULIA_MPI_HAS_CUDA: "true"
+  MPITRAMPOLINE_LIB: "/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/lib64/libmpiwrapper.so"
+  MPITRAMPOLINE_MPIEXEC: "/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/bin/mpiwrapperexec"
 
 timeout_in_minutes: 1440
 
@@ -23,8 +27,13 @@ steps:
     key: "init_cpu_env"
     command:
 
-      - echo "--- Configure MPI"
-      - julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()'
+      # Local MPIPreferences take precedence over the one set by JuliaProject.toml,
+      # so we make sure that we remove them to use MPITrampoline.
+      #
+      # TODO: This can be removed once all the branches that use MPIPreferences
+      # are deleted
+      - echo "--- Remove MPIPreferences"
+      - "rm -f ${JULIA_DEPOT_PATH}/environments/v1.9/LocalPreferences.toml"
 
       - echo "--- Instantiate AMIP env"
       - "julia --project=experiments/AMIP/modular/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"

.buildkite/pipeline.yml

@@ -1,7 +1,7 @@
 agents:
   queue: central
   slurm_time: 24:00:00
-  modules: julia/1.9.4 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
+  modules: julia/1.9.4 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 nsight-systems/2023.3.1
 
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
@@ -10,22 +10,31 @@ env:
   OMPI_MCA_opal_warn_on_missing_libcuda: 0
   JULIA_MAX_NUM_PRECOMPILE_FILES: 100
   GKSwstype: 100
+  SLURM_KILL_BAD_EXIT: 1
+  JULIA_NVTX_CALLBACKS: gc
+  JULIA_CUDA_MEMORY_POOL: none
+  JULIA_MPI_HAS_CUDA: "true"
+  MPITRAMPOLINE_LIB: "/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/lib64/libmpiwrapper.so"
+  MPITRAMPOLINE_MPIEXEC: "/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/bin/mpiwrapperexec"
 
   CONFIG_PATH: "config/model_configs"
   PERF_CONFIG_PATH: "config/perf_configs"
   MPI_CONFIG_PATH: "config/mpi_configs"
 
-  LD_LIBRARY_PATH: "/central/scratch/esm/slurm-buildkite/climaatmos-ci/depot/cpu/artifacts/e9feeabfa989549ac1dac61e4df5f2849149ad6d/lib/:${LD_LIBRARY_PATH}"
-
 timeout_in_minutes: 1440
 
 steps:
   - label: "init environment :computer:"
     key: "init_cpu_env"
     command:
 
-      - echo "--- Configure MPI"
-      - julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()'
+      # Local MPIPreferences take precedence over the one set by JuliaProject.toml,
+      # so we make sure that we remove them to use MPITrampoline.
+      #
+      # TODO: This can be removed once all the branches that use MPIPreferences
+      # are deleted
+      - echo "--- Remove MPIPreferences"
+      - "rm -f ${JULIA_DEPOT_PATH}/environments/v1.9/LocalPreferences.toml"
 
       - echo "--- Instantiate package env"
       - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"

.dev/Project.toml

@@ -1,8 +1,5 @@
 [deps]
-CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
-ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884"
 JuliaFormatter = "98e50ef6-434e-11e9-1051-2b60c6c9e899"
 
 [compat]
-CSTParser = "3.3.1"
-JuliaFormatter = "0.22"
+JuliaFormatter = "1"

Project.toml

@@ -25,16 +25,16 @@ Thermodynamics = "b60c26fb-14c3-4610-9d3e-2d17fe7ff00c"
 
 [compat]
 CLIMAParameters = "0.4, 0.5, 0.6, 0.7"
-ClimaAtmos = "0.16.3"
+ClimaAtmos = "0.17.1"
 ClimaComms = "0.5.6"
-ClimaCore = "0.10"
-ClimaCoreTempestRemap = "0.3.0 - 0.3.10"
+ClimaCore = "0.11.3"
+ClimaCoreTempestRemap = "0.3"
 ClimaLSM = "0.4"
 Dates = "1"
 DocStringExtensions = "0.8, 0.9"
-Insolation = "0.6"
+Insolation = "0.8"
 JLD2 = "0.4"
-NCDatasets = "0.11, 0.12"
+NCDatasets = "0.11, 0.12, 0.13"
 Plots = "1.39.0"
 SciMLBase = "1, 2"
 StaticArrays = "1"