Use MPITrampoline

.buildkite/JuliaProject.toml

@@ -1,22 +1,15 @@
 [extras]
-CUDA_Driver_jll = "4ee394cb-3365-5eb0-8335-949819d2adfc"
 CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
-HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59"
+CUDA_Driver_jll = "4ee394cb-3365-5eb0-8335-949819d2adfc"
 MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 
 [preferences.CUDA_Driver_jll]
 compat = false
 
 [preferences.CUDA_Runtime_jll]
-version = "local"
-
-[preferences.HDF5_jll]
-libhdf5_path = "libhdf5"
-libhdf5_hl_path = "libhdf5_hl"
+version = "12.2"
+local = "true"
 
 [preferences.MPIPreferences]
 _format = "1.0"
-abi = "OpenMPI"
-binary = "system"
-libmpi = "libmpi"
-mpiexec = "mpiexec"
+binary = "MPItrampoline_jll"

.buildkite/longruns/pipeline.yml

@@ -1,7 +1,7 @@
 agents:
   queue: central
   slurm_time: 24:00:00
-  modules: julia/1.9.4 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.3.1
+  modules: julia/1.9.4 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 nsight-systems/2023.3.1
 
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
@@ -15,7 +15,11 @@ env:
   CONFIG_PATH: "config/longrun_configs"
   PERF_CONFIG_PATH: "config/perf_configs"
 
-  LD_LIBRARY_PATH: "/central/scratch/esm/slurm-buildkite/climaatmos-ci/depot/cpu/artifacts/e9feeabfa989549ac1dac61e4df5f2849149ad6d/lib/:${LD_LIBRARY_PATH}"
+  JULIA_NVTX_CALLBACKS: gc
+  JULIA_CUDA_MEMORY_POOL: none
+  JULIA_MPI_HAS_CUDA: "true"
+  MPITRAMPOLINE_LIB: "/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/lib64/libmpiwrapper.so"
+  MPITRAMPOLINE_MPIEXEC: "/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/bin/mpiwrapperexec"
 
 timeout_in_minutes: 1440
 
@@ -24,8 +28,13 @@ steps:
     key: "init_cpu_env"
     command:
 
-      - echo "--- Configure MPI"
-      - julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()'
+      # Local MPIPreferences take precedence over the one set by JuliaProject.toml,
+      # so we make sure that we remove them to use MPITrampoline.
+      #
+      # TODO: This can be removed once all the branches that use MPIPreferences
+      # are deleted
+      - echo "--- Remove MPIPreferences"
+      - "rm -f ${JULIA_DEPOT_PATH}/environments/v1.9/LocalPreferences.toml"
 
       - echo "--- Instantiate AMIP env"
       - "julia --project=experiments/AMIP/modular/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"

.buildkite/pipeline.yml

@@ -1,7 +1,7 @@
 agents:
   queue: central
   slurm_time: 24:00:00
-  modules: julia/1.9.4 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.3.1
+  modules: julia/1.9.4 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 nsight-systems/2023.3.1
 
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
@@ -11,13 +11,17 @@ env:
   OMPI_MCA_opal_warn_on_missing_libcuda: 0
   JULIA_MAX_NUM_PRECOMPILE_FILES: 100
   GKSwstype: 100
+  SLURM_KILL_BAD_EXIT: 1
+  JULIA_NVTX_CALLBACKS: gc
+  JULIA_CUDA_MEMORY_POOL: none
+  JULIA_MPI_HAS_CUDA: "true"
+  MPITRAMPOLINE_LIB: "/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/lib64/libmpiwrapper.so"
+  MPITRAMPOLINE_MPIEXEC: "/groups/esm/software/MPIwrapper/ompi4.1.5_cuda-12.2/bin/mpiwrapperexec"
 
   CONFIG_PATH: "config/model_configs"
   PERF_CONFIG_PATH: "config/perf_configs"
   MPI_CONFIG_PATH: "config/mpi_configs"
 
-  LD_LIBRARY_PATH: "/central/scratch/esm/slurm-buildkite/climaatmos-ci/depot/cpu/artifacts/e9feeabfa989549ac1dac61e4df5f2849149ad6d/lib/:${LD_LIBRARY_PATH}"
-
 timeout_in_minutes: 1440
 
 steps:
@@ -30,8 +34,13 @@ steps:
           rm -rf ${JULIA_DEPOT_PATH}
         fi
 
-      - echo "--- Configure MPI"
-      - julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()'
+      # Local MPIPreferences take precedence over the one set by JuliaProject.toml,
+      # so we make sure that we remove them to use MPITrampoline.
+      #
+      # TODO: This can be removed once all the branches that use MPIPreferences
+      # are deleted
+      - echo "--- Remove MPIPreferences"
+      - "rm -f ${JULIA_DEPOT_PATH}/environments/v1.9/LocalPreferences.toml"
 
       - echo "--- Instantiate package env"
       - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true)'"