Fix some pipelines

CliMA · Jun 5, 2024 · de081f9 · de081f9
1 parent 61d2ac8
commit de081f9
Show file tree

Hide file tree

Showing 6 changed files with 120 additions and 13 deletions.
diff --git a/.buildkite/comparison/pipeline.sh b/.buildkite/comparison/pipeline.sh
@@ -88,7 +88,7 @@ cat << EOM
         - "$job_id/scaling_data_${nprocs}_processes.jld2"
         - "$job_id/report.*.nsys-rep"
       env:
-        CLIMACORE_DISTRIBUTED: "MPI"
+        CLIMACOMMS_CONTEXT: "MPI"
         NPROCS: $nprocs
       agents:
         config: cpu

diff --git a/.buildkite/gpu_pipeline/pipeline.yml b/.buildkite/gpu_pipeline/pipeline.yml
@@ -56,6 +56,8 @@ steps:
             --config_file ${GPU_CONFIG_PATH}target_gpu_implicit_baroclinic_wave.yml
             --job_id target_gpu_implicit_baroclinic_wave
         artifact_paths: "target_gpu_implicit_baroclinic_wave/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_cpus_per_task: 4
@@ -71,6 +73,8 @@ steps:
             --config_file ${GPU_CONFIG_PATH}gpu_hs_rhoe_equil_55km_nz63_0M.yml
             --job_id gpu_hs_rhoe_equil_55km_nz63_0M
         artifact_paths: "gpu_hs_rhoe_equil_55km_nz63_0M/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_cpus_per_task: 4
@@ -87,6 +91,9 @@ steps:
             --config_file ${GPU_CONFIG_PATH}gpu_hs_rhoe_equil_55km_nz63_0M_4process.yml
             --job_id gpu_hs_rhoe_equil_55km_nz63_0M_4process
         artifact_paths: "gpu_hs_rhoe_equil_55km_nz63_0M_4process/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
@@ -104,6 +111,9 @@ steps:
             --config_file ${GPU_CONFIG_PATH}target_gpu_implicit_baroclinic_wave_4process.yml
             --job_id target_gpu_implicit_baroclinic_wave_4process
         artifact_paths: "target_gpu_implicit_baroclinic_wave_4process/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
@@ -123,6 +133,9 @@ steps:
             --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ss_1process.yml
             --job_id gpu_aquaplanet_dyamond_ss_1process
         artifact_paths: "gpu_aquaplanet_dyamond_ss_1process/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
@@ -140,6 +153,9 @@ steps:
             --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ss_2process.yml
             --job_id gpu_aquaplanet_dyamond_ss_2process
         artifact_paths: "gpu_aquaplanet_dyamond_ss_2process/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
@@ -157,6 +173,9 @@ steps:
             --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ss_4process.yml
             --job_id gpu_aquaplanet_dyamond_ss_4process
         artifact_paths: "gpu_aquaplanet_dyamond_ss_4process/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
@@ -173,6 +192,8 @@ steps:
           - >
             julia --color=yes --project=examples post_processing/plot_gpu_strong_scaling.jl gpu_aquaplanet_dyamond_ss
         artifact_paths: "gpu_aquaplanet_dyamond_ss/*"
+        env:
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_cpus_per_task: 1
           slurm_ntasks: 1
@@ -190,6 +211,9 @@ steps:
             --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ws_1process.yml
             --job_id gpu_aquaplanet_dyamond_ws_1process
         artifact_paths: "gpu_aquaplanet_dyamond_ws_1process/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
@@ -207,6 +231,9 @@ steps:
             --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ws_2process.yml
             --job_id gpu_aquaplanet_dyamond_ws_2process
         artifact_paths: "gpu_aquaplanet_dyamond_ws_2process/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
@@ -224,6 +251,9 @@ steps:
             --config_file ${GPU_CONFIG_PATH}gpu_aquaplanet_dyamond_ws_4process.yml
             --job_id gpu_aquaplanet_dyamond_ws_4process
         artifact_paths: "gpu_aquaplanet_dyamond_ws_4process/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
@@ -240,6 +270,9 @@ steps:
           - >
             julia --color=yes --project=examples post_processing/plot_gpu_weak_scaling.jl gpu_aquaplanet_dyamond_ws
         artifact_paths: "gpu_aquaplanet_dyamond_ws/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_cpus_per_task: 1
           slurm_ntasks: 1
@@ -259,6 +292,9 @@ steps:
             --config_file ${MODEL_CONFIG_PATH}aquaplanet_diagedmf.yml
             --job_id gpu_aquaplanet_diagedmf
         artifact_paths: "gpu_aquaplanet_diagedmf/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
@@ -272,6 +308,8 @@ steps:
           --config_file ${MODEL_CONFIG_PATH}aquaplanet_diagedmf.yml
           --job_id gpu_aquaplanet_diagedmf_benchmark
         artifact_paths: "gpu_aquaplanet_diagedmf_benchmark/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
 
@@ -287,6 +325,9 @@ steps:
             --config_file ${MODEL_CONFIG_PATH}aquaplanet_progedmf.yml
             --job_id gpu_aquaplanet_progedmf
         artifact_paths: "gpu_aquaplanet_progedmf/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_gpus_per_task: 1
           slurm_cpus_per_task: 4
@@ -300,5 +341,7 @@ steps:
           --config_file ${MODEL_CONFIG_PATH}aquaplanet_progedmf.yml
           --job_id gpu_aquaplanet_progedmf_benchmark
         artifact_paths: "gpu_aquaplanet_progedmf_benchmark/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml
@@ -47,6 +47,8 @@ steps:
       #   command:
       #     - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
       #   artifact_paths: "$$JOB_NAME/output_active/*"
+      #   env:
+      #     CLIMACOMMS_CONTEXT: "MPI"
       #   agents:
       #     slurm_ntasks: 32
       #     slurm_mem_per_cpu: 32GB
@@ -58,6 +60,8 @@ steps:
         command:
           - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_ntasks: 32
           slurm_nodes: 2
@@ -69,6 +73,8 @@ steps:
         command:
           - srun julia --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
           slurm_ntasks: 64
           slurm_nodes: 4

diff --git a/.buildkite/longruns_gpu/pipeline.yml b/.buildkite/longruns_gpu/pipeline.yml
@@ -46,6 +46,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -56,6 +58,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -66,6 +70,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -76,6 +82,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -86,6 +94,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -97,7 +107,8 @@ steps:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
         env:
-          CLIMACORE_DISTRIBUTED: "MPI"
+          CLIMACOMMS_CONTEXT: "MPI"
+          CLIMACOMMS_DEVICE: "CUDA"
           JOB_NAME: "longrun_hs_rhoe_equil_55km_nz63_0M_deepatmos"
         agents:
           slurm_gpus: 1
@@ -111,7 +122,8 @@ steps:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
         env:
-          CLIMACORE_DISTRIBUTED: "MPI"
+          CLIMACOMMS_CONTEXT: "MPI"
+          CLIMACOMMS_DEVICE: "CUDA"
           JOB_NAME: "longrun_aquaplanet_rhoe_equil_55km_nz63_gray_0M"
         agents:
           slurm_gpus: 1
@@ -121,6 +133,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -131,6 +145,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -141,6 +157,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -151,6 +169,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -161,6 +181,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 6:00:00
@@ -171,6 +193,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 6:00:00
@@ -181,6 +205,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -195,6 +221,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00
@@ -209,6 +237,8 @@ steps:
         command:
           - srun julia --color=yes --project=examples examples/hybrid/driver.jl --config_file $CONFIG_PATH/$$JOB_NAME.yml --job_id $$JOB_NAME
         artifact_paths: "$$JOB_NAME/output_active/*"
+        env:
+          CLIMACOMMS_DEVICE: "CUDA"
         agents:
           slurm_gpus: 1
           slurm_time: 12:00:00