init

no timeout

explicit long timeout

srun

slurm_mem: 20GB

clima

turn off coupler diags

turn off coupler diags

clean

split clima init

rm concurrency block

clean

.buildkite/longruns/pipeline.yml

@@ -39,14 +39,8 @@ steps:
       JULIA_NUM_PRECOMPILE_TASKS: 8
       JULIA_MAX_NUM_PRECOMPILE_FILES: 50
 
-    concurrency: 1
-    concurrency_group: 'depot/climacoupler-longruns'
-
-  - wait
-
-  # DYAMOND AMIP: 1 day (convection resolving) - running on clima, so need to instantiate again
-  - label: "GPU AMIP SUPERFINE: dyamond_target"
-    key: "gpu_dyamond_target"
+  - label: "init clima :computer:"
+    key: "init_cpu_env_clima"
     command:
       - "echo $$JULIA_DEPOT_PATH"
 
@@ -61,15 +55,14 @@ steps:
       - "julia --project=artifacts -e 'using Pkg; Pkg.status()'"
       - "julia --project=artifacts artifacts/download_artifacts.jl"
 
-      - echo "--- Run simulation"
-      - "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_dyamond_target.yml"
-    artifact_paths: "experiments/AMIP/output/amip/gpu_dyamond_target_artifacts/*"
     agents:
       queue: clima
-      slurm_mem: 20GB
-      slurm_gpus: 1
       modules: common
-    soft_fail: true
+    env:
+      JULIA_NUM_PRECOMPILE_TASKS: 8
+      JULIA_MAX_NUM_PRECOMPILE_FILES: 50
+
+  - wait
 
   - group: "Coupler integration and conservation tests"
 
@@ -289,13 +282,35 @@ steps:
 
       - label: "GPU AMIP FINE: new target amip: topo + diagedmf"
         key: "gpu_amip_target_topo_diagedmf"
-        command: "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf.yml"
+        command:
+          - echo "--- Run simulation"
+          - "srun julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_amip_target_topo_diagedmf.yml"
         artifact_paths: "experiments/AMIP/output/amip/gpu_amip_target_topo_diagedmf_artifacts/*"
+        timeout_in_minutes: 1440
+        env:
+          CLIMACOMMS_CONTEXT: "MPI"
         agents:
-          slurm_gpus: 1
-          slurm_mem: 16GB
+          queue: clima
+          slurm_ntasks: 4
+          slurm_gpus: 4
+          slurm_mem: 20GB
+          modules: common
         soft_fail: true
 
+  # DYAMOND AMIP: 1 day (convection resolving) - running on clima, so need to instantiate again
+  - label: "GPU AMIP SUPERFINE: dyamond_target"
+    key: "gpu_dyamond_target"
+    command:
+      - echo "--- Run simulation"
+      - "julia --color=yes --project=experiments/AMIP/ experiments/AMIP/coupler_driver.jl --config_file $CONFIG_PATH/gpu_dyamond_target.yml"
+    artifact_paths: "experiments/AMIP/output/amip/gpu_dyamond_target_artifacts/*"
+    agents:
+      queue: clima
+      slurm_mem: 20GB
+      slurm_gpus: 1
+      modules: common
+    soft_fail: true
+
   - wait
 
   - group: "Job analysis and reporting"
@@ -323,7 +338,7 @@ steps:
 
       - label: ":envelope: Slack report: target AMIP"
         command:
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/amip_target_topo_diagedmf_artifacts/amip_ncep.png -m png -n amip_fine -x "Target AMIP v NCEP Last Month Mean"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/output/amip/amip_target_topo_diagedmf_artifacts/amip_ncep.png -m png -n amip_fine -x "90d Target AMIP v NCEP Last Month Mean"
           - |
             find experiments/AMIP/output/amip/amip_target_topo_diagedmf_artifacts/ -type f -name 'bias*.png' -print0 | while IFS= read -r -d '' file; do
               slack-upload -c "#coupler-report" -f "$$file" -m png -n "$$(basename "$$file" .png)" -x "$$(basename "$$file" .png)"

config/longrun_configs/gpu_amip_target_topo_diagedmf.yml

@@ -19,7 +19,7 @@ netcdf_output_at_levels: true
 run_name: "gpu_amip_target_topo_diagedmf"
 start_date: "19790301"
 surface_setup: "PrescribedSurface"
-t_end: "40days"
+t_end: "90days"
 topo_smoothing: true
 topography: "Earth"
 turb_flux_partition: "CombinedStateFluxes"