Enhance long run pipeline with granular testing sets

.buildkite/longruns/pipeline.yml

@@ -16,7 +16,6 @@ env:
 
   LD_LIBRARY_PATH: "/central/scratch/esm/slurm-buildkite/climaatmos-ci/depot/cpu/artifacts/e9feeabfa989549ac1dac61e4df5f2849149ad6d/lib/:${LD_LIBRARY_PATH}"
 
-
 timeout_in_minutes: 1440
 
 steps:
@@ -51,27 +50,109 @@ steps:
 
   - wait
 
-  - group: "Targeted resolution coupled AMIP long runs"
+  - group: "Coupler integration and conservation tests"
 
     steps:
 
-      - label: "Slabplanet: default"
-        key: "slabplanet_default_longrun"
-        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_default_longrun.yml"
-        artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/*"
+      # Integration tests - the expected results were also confirmed locally
+      - label: "Slabplanet_aqua: nocouple"
+        key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*"
         env:
           BUILD_HISTORY_HANDLE: ""
         agents:
           slurm_ntasks_per_node: 1
           slurm_nodes: 1
           slurm_mem_per_cpu: 16G
 
-      # DYAMOND AMIP: 1 day (convection resolving)
+      - label: "Slabplanet_aqua: couple"
+        key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
 
-      - label: "MPI AMIP SUPERFINE: dyamond_target"
-        key: "dyamond_target"
-        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/dyamond_target.yml"
-        artifact_paths: "experiments/AMIP/modular/output/amip/dyamond_target_artifacts/*"
+      - label: "Slabplanet_aqua: coupler fluxes"
+        key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+
+      - label: "Slabplanet_aqua: coupler fluxes, evolving ocean"
+        key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+
+      - label: "Slabplanet_terra: coupler fluxes, evolving bucket"
+        key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_terra.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_terra/slabplanet_terra_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+
+      - label: "Slabplanet: coupler fluxes, evolving ocean and land"
+        key: "slabplanet_coupler_sf_evolve_ocn"
+        command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 1
+          slurm_nodes: 1
+          slurm_mem_per_cpu: 16G
+
+  - group: "Current target tests: idealized surface"
+
+    steps:
+
+      - label: "ClimaAtmos standalone target"
+        command:
+          - srun julia --project=experiments/AMIP/modular/ target/atmos_driver.jl --config_file target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml
+        artifact_paths: "longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation/*"
+        env:
+          BUILD_HISTORY_HANDLE: ""
+          CLIMACORE_DISTRIBUTED: "MPI"
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc"
+        key: "slabplanet_aqua_target_nocouple"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_target_nocouple_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc"
+        key: "slabplanet_aqua_target"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_target_artifacts/*"
         env:
           CLIMACORE_DISTRIBUTED: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -80,11 +161,82 @@ steps:
           slurm_nodes: 4
           slurm_mem_per_cpu: 16G
 
-      # mid-resolution AMIP: longrun (140 days)
-      - label: "MPI AMIP FINE: target longrun"
-        key: "amip_longrun_target"
-        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_longrun_target.yml"
-        artifact_paths: "experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/*"
+      - label: "TARGET IDEALIZED: new target aqua - evolving slab ocean T"
+        key: "slabplanet_aqua_target_evolve_ocn"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket"
+        key: "slabplanet_target"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_target.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_target_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket"
+        key: "slabplanet_target_evolve_ocn"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml"
+        artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_target_evolve_ocn_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+
+  - group: "Current target tests: AMIP surface"
+
+    steps:
+
+      - label: "MPI AMIP FINE: new target amip"
+        key: "amip_target"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_target.yml"
+        artifact_paths: "experiments/AMIP/modular/output/amip/amip_target_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+      - label: "MPI AMIP FINE: new target amip: topo"
+        key: "amip_target_topo"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_target_topo.yml"
+        artifact_paths: "experiments/AMIP/modular/output/amip/amip_target_topo_artifacts/*"
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+          BUILD_HISTORY_HANDLE: ""
+        agents:
+          slurm_ntasks_per_node: 16
+          slurm_nodes: 4
+          slurm_mem_per_cpu: 16G
+
+
+  - group: "Other AMIP targets"
+
+    steps:
+
+      # DYAMOND AMIP: 1 day (convection resolving)
+
+      - label: "MPI AMIP SUPERFINE: dyamond_target"
+        key: "dyamond_target"
+        command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/dyamond_target.yml"
+        artifact_paths: "experiments/AMIP/modular/output/amip/dyamond_target_artifacts/*"
         env:
           CLIMACORE_DISTRIBUTED: "MPI"
           BUILD_HISTORY_HANDLE: ""
@@ -190,14 +342,14 @@ steps:
 
       - label: ":envelope: Slack report: Slabplanet"
         command:
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)"
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation"
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)"
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation"
 
       - label: ":envelope: Slack report: target AMIP"
         command:
-          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun"
+          - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun"
 
       - label: ":envelope: Slack report: Flame Diff"
         command:

.buildkite/pipeline.yml

@@ -1,6 +1,5 @@
 agents:
   queue: central
-  slurm_mem: 8G
   slurm_time: 24:00:00
   modules: julia/1.9.3 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
 
@@ -24,7 +23,6 @@ steps:
   - label: "init environment :computer:"
     key: "init_cpu_env"
     command:
-      # - "echo $$JULIA_DEPOT_PATH"
 
       - echo "--- Configure MPI"
       - julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()'

Project.toml

@@ -18,6 +18,7 @@ NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
 OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
+SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 SurfaceFluxes = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f"
@@ -28,7 +29,7 @@ UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
 
 [compat]
 CLIMAParameters = "0.4, 0.5, 0.6, 0.7"
-ClimaAtmos = "0.16"
+ClimaAtmos = "=0.16.2"
 ClimaComms = "0.5"
 ClimaCore = "0.10"
 ClimaCoreTempestRemap = "0.3"
@@ -40,6 +41,7 @@ NCDatasets = "0.11, 0.12"
 OrdinaryDiffEq = "5, 6"
 Plots = "1.39.0"
 SciMLBase = "1"
+SimpleNonlinearSolve = "=0.1.23"
 StaticArrays = "1"
 SurfaceFluxes = "0.7"
 TempestRemap_jll = "2"

config/longrun_configs/amip_target.yml

@@ -0,0 +1,18 @@
+run_name: "amip_target"
+anim: true
+dt_cpl: 150
+energy_check: false
+mode_name: "amip"
+mono_surface: false
+dt: "150secs"
+t_end: "200days"
+job_id: "amip_target"
+dt_save_to_sol: "10days"
+dt_save_to_disk: "20days"
+apply_limiter: false
+hourly_checkpoint: true
+turb_flux_partition: "CombinedStateFluxes"
+atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml"
+surface_setup: "PrescribedSurface"
+land_albedo_type: "map_temporal"
+start_date: "19790301"

config/longrun_configs/amip_target_topo.yml

@@ -0,0 +1,21 @@
+run_name: "amip_target_topo"
+anim: true
+dt_cpl: 75
+energy_check: false
+mode_name: "amip"
+mono_surface: false
+dt: "150secs"
+t_end: "100days"
+job_id: "amip_target_topo"
+dt_save_to_sol: "10days"
+dt_save_to_disk: "20days"
+apply_limiter: false
+hourly_checkpoint: true
+turb_flux_partition: "CombinedStateFluxes"
+atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml"
+surface_setup: "PrescribedSurface"
+land_albedo_type: "map_temporal"
+start_date: "19790301"
+topography: "Earth"
+topo_smoothing: true
+use_reference_state: false

config/longrun_configs/dyamond_target.yml

@@ -10,5 +10,6 @@ dt_save_to_sol: "0.5days"
 dt_save_to_disk: "0.5days"
 turb_flux_partition: "CombinedStateFluxes"
 atmos_config_file: "config/longrun_configs/longrun_aquaplanet_dyamond.yml"
-atmos_toml_file: "toml/longrun_aquaplanet_dyamond.toml"
-hourly_checkpoint: false
+monthly_checkpoint: false
+land_albedo_type: "map_temporal"
+start_date: "19790301"

config/longrun_configs/slabplanet_aqua_atmos_sf_couple.yml

@@ -0,0 +1,21 @@
+run_name: "slabplanet_aqua_atmos_sf_couple"
+job_id: "slabplanet_aqua_atmos_sf_couple"
+moist: "equil"
+vert_diff: "true"
+rad: "gray"
+precip_model: "0M"
+energy_check: true
+mode_name: "slabplanet_aqua"
+t_end: "20days"
+dt_save_to_sol: "10days"
+dt_cpl: 200
+dt: "200secs"
+mono_surface: true
+h_elem: 6
+anim: true
+apply_limiter: false
+turb_flux_partition: "CombinedStateFluxes"
+surface_setup: "DefaultMoninObukhov"
+land_albedo_type: "function"
+evolving_ocean: false
+start_date: "19790301"

config/longrun_configs/slabplanet_aqua_atmos_sf_nocouple.yml

@@ -0,0 +1,21 @@
+run_name: "slabplanet_aqua_atmos_sf_nocouple"
+job_id: "slabplanet_aqua_atmos_sf_nocouple"
+moist: "equil"
+vert_diff: "true"
+rad: "gray"
+precip_model: "0M"
+energy_check: true
+mode_name: "slabplanet_aqua"
+t_end: "20days"
+dt_save_to_sol: "10days"
+dt_cpl: 1728000
+dt: "200secs"
+mono_surface: true
+h_elem: 6
+anim: true
+apply_limiter: false
+turb_flux_partition: "CombinedStateFluxes"
+surface_setup: "DefaultMoninObukhov"
+land_albedo_type: "function"
+evolving_ocean: false
+start_date: "19790301"