Skip to content

Commit

Permalink
set up surface hooks
Browse files Browse the repository at this point in the history
add bk pipelines

clean

fix long pip

clean interactive setup

temp manifest check in

try

no pkg precomp

revert manifest

revert gitignore

spec atmos version

down SimpleNonlinearSolve

fix path

init pkg env

down OpenMPI_jll

down OpenMPI_jll

HDF5_jll 1.12.2

try

aqua test spec

tst

rev tst

rev tst

rev tst backslash

rev tst backslash

rm bs

rm bs

rm us

rm us

dbl {

artifacts

ca

new t

new t

short path

bs

no us

dot

revs

rm aug

rm old tomls

clean yml

new_target to target

build_history test

revert test
  • Loading branch information
LenkaNovak committed Nov 9, 2023
1 parent 03bbcac commit 8dfbaaa
Show file tree
Hide file tree
Showing 33 changed files with 683 additions and 140 deletions.
194 changes: 173 additions & 21 deletions .buildkite/longruns/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ env:

LD_LIBRARY_PATH: "/central/scratch/esm/slurm-buildkite/climaatmos-ci/depot/cpu/artifacts/e9feeabfa989549ac1dac61e4df5f2849149ad6d/lib/:${LD_LIBRARY_PATH}"


timeout_in_minutes: 1440

steps:
Expand Down Expand Up @@ -51,27 +50,109 @@ steps:

- wait

- group: "Targeted resolution coupled AMIP long runs"
- group: "Coupler integration and conservation tests"

steps:

- label: "Slabplanet: default"
key: "slabplanet_default_longrun"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_default_longrun.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/*"
# Integration tests - the expected results were also confirmed locally
- label: "Slabplanet_aqua: nocouple"
key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*"
env:
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 1
slurm_nodes: 1
slurm_mem_per_cpu: 16G

# DYAMOND AMIP: 1 day (convection resolving)
- label: "Slabplanet_aqua: couple"
key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*"
env:
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 1
slurm_nodes: 1
slurm_mem_per_cpu: 16G

- label: "MPI AMIP SUPERFINE: dyamond_target"
key: "dyamond_target"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/dyamond_target.yml"
artifact_paths: "experiments/AMIP/modular/output/amip/dyamond_target_artifacts/*"
- label: "Slabplanet_aqua: coupler fluxes"
key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*"
env:
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 1
slurm_nodes: 1
slurm_mem_per_cpu: 16G

- label: "Slabplanet_aqua: coupler fluxes, evolving ocean"
key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*"
env:
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 1
slurm_nodes: 1
slurm_mem_per_cpu: 16G

- label: "Slabplanet_terra: coupler fluxes, evolving bucket"
key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_terra.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet_terra/slabplanet_terra_artifacts/*"
env:
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 1
slurm_nodes: 1
slurm_mem_per_cpu: 16G

- label: "Slabplanet: coupler fluxes, evolving ocean and land"
key: "slabplanet_coupler_sf_evolve_ocn"
command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/*"
env:
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 1
slurm_nodes: 1
slurm_mem_per_cpu: 16G

- group: "Current target tests: idealized surface"

steps:

- label: "ClimaAtmos standalone target"
command:
- srun julia --project=experiments/AMIP/modular/ target/atmos_driver.jl --config_file target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml
artifact_paths: "longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation/*"
env:
BUILD_HISTORY_HANDLE: ""
CLIMACORE_DISTRIBUTED: "MPI"
agents:
slurm_ntasks_per_node: 16
slurm_nodes: 4
slurm_mem_per_cpu: 16G

- label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc"
key: "slabplanet_aqua_target_nocouple"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_target_nocouple_artifacts/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 16
slurm_nodes: 4
slurm_mem_per_cpu: 16G

- label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc"
key: "slabplanet_aqua_target"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_target_artifacts/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
BUILD_HISTORY_HANDLE: ""
Expand All @@ -80,11 +161,82 @@ steps:
slurm_nodes: 4
slurm_mem_per_cpu: 16G

# mid-resolution AMIP: longrun (140 days)
- label: "MPI AMIP FINE: target longrun"
key: "amip_longrun_target"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_longrun_target.yml"
artifact_paths: "experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/*"
- label: "TARGET IDEALIZED: new target aqua - evolving slab ocean T"
key: "slabplanet_aqua_target_evolve_ocn"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn_artifacts/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 16
slurm_nodes: 4
slurm_mem_per_cpu: 16G

- label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket"
key: "slabplanet_target"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_target.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_target_artifacts/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 16
slurm_nodes: 4
slurm_mem_per_cpu: 16G

- label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket"
key: "slabplanet_target_evolve_ocn"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml"
artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_target_evolve_ocn_artifacts/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 16
slurm_nodes: 4
slurm_mem_per_cpu: 16G


- group: "Current target tests: AMIP surface"

steps:

- label: "MPI AMIP FINE: new target amip"
key: "amip_target"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_target.yml"
artifact_paths: "experiments/AMIP/modular/output/amip/amip_target_artifacts/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 16
slurm_nodes: 4
slurm_mem_per_cpu: 16G

- label: "MPI AMIP FINE: new target amip: topo"
key: "amip_target_topo"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_target_topo.yml"
artifact_paths: "experiments/AMIP/modular/output/amip/amip_target_topo_artifacts/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
BUILD_HISTORY_HANDLE: ""
agents:
slurm_ntasks_per_node: 16
slurm_nodes: 4
slurm_mem_per_cpu: 16G


- group: "Other AMIP targets"

steps:

# DYAMOND AMIP: 1 day (convection resolving)

- label: "MPI AMIP SUPERFINE: dyamond_target"
key: "dyamond_target"
command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/dyamond_target.yml"
artifact_paths: "experiments/AMIP/modular/output/amip/dyamond_target_artifacts/*"
env:
CLIMACORE_DISTRIBUTED: "MPI"
BUILD_HISTORY_HANDLE: ""
Expand Down Expand Up @@ -190,14 +342,14 @@ steps:

- label: ":envelope: Slack report: Slabplanet"
command:
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation"

- label: ":envelope: Slack report: target AMIP"
command:
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun"
- slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun"

- label: ":envelope: Slack report: Flame Diff"
command:
Expand Down
2 changes: 0 additions & 2 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
agents:
queue: central
slurm_mem: 8G
slurm_time: 24:00:00
modules: julia/1.9.3 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1

Expand All @@ -24,7 +23,6 @@ steps:
- label: "init environment :computer:"
key: "init_cpu_env"
command:
# - "echo $$JULIA_DEPOT_PATH"

- echo "--- Configure MPI"
- julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()'
Expand Down
5 changes: 4 additions & 1 deletion Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ ClimaCoreTempestRemap = "d934ef94-cdd4-4710-83d6-720549644b70"
ClimaLSM = "7884a58f-fab6-4fd0-82bb-ecfedb2d8430"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
Insolation = "e98cc03f-d57e-4e3c-b70c-8d51efe9e0d8"
JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab"
OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed"
Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462"
SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7"
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
SurfaceFluxes = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f"
Expand All @@ -28,7 +30,7 @@ UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"

[compat]
CLIMAParameters = "0.4, 0.5, 0.6, 0.7"
ClimaAtmos = "0.16"
ClimaAtmos = "=0.16.2"
ClimaComms = "0.5"
ClimaCore = "0.10"
ClimaCoreTempestRemap = "0.3"
Expand All @@ -40,6 +42,7 @@ NCDatasets = "0.11, 0.12"
OrdinaryDiffEq = "5, 6"
Plots = "1.39.0"
SciMLBase = "1"
SimpleNonlinearSolve = "=0.1.23"
StaticArrays = "1"
SurfaceFluxes = "0.7"
TempestRemap_jll = "2"
Expand Down
22 changes: 0 additions & 22 deletions config/longrun_configs/amip_longrun_target.yml

This file was deleted.

18 changes: 18 additions & 0 deletions config/longrun_configs/amip_target.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
run_name: "amip_target"
anim: true
dt_cpl: 150
energy_check: false
mode_name: "amip"
mono_surface: false
dt: "150secs"
t_end: "300days"
job_id: "amip_target"
dt_save_to_sol: "5days"
dt_save_to_disk: "5days"
apply_limiter: false
hourly_checkpoint: true
turb_flux_partition: "CombinedStateFluxes"
atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml"
surface_setup: "PrescribedSurface"
land_albedo_type: "map_temporal"
start_date: "19790301"
21 changes: 21 additions & 0 deletions config/longrun_configs/amip_target_topo.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
run_name: "amip_target_topo"
anim: true
dt_cpl: 75
energy_check: false
mode_name: "amip"
mono_surface: false
dt: "150secs"
t_end: "100days"
job_id: "amip_target_topo"
dt_save_to_sol: "5days"
dt_save_to_disk: "5days"
apply_limiter: false
hourly_checkpoint: true
turb_flux_partition: "CombinedStateFluxes"
atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml"
surface_setup: "PrescribedSurface"
land_albedo_type: "map_temporal"
start_date: "19790301"
topography: "Earth"
topo_smoothing: true
use_reference_state: false
5 changes: 3 additions & 2 deletions config/longrun_configs/dyamond_target.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,6 @@ dt_save_to_sol: "0.5days"
dt_save_to_disk: "0.5days"
turb_flux_partition: "CombinedStateFluxes"
atmos_config_file: "config/longrun_configs/longrun_aquaplanet_dyamond.yml"
atmos_toml_file: "toml/longrun_aquaplanet_dyamond.toml"
hourly_checkpoint: false
monthly_checkpoint: false
land_albedo_type: "map_temporal"
start_date: "19790301"
21 changes: 21 additions & 0 deletions config/longrun_configs/slabplanet_aqua_atmos_sf_couple.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
run_name: "slabplanet_aqua_atmos_sf_couple"
job_id: "slabplanet_aqua_atmos_sf_couple"
moist: "equil"
vert_diff: "true"
rad: "gray"
precip_model: "0M"
energy_check: true
mode_name: "slabplanet_aqua"
t_end: "20days"
dt_save_to_sol: "10days"
dt_cpl: 200
dt: "200secs"
mono_surface: true
h_elem: 6
anim: true
apply_limiter: false
turb_flux_partition: "CombinedStateFluxes"
surface_setup: "DefaultMoninObukhov"
land_albedo_type: "function"
evolving_ocean: false
start_date: "19790301"
Loading

0 comments on commit 8dfbaaa

Please sign in to comment.