From 7e88da0e6b9c0a838c86bbd824499ef4cfa629a3 Mon Sep 17 00:00:00 2001 From: LenkaNovak Date: Mon, 6 Nov 2023 11:28:50 -0800 Subject: [PATCH] set up surface hooks add bk pipelines clean fix long pip clean interactive setup temp manifest check in try no pkg precomp revert manifest revert gitignore spec atmos version down SimpleNonlinearSolve fix path init pkg env down OpenMPI_jll down OpenMPI_jll HDF5_jll 1.12.2 try aqua test spec tst rev tst rev tst rev tst backslash rev tst backslash rm bs rm bs rm us rm us dbl { artifacts ca new t new t short path bs no us dot revs rm aug rm old tomls clean yml new_target to target build_history test revert test revert HTTP shorter long runs --- .buildkite/longruns/pipeline.yml | 194 ++++++++++++++++-- .buildkite/pipeline.yml | 2 - Project.toml | 4 +- .../longrun_configs/amip_longrun_target.yml | 22 -- config/longrun_configs/amip_target.yml | 18 ++ config/longrun_configs/amip_target_topo.yml | 21 ++ config/longrun_configs/dyamond_target.yml | 5 +- .../slabplanet_aqua_atmos_sf_couple.yml | 21 ++ .../slabplanet_aqua_atmos_sf_nocouple.yml | 21 ++ .../slabplanet_aqua_coupler_sf.yml | 21 ++ .../slabplanet_aqua_coupler_sf_evolve_ocn.yml | 21 ++ .../slabplanet_aqua_target.yml | 19 ++ .../slabplanet_aqua_target_evolve_ocn.yml | 19 ++ .../slabplanet_aqua_target_nocouple.yml | 19 ++ .../slabplanet_coupler_sf_evolve_ocn.yml | 21 ++ .../slabplanet_default_longrun.yml | 16 -- config/longrun_configs/slabplanet_target.yml | 19 ++ .../slabplanet_target_evolve_ocn.yml | 19 ++ config/longrun_configs/slabplanet_terra.yml | 20 ++ config/model_configs/interactive_debug.yml | 26 ++- config/model_configs/slabplanet_eisenman.yml | 2 +- experiments/AMIP/modular/cli_options.jl | 14 +- .../components/atmosphere/climaatmos_init.jl | 50 +++-- .../modular/components/land/bucket_init.jl | 32 +-- .../components/ocean/prescr_seaice_init.jl | 4 +- .../components/ocean/slab_ocean_init.jl | 25 ++- .../AMIP/modular/coupler_driver_modular.jl | 87 ++++++-- .../AMIP/modular/user_io/user_diagnostics.jl | 8 +- target/atmos_driver.jl | 33 +++ ...uilmoist_nz63_0M_55km_rs35km_clearsky.toml | 10 + ...3_0M_55km_rs35km_clearsky_tvinsolation.yml | 21 ++ test/Project.toml | 4 + toml/default_coarse.toml | 4 + 33 files changed, 682 insertions(+), 140 deletions(-) delete mode 100644 config/longrun_configs/amip_longrun_target.yml create mode 100644 config/longrun_configs/amip_target.yml create mode 100644 config/longrun_configs/amip_target_topo.yml create mode 100644 config/longrun_configs/slabplanet_aqua_atmos_sf_couple.yml create mode 100644 config/longrun_configs/slabplanet_aqua_atmos_sf_nocouple.yml create mode 100644 config/longrun_configs/slabplanet_aqua_coupler_sf.yml create mode 100644 config/longrun_configs/slabplanet_aqua_coupler_sf_evolve_ocn.yml create mode 100644 config/longrun_configs/slabplanet_aqua_target.yml create mode 100644 config/longrun_configs/slabplanet_aqua_target_evolve_ocn.yml create mode 100644 config/longrun_configs/slabplanet_aqua_target_nocouple.yml create mode 100644 config/longrun_configs/slabplanet_coupler_sf_evolve_ocn.yml delete mode 100644 config/longrun_configs/slabplanet_default_longrun.yml create mode 100644 config/longrun_configs/slabplanet_target.yml create mode 100644 config/longrun_configs/slabplanet_target_evolve_ocn.yml create mode 100644 config/longrun_configs/slabplanet_terra.yml create mode 100644 target/atmos_driver.jl create mode 100644 target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky.toml create mode 100644 target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml create mode 100644 toml/default_coarse.toml diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml index 8d765fe5d..df6fdb2a6 100644 --- a/.buildkite/longruns/pipeline.yml +++ b/.buildkite/longruns/pipeline.yml @@ -16,7 +16,6 @@ env: LD_LIBRARY_PATH: "/central/scratch/esm/slurm-buildkite/climaatmos-ci/depot/cpu/artifacts/e9feeabfa989549ac1dac61e4df5f2849149ad6d/lib/:${LD_LIBRARY_PATH}" - timeout_in_minutes: 1440 steps: @@ -51,14 +50,15 @@ steps: - wait - - group: "Targeted resolution coupled AMIP long runs" + - group: "Coupler integration and conservation tests" steps: - - label: "Slabplanet: default" - key: "slabplanet_default_longrun" - command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_default_longrun.yml" - artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/*" + # Integration tests - the expected results were also confirmed locally + - label: "Slabplanet_aqua: nocouple" + key: "slabplanet_aqua_atmos_sf_nocouple" # SF at each Atmos stage, no coupling, prescribed SST from atmos + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_nocouple.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_nocouple_artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -66,12 +66,93 @@ steps: slurm_nodes: 1 slurm_mem_per_cpu: 16G - # DYAMOND AMIP: 1 day (convection resolving) + - label: "Slabplanet_aqua: couple" + key: "slabplanet_aqua_atmos_sf_couple" # SF at each Atmos stage, coupling, prescribed SST from coupler - identical results to the above confirm 1) initial conditions in Atmos are unchanged compared to the slab, 2) coupling not introducing variability when constant surface + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_atmos_sf_couple.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_atmos_sf_couple_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G - - label: "MPI AMIP SUPERFINE: dyamond_target" - key: "dyamond_target" - command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/dyamond_target.yml" - artifact_paths: "experiments/AMIP/modular/output/amip/dyamond_target_artifacts/*" + - label: "Slabplanet_aqua: coupler fluxes" + key: "slabplanet_aqua_coupler_sf" # SF at each coupler timestep, constant ocean - comparing to the above runs, this tests the sensitivity of less frequent flux calculation + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet_aqua: coupler fluxes, evolving ocean" + key: "slabplanet_aqua_coupler_sf_evolve_ocn" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving ocean + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_coupler_sf_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_coupler_sf_evolve_ocn_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet_terra: coupler fluxes, evolving bucket" + key: "slabplanet_terra" # SF at each coupler timestep, evolving ocean - comparing to the above run, tests the sensitivity of evolving bucket + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_terra.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_terra/slabplanet_terra_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - label: "Slabplanet: coupler fluxes, evolving ocean and land" + key: "slabplanet_coupler_sf_evolve_ocn" + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_coupler_sf_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/*" + env: + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 1 + slurm_nodes: 1 + slurm_mem_per_cpu: 16G + + - group: "Current target tests: idealized surface" + + steps: + + - label: "ClimaAtmos standalone target" + command: + - srun julia --project=experiments/AMIP/modular/ target/atmos_driver.jl --config_file target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml + artifact_paths: "longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation/*" + env: + BUILD_HISTORY_HANDLE: "" + CLIMACORE_DISTRIBUTED: "MPI" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, nocouple, atmos flux calc" + key: "slabplanet_aqua_target_nocouple" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_nocouple.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_target_nocouple_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target aqua - fixed ocean T, coupler flux calc" + key: "slabplanet_aqua_target" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_target.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_target_artifacts/*" env: CLIMACORE_DISTRIBUTED: "MPI" BUILD_HISTORY_HANDLE: "" @@ -80,11 +161,82 @@ steps: slurm_nodes: 4 slurm_mem_per_cpu: 16G - # mid-resolution AMIP: longrun (140 days) - - label: "MPI AMIP FINE: target longrun" - key: "amip_longrun_target" - command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_longrun_target.yml" - artifact_paths: "experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/*" + - label: "TARGET IDEALIZED: new target aqua - evolving slab ocean T" + key: "slabplanet_aqua_target_evolve_ocn" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_aqua_target_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet_aqua/slabplanet_aqua_target_evolve_ocn_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target slab - fixed ocean T, bucket" + key: "slabplanet_target" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_target.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_target_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "TARGET IDEALIZED: new target slab - evolving slab ocean T, bucket" + key: "slabplanet_target_evolve_ocn" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/slabplanet_target_evolve_ocn.yml" + artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_target_evolve_ocn_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + + - group: "Current target tests: AMIP surface" + + steps: + + - label: "MPI AMIP FINE: new target amip" + key: "amip_target" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_target.yml" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_target_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + - label: "MPI AMIP FINE: new target amip: topo" + key: "amip_target_topo" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/amip_target_topo.yml" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_target_topo_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks_per_node: 16 + slurm_nodes: 4 + slurm_mem_per_cpu: 16G + + + - group: "Other AMIP targets" + + steps: + + # DYAMOND AMIP: 1 day (convection resolving) + + - label: "MPI AMIP SUPERFINE: dyamond_target" + key: "dyamond_target" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --config_file $CONFIG_PATH/dyamond_target.yml" + artifact_paths: "experiments/AMIP/modular/output/amip/dyamond_target_artifacts/*" env: CLIMACORE_DISTRIBUTED: "MPI" BUILD_HISTORY_HANDLE: "" @@ -190,14 +342,14 @@ steps: - label: ":envelope: Slack report: Slabplanet" command: - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)" - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)" - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_log_bucket.png -m png -n slab_coarse_w_log -x "Slabplanet water conservation (log error)" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_coupler_sf_evolve_ocn_artifacts/total_water_bucket.png -m png -n slab_coarse_w -x "Slabplanet water conservation" - label: ":envelope: Slack report: target AMIP" command: - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Target Longrun" - label: ":envelope: Slack report: Flame Diff" command: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f650e7d07..86201f421 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,6 +1,5 @@ agents: queue: central - slurm_mem: 8G slurm_time: 24:00:00 modules: julia/1.9.3 cuda/12.2 ucx/1.14.1_cuda-12.2 openmpi/4.1.5_cuda-12.2 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1 @@ -24,7 +23,6 @@ steps: - label: "init environment :computer:" key: "init_cpu_env" command: - # - "echo $$JULIA_DEPOT_PATH" - echo "--- Configure MPI" - julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()' diff --git a/Project.toml b/Project.toml index 761dcec11..64c62ec56 100644 --- a/Project.toml +++ b/Project.toml @@ -18,6 +18,7 @@ NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" +SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" SurfaceFluxes = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f" @@ -28,7 +29,7 @@ UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" [compat] CLIMAParameters = "0.4, 0.5, 0.6, 0.7" -ClimaAtmos = "0.16" +ClimaAtmos = "=0.16.2" ClimaComms = "0.5" ClimaCore = "0.10" ClimaCoreTempestRemap = "0.3" @@ -40,6 +41,7 @@ NCDatasets = "0.11, 0.12" OrdinaryDiffEq = "5, 6" Plots = "1.39.0" SciMLBase = "1" +SimpleNonlinearSolve = "=0.1.23" StaticArrays = "1" SurfaceFluxes = "0.7" TempestRemap_jll = "2" diff --git a/config/longrun_configs/amip_longrun_target.yml b/config/longrun_configs/amip_longrun_target.yml deleted file mode 100644 index 1b2ff5114..000000000 --- a/config/longrun_configs/amip_longrun_target.yml +++ /dev/null @@ -1,22 +0,0 @@ -run_name: "amip_longrun_target" -anim: true -dt_cpl: 150 -energy_check: false -mode_name: "amip" -mono_surface: false -vert_diff: "true" -moist: "equil" -rad: "clearsky" -precip_model: "0M" -z_elem: 35 -dz_bottom: 50 -h_elem: 12 -kappa_4: 3e16 -rayleigh_sponge: true -alpha_rayleigh_uh: 0 -dt: "100secs" -t_end: "100days" # TODO this has been decreased from 140 days to avoid instability #460 -job_id: "amip_longrun_target" -dt_save_to_sol: "5days" -dt_save_to_disk: "1days" -apply_limiter: false diff --git a/config/longrun_configs/amip_target.yml b/config/longrun_configs/amip_target.yml new file mode 100644 index 000000000..a7e80a2b0 --- /dev/null +++ b/config/longrun_configs/amip_target.yml @@ -0,0 +1,18 @@ +run_name: "amip_target" +anim: true +dt_cpl: 150 +energy_check: false +mode_name: "amip" +mono_surface: false +dt: "150secs" +t_end: "200days" +job_id: "amip_target" +dt_save_to_sol: "10days" +dt_save_to_disk: "20days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "map_temporal" +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/amip_target_topo.yml b/config/longrun_configs/amip_target_topo.yml new file mode 100644 index 000000000..62631bc84 --- /dev/null +++ b/config/longrun_configs/amip_target_topo.yml @@ -0,0 +1,21 @@ +run_name: "amip_target_topo" +anim: true +dt_cpl: 75 +energy_check: false +mode_name: "amip" +mono_surface: false +dt: "150secs" +t_end: "100days" +job_id: "amip_target_topo" +dt_save_to_sol: "10days" +dt_save_to_disk: "20days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "map_temporal" +start_date: "19790301" +topography: "Earth" +topo_smoothing: true +use_reference_state: false \ No newline at end of file diff --git a/config/longrun_configs/dyamond_target.yml b/config/longrun_configs/dyamond_target.yml index fbae6b5f3..291b813b6 100644 --- a/config/longrun_configs/dyamond_target.yml +++ b/config/longrun_configs/dyamond_target.yml @@ -10,5 +10,6 @@ dt_save_to_sol: "0.5days" dt_save_to_disk: "0.5days" turb_flux_partition: "CombinedStateFluxes" atmos_config_file: "config/longrun_configs/longrun_aquaplanet_dyamond.yml" -atmos_toml_file: "toml/longrun_aquaplanet_dyamond.toml" -hourly_checkpoint: false +monthly_checkpoint: false +land_albedo_type: "map_temporal" +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_aqua_atmos_sf_couple.yml b/config/longrun_configs/slabplanet_aqua_atmos_sf_couple.yml new file mode 100644 index 000000000..89e592ccd --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_atmos_sf_couple.yml @@ -0,0 +1,21 @@ +run_name: "slabplanet_aqua_atmos_sf_couple" +job_id: "slabplanet_aqua_atmos_sf_couple" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet_aqua" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" +mono_surface: true +h_elem: 6 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "DefaultMoninObukhov" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_aqua_atmos_sf_nocouple.yml b/config/longrun_configs/slabplanet_aqua_atmos_sf_nocouple.yml new file mode 100644 index 000000000..e7420b0c2 --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_atmos_sf_nocouple.yml @@ -0,0 +1,21 @@ +run_name: "slabplanet_aqua_atmos_sf_nocouple" +job_id: "slabplanet_aqua_atmos_sf_nocouple" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet_aqua" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 1728000 +dt: "200secs" +mono_surface: true +h_elem: 6 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "DefaultMoninObukhov" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_aqua_coupler_sf.yml b/config/longrun_configs/slabplanet_aqua_coupler_sf.yml new file mode 100644 index 000000000..d02caca22 --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_coupler_sf.yml @@ -0,0 +1,21 @@ +run_name: "slabplanet_aqua_coupler_sf" +job_id: "slabplanet_aqua_coupler_sf" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet_aqua" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" +mono_surface: true +h_elem: 6 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_aqua_coupler_sf_evolve_ocn.yml b/config/longrun_configs/slabplanet_aqua_coupler_sf_evolve_ocn.yml new file mode 100644 index 000000000..1a3689004 --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_coupler_sf_evolve_ocn.yml @@ -0,0 +1,21 @@ +run_name: "slabplanet_aqua_coupler_sf_evolve_ocn" +job_id: "slabplanet_aqua_coupler_sf_evolve_ocn" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet_aqua" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" +mono_surface: true +h_elem: 6 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: true +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_aqua_target.yml b/config/longrun_configs/slabplanet_aqua_target.yml new file mode 100644 index 000000000..2b65cd9ab --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_target.yml @@ -0,0 +1,19 @@ +run_name: "slabplanet_aqua_target" +anim: true +dt_cpl: 150 +energy_check: false +mode_name: "slabplanet_aqua" +mono_surface: false +dt: "150secs" +t_end: "200days" +job_id: "slabplanet_aqua_target" +dt_save_to_sol: "10days" +dt_save_to_disk: "20days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" diff --git a/config/longrun_configs/slabplanet_aqua_target_evolve_ocn.yml b/config/longrun_configs/slabplanet_aqua_target_evolve_ocn.yml new file mode 100644 index 000000000..56da68807 --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_target_evolve_ocn.yml @@ -0,0 +1,19 @@ +run_name: "slabplanet_aqua_target_evolve_ocn" +anim: true +dt_cpl: 150 +energy_check: false +mode_name: "slabplanet_aqua" +mono_surface: false +dt: "150secs" +t_end: "200days" +job_id: "slabplanet_target_evolve_ocn" +dt_save_to_sol: "10days" +dt_save_to_disk: "20days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: true +start_date: "19790301" diff --git a/config/longrun_configs/slabplanet_aqua_target_nocouple.yml b/config/longrun_configs/slabplanet_aqua_target_nocouple.yml new file mode 100644 index 000000000..70b04f577 --- /dev/null +++ b/config/longrun_configs/slabplanet_aqua_target_nocouple.yml @@ -0,0 +1,19 @@ +run_name: "slabplanet_aqua_target_nocouple" +anim: true +dt_cpl: 8639800 +energy_check: false +mode_name: "slabplanet_aqua" +mono_surface: false +dt: "150secs" +t_end: "200days" +job_id: "slabplanet_aqua_target_nocouple" +dt_save_to_sol: "10days" +dt_save_to_disk: "20days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "DefaultMoninObukhov" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" diff --git a/config/longrun_configs/slabplanet_coupler_sf_evolve_ocn.yml b/config/longrun_configs/slabplanet_coupler_sf_evolve_ocn.yml new file mode 100644 index 000000000..2ff58dddf --- /dev/null +++ b/config/longrun_configs/slabplanet_coupler_sf_evolve_ocn.yml @@ -0,0 +1,21 @@ +run_name: "slabplanet_coupler_sf_evolve_ocn" +job_id: "slabplanet_coupler_sf_evolve_ocn" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" +mono_surface: true +h_elem: 6 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: true +start_date: "19790301" \ No newline at end of file diff --git a/config/longrun_configs/slabplanet_default_longrun.yml b/config/longrun_configs/slabplanet_default_longrun.yml deleted file mode 100644 index e635c8b44..000000000 --- a/config/longrun_configs/slabplanet_default_longrun.yml +++ /dev/null @@ -1,16 +0,0 @@ -run_name: "slabplanet_default_longrun" -moist: "equil" -vert_diff: "true" -rad: "gray" -energy_check: true -mode_name: "slabplanet" -t_end: "60days" -dt_save_to_sol: "10days" -dt_cpl: 200 -dt: "200secs" -mono_surface: true -h_elem: 6 -precip_model: "0M" -anim: true -apply_limiter: false -job_id: "slabplanet_default_longrun" diff --git a/config/longrun_configs/slabplanet_target.yml b/config/longrun_configs/slabplanet_target.yml new file mode 100644 index 000000000..251a0de5a --- /dev/null +++ b/config/longrun_configs/slabplanet_target.yml @@ -0,0 +1,19 @@ +run_name: "slabplanet_target" +anim: true +dt_cpl: 150 +energy_check: false +mode_name: "slabplanet" +mono_surface: false +dt: "150secs" +t_end: "200days" +job_id: "slabplanet_target" +dt_save_to_sol: "10days" +dt_save_to_disk: "20days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: false +start_date: "19790301" diff --git a/config/longrun_configs/slabplanet_target_evolve_ocn.yml b/config/longrun_configs/slabplanet_target_evolve_ocn.yml new file mode 100644 index 000000000..66c154ecb --- /dev/null +++ b/config/longrun_configs/slabplanet_target_evolve_ocn.yml @@ -0,0 +1,19 @@ +run_name: "slabplanet_target_evolve_ocn" +anim: true +dt_cpl: 150 +energy_check: false +mode_name: "slabplanet" +mono_surface: false +dt: "150secs" +t_end: "200days" +job_id: "slabplanet_target_evolve_ocn" +dt_save_to_sol: "10days" +dt_save_to_disk: "20days" +apply_limiter: false +hourly_checkpoint: true +turb_flux_partition: "CombinedStateFluxes" +atmos_config_file: "config/longrun_configs/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: true +start_date: "19790301" diff --git a/config/longrun_configs/slabplanet_terra.yml b/config/longrun_configs/slabplanet_terra.yml new file mode 100644 index 000000000..affb7b48e --- /dev/null +++ b/config/longrun_configs/slabplanet_terra.yml @@ -0,0 +1,20 @@ +run_name: "slabplanet_terra" +job_id: "slabplanet_terra" +moist: "equil" +vert_diff: "true" +rad: "gray" +precip_model: "0M" +energy_check: true +mode_name: "slabplanet_terra" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" +mono_surface: true +h_elem: 4 +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +start_date: "19790301" \ No newline at end of file diff --git a/config/model_configs/interactive_debug.yml b/config/model_configs/interactive_debug.yml index a5ec7c15d..e817a5f20 100644 --- a/config/model_configs/interactive_debug.yml +++ b/config/model_configs/interactive_debug.yml @@ -1,17 +1,21 @@ +run_name: "interactive_debug_run" +job_id: "interactive_debug_run" moist: "equil" vert_diff: "true" -rad: "allskywithclear" +rad: "gray" +precip_model: "0M" energy_check: true mode_name: "slabplanet" -t_end: "10days" -dt_save_to_sol: "0.5days" -dt_cpl: 400 -dt: "400secs" +t_end: "20days" +dt_save_to_sol: "10days" +dt_cpl: 200 +dt: "200secs" mono_surface: true -turb_flux_partition: "CombinedStateFluxes" h_elem: 4 -dt_save_restart: "5days" -precip_model: "0M" -run_name: "interactive_debug_run" -job_id: "interactive_debug_run" -hourly_checkpoint: true \ No newline at end of file +anim: true +apply_limiter: false +turb_flux_partition: "CombinedStateFluxes" +surface_setup: "PrescribedSurface" +land_albedo_type: "function" +evolving_ocean: true +start_date: "19790301" \ No newline at end of file diff --git a/config/model_configs/slabplanet_eisenman.yml b/config/model_configs/slabplanet_eisenman.yml index a1322309b..b93b7d9e9 100644 --- a/config/model_configs/slabplanet_eisenman.yml +++ b/config/model_configs/slabplanet_eisenman.yml @@ -5,7 +5,7 @@ vert_diff: "true" rad: "gray" energy_check: true mode_name: "slabplanet_eisenman" -t_end: "10days" +t_end: "2days" dt_save_to_sol: "9days" dt_cpl: 200 dt: "200secs" diff --git a/experiments/AMIP/modular/cli_options.jl b/experiments/AMIP/modular/cli_options.jl index 335b5f1c2..9b05b862b 100644 --- a/experiments/AMIP/modular/cli_options.jl +++ b/experiments/AMIP/modular/cli_options.jl @@ -19,7 +19,7 @@ function argparse_settings() arg_type = Bool default = false "--mode_name" - help = "Mode of coupled simulation. [`amip`, `slabplanet`]" + help = "Mode of coupled simulation. [`amip`, `slabplanet`, `slabplanet_aqua`, `slabplanet_terra`, `slabplanet_eisenman`]" arg_type = String default = "amip" "--mono_surface" @@ -52,6 +52,12 @@ function argparse_settings() help = "Floating point precision [`Float64` (default), `Float32`]" arg_type = String default = "Float64" + "--coupler_toml_file" + help = "A toml file used to overwrite the model parameters. If nothing is specified, the default parameters are used." + "--evolving_ocean" + help = "Boolean flag indicating whether to use a dynamic slab ocean model or constant surface temperatures" + arg_type = Bool + default = true # ClimaAtmos specific "--surface_setup" help = "Triggers ClimaAtmos into the coupled mode [`PrescribedSurface` (default)]" # retained here for standalone Atmos benchmarks @@ -59,8 +65,6 @@ function argparse_settings() default = "PrescribedSurface" "--atmos_config_file" help = "A yaml file used to set the atmospheric model configuration. If nothing is specified, the default configuration is used." - "--atmos_toml_file" - help = "A toml file used to overwrite the atmospheric model parameters. If nothing is specified, the default parameters are used." # ClimaLSM specific "--land_albedo_type" help = "Access land surface albedo information from data file. [`function`, `map_static`, `map_temporal`]" @@ -70,6 +74,10 @@ function argparse_settings() help = "Type of land domain. [`sphere` (default), `single_column`]" arg_type = String default = "sphere" + "--land_temperature_anomaly" + help = "Type of temperature anomaly for bucket model. [`zonally_asymmetric`, `amip`, `aquaplanet` (default)]" + arg_type = String + default = "aquaplanet" end return s end diff --git a/experiments/AMIP/modular/components/atmosphere/climaatmos_init.jl b/experiments/AMIP/modular/components/atmosphere/climaatmos_init.jl index fa507fe71..2bbe1b204 100644 --- a/experiments/AMIP/modular/components/atmosphere/climaatmos_init.jl +++ b/experiments/AMIP/modular/components/atmosphere/climaatmos_init.jl @@ -1,6 +1,8 @@ # atmos_init: for ClimaAtmos pre-AMIP interface -using ClimaAtmos: RRTMGPI +import ClimaAtmos as CA import ClimaAtmos: CT1, CT2, CT12, CT3, C3, C12, unit_basis_vector_data, ⊗ +import SurfaceFluxes as SF + import ClimaCoupler.FluxCalculator: atmos_turbulent_fluxes!, calculate_surface_air_density, @@ -41,10 +43,19 @@ function get_atmos_config(coupler_dict) Dict("config_file" => atmos_config_file), ) end - atmos_toml_file = coupler_dict["atmos_toml_file"] - if !isnothing(atmos_toml_file) - @info "Overwriting Atmos parameters from $atmos_toml_file" - atmos_config = merge(atmos_config, Dict("toml" => [joinpath(pkgdir(CA), atmos_toml_file)])) + + # use coupler toml if atmos is not defined + atmos_toml_file = atmos_config["toml"] + coupler_toml_file = coupler_dict["coupler_toml_file"] + default_toml_file = "toml/default_coarse.toml" + + toml_file = !isempty(atmos_toml_file) ? joinpath(pkgdir(CA), atmos_toml_file[1]) : nothing + toml_file = !isnothing(coupler_toml_file) ? joinpath(pkgdir(ClimaCoupler), coupler_toml_file) : toml_file + toml_file = isnothing(toml_file) ? joinpath(pkgdir(ClimaCoupler), default_toml_file) : toml_file + + if !isnothing(toml_file) + @info "Overwriting Atmos parameters from $toml_file" + atmos_config = merge(atmos_config, Dict("toml" => [toml_file])) end return atmos_config end @@ -123,14 +134,14 @@ function update_field!(atmos_sim::ClimaAtmosSimulation, ::Val{:co2_gm}, field) end # extensions required by the Interfacer function update_field!(sim::ClimaAtmosSimulation, ::Val{:surface_temperature}, csf) - sim.integrator.p.radiation_model.surface_temperature .= RRTMGPI.field2array(csf.T_S) + sim.integrator.p.radiation_model.surface_temperature .= CA.RRTMGPI.field2array(csf.T_S) end function update_field!(sim::ClimaAtmosSimulation, ::Val{:albedo}, field) sim.integrator.p.radiation_model.diffuse_sw_surface_albedo .= - reshape(RRTMGPI.field2array(field), 1, length(parent(field))) + reshape(CA.RRTMGPI.field2array(field), 1, length(parent(field))) sim.integrator.p.radiation_model.direct_sw_surface_albedo .= - reshape(RRTMGPI.field2array(field), 1, length(parent(field))) + reshape(CA.RRTMGPI.field2array(field), 1, length(parent(field))) end # get_surface_params required by FluxCalculator (partitioned fluxes) @@ -241,11 +252,20 @@ requires that we define a new temporary parameter Tuple, `new_p`, and save the n in it. We do not want `new_p` to live in the atmospheric model permanently, because that would also trigger flux calculation during Atmos `step!`. We only want to trigger this once per coupling timestep from ClimaCoupler. + +For debigging atmos, we can set the following atmos defaults: + csf.z0m_S .= 1.0e-5 + csf.z0b_S .= 1.0e-5 + csf.beta .= 1 + csf = merge(csf, (;q_sfc = nothing)) """ function atmos_turbulent_fluxes!(atmos_sim::ClimaAtmosSimulation, csf) - new_p = get_new_cache(atmos_sim, csf) - CA.SurfaceConditions.update_surface_conditions!(atmos_sim.integrator.u, new_p, atmos_sim.integrator.t) - atmos_sim.integrator.p.sfc_conditions .= new_p.sfc_conditions + + if isnothing(atmos_sim.integrator.p.sfc_setup) # trigger flux calculation if not done in Atmos internally + new_p = get_new_cache(atmos_sim, csf) + CA.SurfaceConditions.update_surface_conditions!(atmos_sim.integrator.u, new_p, atmos_sim.integrator.t) + atmos_sim.integrator.p.sfc_conditions .= new_p.sfc_conditions + end end """ @@ -292,19 +312,19 @@ function get_field(atmos_sim::ClimaAtmosSimulation, ::Val{:F_radiative_TOA}) n_faces = length(z[:, 1, 1, 1, 1]) LWd_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_dn), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_dn), face_space), n_faces - half, ) LWu_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_up), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_up), face_space), n_faces - half, ) SWd_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_dn), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_dn), face_space), n_faces - half, ) SWu_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_up), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_up), face_space), n_faces - half, ) diff --git a/experiments/AMIP/modular/components/land/bucket_init.jl b/experiments/AMIP/modular/components/land/bucket_init.jl index 654827737..2b103d984 100644 --- a/experiments/AMIP/modular/components/land/bucket_init.jl +++ b/experiments/AMIP/modular/components/land/bucket_init.jl @@ -166,6 +166,7 @@ function bucket_init( tspan::Tuple{FT, FT}, config::String, albedo_type::String, + land_temperature_anomaly::String, comms_ctx::AbstractCommsContext, regrid_dirpath::String; space, @@ -188,14 +189,22 @@ function bucket_init( α_snow = FT(0.8) # snow albedo if albedo_type == "map_static" # Read in albedo from static data file (default type) # By default, this uses a file containing bareground albedo without a time component. Snow albedo is specified separately. + if ClimaComms.iamroot(comms_ctx) + albedo = BulkAlbedoStatic{FT}(regrid_dirpath, α_snow = α_snow, comms = comms_ctx) + end + ClimaComms.barrier(comms_ctx) albedo = BulkAlbedoStatic{FT}(regrid_dirpath, α_snow = α_snow, comms = comms_ctx) elseif albedo_type == "map_temporal" # Read in albedo from data file containing data over time # By default, this uses a file containing linearly-interpolated monthly data of total albedo, generated by CESM2's land model (CLM). + if ClimaComms.iamroot(comms_ctx) + albedo = BulkAlbedoTemporal{FT}(regrid_dirpath, date_ref, t_start, space) + end + ClimaComms.barrier(comms_ctx) albedo = BulkAlbedoTemporal{FT}(regrid_dirpath, date_ref, t_start, space) elseif albedo_type == "function" # Use prescribed function of lat/lon for surface albedo function α_sfc(coordinate_point) (; lat, long) = coordinate_point - return typeof(lat)(0.4) + return typeof(lat)(0.38) end albedo = BulkAlbedoFunction{FT}(α_snow, α_sfc) else @@ -203,12 +212,12 @@ function bucket_init( end σS_c = FT(0.2) - W_f = FT(0.5) + W_f = FT(10) d_soil = FT(3.5) # soil depth z_0m = FT(1e-2) z_0b = FT(1e-3) κ_soil = FT(0.7) - ρc_soil = FT(2e6) + ρc_soil = FT(2e8) t_crit = dt # This is the timescale on which snow exponentially damps to zero, in the case where all # the snow would melt in time t_crit. It prevents us from having to specially time step in cases where # all the snow melts in a single timestep. @@ -222,29 +231,28 @@ function bucket_init( # Initial conditions with no moisture Y, p, coords = initialize(model) - anomaly = false - anomaly_tropics = false - hs_sfc = false Y.bucket.T = map(coords.subsurface) do coord - T_sfc_0 = FT(285.0) + T_sfc_0 = FT(271.0) radlat = coord.lat / FT(180) * pi ΔT = FT(0) - if anomaly == true + if land_temperature_anomaly == "zonally_asymmetric" anom_ampl = FT(0)# this is zero, no anomaly lat_0 = FT(60) / FT(180) * pi lon_0 = FT(-90) / FT(180) * pi radlon = coord.long / FT(180) * pi stdev = FT(5) / FT(180) * pi ΔT = anom_ampl * exp(-((radlat - lat_0)^2 / 2stdev^2 + (radlon - lon_0)^2 / 2stdev^2)) - elseif hs_sfc == true - ΔT = -FT(60) * sin(radlat)^2 - elseif anomaly_tropics == true + elseif land_temperature_anomaly == "aquaplanet" + ΔT = FT(29) * exp(-coord.lat^2 / (2 * 26^2)) + elseif land_temperature_anomaly == "amip" ΔT = FT(40 * cos(radlat)^4) + else + ΔT = FT(0) end T_sfc_0 + ΔT end - Y.bucket.W .= 0.5#0.14 + Y.bucket.W .= 10#0.14 Y.bucket.Ws .= 0.0 Y.bucket.σS .= 0.0 P_liq = zeros(axes(Y.bucket.W)) .+ FT(0.0) diff --git a/experiments/AMIP/modular/components/ocean/prescr_seaice_init.jl b/experiments/AMIP/modular/components/ocean/prescr_seaice_init.jl index 7f4c7a9d5..e0fb56f43 100644 --- a/experiments/AMIP/modular/components/ocean/prescr_seaice_init.jl +++ b/experiments/AMIP/modular/components/ocean/prescr_seaice_init.jl @@ -78,8 +78,8 @@ function ice_rhs!(du, u, p, _) rhs = @. (-F_turb_energy - F_radiative + F_conductive) / (params.h * params.ρ * params.c) # do not count tendencies that lead to temperatures above freezing, and mask out no-ice areas - unphysical = @. Regridder.binary_mask.(T_freeze - (Y.T_sfc + FT(rhs) * p.dt), threshold = FT(0)) .* - Regridder.binary_mask.(area_fraction, threshold = eps(FT)) + area_mask = Regridder.binary_mask.(area_fraction, threshold = eps(FT)) + unphysical = @. Regridder.binary_mask.(T_freeze - (Y.T_sfc + FT(rhs) * p.dt), threshold = FT(0)) .* area_mask parent(dY.T_sfc) .= parent(rhs .* unphysical) @. p.q_sfc = TD.q_vap_saturation_generic.(p.thermo_params, Y.T_sfc, p.ρ_sfc, TD.Ice()) diff --git a/experiments/AMIP/modular/components/ocean/slab_ocean_init.jl b/experiments/AMIP/modular/components/ocean/slab_ocean_init.jl index 42ee4fb34..44752c2be 100644 --- a/experiments/AMIP/modular/components/ocean/slab_ocean_init.jl +++ b/experiments/AMIP/modular/components/ocean/slab_ocean_init.jl @@ -32,6 +32,7 @@ struct OceanSlabParameters{FT <: AbstractFloat} z0m::FT z0b::FT α::FT + evolving_switch::FT end name(::SlabOceanSimulation) = "SlabOceanSimulation" @@ -45,7 +46,7 @@ function slab_ocean_space_init(::Type{FT}, space, p) where {FT} T_sfc_0 = FT(p.T_init) #- FT(275) # close to the average of T_1 in atmos anomaly = false - anomaly_tropics = false + anomaly_tropics = true anom = FT(0) radlat = coord.lat / FT(180) * pi if anomaly == true @@ -55,7 +56,7 @@ function slab_ocean_space_init(::Type{FT}, space, p) where {FT} stdev = FT(5) / FT(180) * pi anom = anom_ampl * exp(-((radlat - lat_0)^2 / 2stdev^2 + (radlon - lon_0)^2 / 2stdev^2)) elseif anomaly_tropics == true - anom = FT(20 * cos(radlat)^4) + anom = FT(29) * exp(-coord.lat^2 / (2 * 26^2)) end T_sfc = T_sfc_0 + anom end @@ -71,7 +72,7 @@ function slab_ocean_rhs!(dY, Y, cache, t) p, F_turb_energy, F_radiative, area_fraction = cache FT = eltype(Y.T_sfc) rhs = @. -(F_turb_energy + F_radiative) / (p.h * p.ρ * p.c) - parent(dY.T_sfc) .= parent(rhs .* Regridder.binary_mask.(area_fraction, threshold = eps(FT))) + parent(dY.T_sfc) .= parent(rhs .* Regridder.binary_mask.(area_fraction, threshold = eps(FT))) * p.evolving_switch @. cache.q_sfc = TD.q_vap_saturation_generic.(cache.thermo_params, Y.T_sfc, cache.ρ_sfc, TD.Liquid()) end @@ -81,9 +82,21 @@ end Initializes the `DiffEq` problem, and creates a Simulation-type object containing the necessary information for `step!` in the coupling loop. """ -function ocean_init(::Type{FT}; tspan, dt, saveat, space, area_fraction, thermo_params, stepper = CTS.RK4()) where {FT} - - params = OceanSlabParameters(FT(20), FT(1500.0), FT(800.0), FT(280.0), FT(1e-3), FT(1e-5), FT(0.06)) +function ocean_init( + ::Type{FT}; + tspan, + dt, + saveat, + space, + area_fraction, + thermo_params, + stepper = CTS.RK4(), + evolving = true, +) where {FT} + + evolving_switch = evolving ? FT(1) : FT(0) + params = + OceanSlabParameters(FT(20), FT(1500.0), FT(800.0), FT(271.0), FT(1e-5), FT(1e-5), FT(0.38), evolving_switch) Y, space = slab_ocean_space_init(FT, space, params) cache = ( diff --git a/experiments/AMIP/modular/coupler_driver_modular.jl b/experiments/AMIP/modular/coupler_driver_modular.jl index 43cb8d1de..06cd47eb0 100644 --- a/experiments/AMIP/modular/coupler_driver_modular.jl +++ b/experiments/AMIP/modular/coupler_driver_modular.jl @@ -167,6 +167,7 @@ mono_surface = config_dict["mono_surface"] hourly_checkpoint = config_dict["hourly_checkpoint"] restart_dir = config_dict["restart_dir"] restart_t = Int(config_dict["restart_t"]) +evolving_ocean = config_dict["evolving_ocean"] ## I/O directory setup if isinteractive() @@ -217,37 +218,38 @@ boundary_space = atmos_sim.domain.face_space.horizontal_space land_fraction = Regridder.land_fraction(FT, REGRID_DIR, comms_ctx, land_mask_data, "LSMASK", boundary_space, mono = mono_surface) -#= -### Land -We use `ClimaLSM.jl`'s bucket model. -=# -land_sim = bucket_init( - FT, - FT.(tspan), - config_dict["land_domain_type"], - config_dict["land_albedo_type"], - comms_ctx, - REGRID_DIR; - dt = FT(Δt_cpl), - space = boundary_space, - saveat = FT(saveat), - area_fraction = land_fraction, - date_ref = date0, - t_start = FT(0), -) - #= ### Ocean and Sea Ice In the `AMIP` mode, all ocean properties are prescribed from a file, while sea-ice temperatures are calculated using observed SIC and assuming a 2m thickness of the ice. In the `SlabPlanet` mode, all ocean and sea ice are dynamical models, namely thermal slabs, with different parameters. + +### Land +If evolving, use `ClimaLSM.jl`'s bucket model. =# @info mode_name if mode_name == "amip" @info "AMIP boundary conditions - do not expect energy conservation" + ## land + land_sim = bucket_init( + FT, + FT.(tspan), + config_dict["land_domain_type"], + config_dict["land_albedo_type"], + config_dict["land_temperature_anomaly"], + comms_ctx, + REGRID_DIR; + dt = FT(Δt_cpl), + space = boundary_space, + saveat = FT(saveat), + area_fraction = land_fraction, + date_ref = date0, + t_start = FT(0), + ) + ## ocean SST_info = bcfile_info_init( FT, @@ -276,6 +278,7 @@ if mode_name == "amip" phase = TD.Liquid(), thermo_params = thermo_params, )) + ## sea ice SIC_info = bcfile_info_init( FT, @@ -323,7 +326,28 @@ if mode_name == "amip" mode_specifics = (; name = mode_name, SST_info = SST_info, SIC_info = SIC_info, CO2_info = CO2_info) -elseif mode_name == "slabplanet" +elseif mode_name in ("slabplanet", "slabplanet_aqua", "slabplanet_terra") + + land_fraction = mode_name == "slabplanet_aqua" ? land_fraction .* 0 : land_fraction + land_fraction = mode_name == "slabplanet_terra" ? land_fraction .* 0 .+ 1 : land_fraction + + ## land + land_sim = bucket_init( + FT, + FT.(tspan), + config_dict["land_domain_type"], + config_dict["land_albedo_type"], + config_dict["land_temperature_anomaly"], + comms_ctx, + REGRID_DIR; + dt = FT(Δt_cpl), + space = boundary_space, + saveat = FT(saveat), + area_fraction = land_fraction, + date_ref = date0, + t_start = FT(0), + ) + ## ocean ocean_sim = ocean_init( FT; @@ -333,6 +357,7 @@ elseif mode_name == "slabplanet" saveat = saveat, area_fraction = (FT(1) .- land_fraction), ## NB: this ocean fraction includes areas covered by sea ice (unlike the one contained in the cs) thermo_params = thermo_params, + evolving = evolving_ocean, ) ## sea ice (here set to zero area coverage) @@ -351,6 +376,24 @@ elseif mode_name == "slabplanet" mode_specifics = (; name = mode_name, SST_info = nothing, SIC_info = nothing) elseif mode_name == "slabplanet_eisenman" + + ## land + land_sim = bucket_init( + FT, + FT.(tspan), + config_dict["land_domain_type"], + config_dict["land_albedo_type"], + config_dict["land_temperature_anomaly"], + comms_ctx, + REGRID_DIR; + dt = FT(Δt_cpl), + space = boundary_space, + saveat = FT(saveat), + area_fraction = land_fraction, + date_ref = date0, + t_start = FT(0), + ) + ## ocean ocean_sim = ocean_init( FT; @@ -449,7 +492,7 @@ end dir_paths = (; output = COUPLER_OUTPUT_DIR, artifacts = COUPLER_ARTIFACTS_DIR) checkpoint_cb = - HourlyCallback(dt = FT(48), func = checkpoint_sims, ref_date = [dates.date[1]], active = hourly_checkpoint) # bi-daily + HourlyCallback(dt = FT(480), func = checkpoint_sims, ref_date = [dates.date[1]], active = hourly_checkpoint) # 20 days update_firstdayofmonth!_cb = MonthlyCallback(dt = FT(1), func = update_firstdayofmonth!, ref_date = [dates.date1[1]], active = true) # for BCReader callbacks = (; checkpoint = checkpoint_cb, update_firstdayofmonth! = update_firstdayofmonth!_cb) @@ -550,7 +593,7 @@ function solve_coupler!(cs) ## print date on the first of month if cs.dates.date[1] >= cs.dates.date1[1] - @show(cs.dates.date[1]) + ClimaComms.iamroot(comms_ctx) ? @show(cs.dates.date[1]) : nothing end if cs.mode.name == "amip" diff --git a/experiments/AMIP/modular/user_io/user_diagnostics.jl b/experiments/AMIP/modular/user_io/user_diagnostics.jl index 42e28c4e6..f455538b5 100644 --- a/experiments/AMIP/modular/user_io/user_diagnostics.jl +++ b/experiments/AMIP/modular/user_io/user_diagnostics.jl @@ -59,19 +59,19 @@ function get_var(cs::CoupledSimulation, ::Val{:toa_fluxes}) n_faces = length(z[:, 1, 1, 1, 1]) LWd_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_dn), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_dn), face_space), n_faces - half, ) LWu_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_up), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_lw_flux_up), face_space), n_faces - half, ) SWd_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_dn), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_dn), face_space), n_faces - half, ) SWu_TOA = Fields.level( - RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_up), face_space), + CA.RRTMGPI.array2field(FT.(atmos_sim.integrator.p.radiation_model.face_sw_flux_up), face_space), n_faces - half, ) diff --git a/target/atmos_driver.jl b/target/atmos_driver.jl new file mode 100644 index 000000000..09eb21798 --- /dev/null +++ b/target/atmos_driver.jl @@ -0,0 +1,33 @@ +using ClimaComms +using Logging +using ClimaAtmos + +const is_distributed = get(ENV, "CLIMACORE_DISTRIBUTED", "") == "MPI" +if is_distributed + const comms_ctx = ClimaComms.context(ClimaComms.CPUSingleThreaded()) + const pid, nprocs = ClimaComms.init(comms_ctx) + + if ClimaComms.iamroot(comms_ctx) + Logging.global_logger(Logging.ConsoleLogger(stderr, Logging.Info)) + else + Logging.global_logger(Logging.NullLogger()) + end +else + using TerminalLoggers: TerminalLogger + const comms_ctx = ClimaComms.SingletonCommsContext() + const pid, nprocs = ClimaComms.init(comms_ctx) +end + + +ClimaComms.barrier(comms_ctx) + +# this tests the standalone runs, using the hybrid/driver +import Random +Random.seed!(1234) + +config = ClimaAtmos.AtmosConfig() +ClimaComms.barrier(comms_ctx) +# ClimaComms.iamroot(comms_ctx) ? @info(config) : nothing +integrator = ClimaAtmos.get_integrator(config) +ClimaComms.barrier(comms_ctx) +sol_res = ClimaAtmos.solve_atmos!(integrator) diff --git a/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky.toml b/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky.toml new file mode 100644 index 000000000..7469c2505 --- /dev/null +++ b/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky.toml @@ -0,0 +1,10 @@ +[alpha_rayleigh_w] +alias = "alpha_rayleigh_w" +value = 1.0 +type = "float" + +[zd_rayleigh] +alias = "zd_rayleigh" +value = 35000.0 +type = "float" + diff --git a/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml b/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml new file mode 100644 index 000000000..aa67c24ad --- /dev/null +++ b/target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation.yml @@ -0,0 +1,21 @@ +FLOAT_TYPE: "Float64" +dt_save_to_disk: "10days" +dt: "150secs" +t_end: "200days" +h_elem: 16 +z_elem: 63 +dz_bottom: 30.0 +dz_top: 3000.0 +z_max: 55000.0 +rayleigh_sponge: true +kappa_4: 2.0e16 +moist: "equil" +precip_model: "0M" +surface_setup: "DefaultMoninObukhov" +vert_diff: "true" +rad: "clearsky" +idealized_insolation: false +dt_rad: "1hours" +job_id: "longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky_tvinsolation" +toml: [target/longrun_aquaplanet_rhoe_equilmoist_nz63_0M_55km_rs35km_clearsky.toml] +start_date: "19790301" \ No newline at end of file diff --git a/test/Project.toml b/test/Project.toml index acd91714f..d228565cb 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -12,6 +12,7 @@ ClimaTimeSteppers = "595c0a79-7f3d-439a-bc5a-b232dc3bde79" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59" Insolation = "e98cc03f-d57e-4e3c-b70c-8d51efe9e0d8" IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" @@ -23,6 +24,7 @@ PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" +SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" SurfaceFluxes = "49b00bb7-8bd4-4f2b-b78c-51cd0450215f" @@ -34,11 +36,13 @@ Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" ArtifactWrappers = "0.2" ClimaCoupler = "0.1" Downloads = "1" +HDF5_jll = "=1.12.2" IntervalSets = "0.5, 0.6, 0.7" MPI = "0.20" NCDatasets = "0.11, 0.12" Pkg = "1" PrettyTables = "2" +SimpleNonlinearSolve = "=0.1.23" StaticArrays = "1" Unitful = "1" julia = "1.8" diff --git a/toml/default_coarse.toml b/toml/default_coarse.toml new file mode 100644 index 000000000..b88717390 --- /dev/null +++ b/toml/default_coarse.toml @@ -0,0 +1,4 @@ +[C_E] +alias = "C_E" +value = 0.044 +type = "float"