Try #1489:

CliMA · Oct 10, 2023 · 50309eb · 50309eb
2 parents 2706600 + 4fbf76f
commit 50309eb
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 28 deletions.
diff --git a/.buildkite/JuliaProject.toml b/.buildkite/JuliaProject.toml
@@ -4,7 +4,8 @@ HDF5_jll = "0234f1f7-429e-5d53-9886-15a909be8d59"
 MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 
 [preferences.CUDA_Runtime_jll]
-version = "local"
+version = "12.2"
+local = "true"
 
 [preferences.HDF5_jll]
 libhdf5_path = "libhdf5"

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -59,7 +59,7 @@ steps:
       - label: "Unit: cuda"
         key: unit_cuda_functional
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/gpu/cuda.jl CUDA"
         agents:
           slurm_gpus: 1
@@ -296,41 +296,41 @@ steps:
       - label: "Unit: cuda Cubed Sphere dss; ne = 32; 2-process test"
         key: "gpu_ddss_ne32_cs_2processes"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/distributed_cuda/ddss_ne32_cs.jl"
         timeout_in_minutes: 15
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           JULIA_CUDA_MEMORY_POOL: "none"
         agents:
           slurm_ntasks: 2
-          slurm_gpus: 2
+          slurm_gpus_per_task: 1
 
       - label: "Unit: cuda Cubed Sphere dss; ne = 32; 3-process test"
         key: "gpu_ddss_ne32_cs_3processes"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/distributed_cuda/ddss_ne32_cs.jl"
         timeout_in_minutes: 15
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           JULIA_CUDA_MEMORY_POOL: "none"
         agents:
           slurm_ntasks: 3
-          slurm_gpus: 3
+          slurm_gpus_per_task: 1
 
       - label: "Unit: cuda Cubed Sphere dss; ne = 32; 4-process test"
         key: "gpu_ddss_ne32_cs_4processes"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/distributed_cuda/ddss_ne32_cs.jl"
         timeout_in_minutes: 15
         env:
           CLIMACOMMS_CONTEXT: "MPI"
           JULIA_CUDA_MEMORY_POOL: "none"
         agents:
           slurm_ntasks: 4
-          slurm_gpus: 4
+          slurm_gpus_per_task: 1
 
   - group: "Unit: Fields"
     steps:
@@ -435,6 +435,12 @@ steps:
         key: unit_hyb_ops_3d
         command: "julia --color=yes --check-bounds=yes --project=test test/Operators/hybrid/3d.jl"
 
+      - label: "Unit: hyb ops 3d CUDA"
+        key: unit_hyb_ops_3d_cuda
+        command: "julia --color=yes --check-bounds=yes --project=test test/Operators/hybrid/3d.jl"
+        agents:
+          slurm_gpus: 1
+
       - label: "Unit: remapping"
         key: unit_remapping
         command: "julia --color=yes --check-bounds=yes --project=test test/Operators/remapping.jl"
@@ -468,33 +474,32 @@ steps:
 
       - label: "Unit: rectilinear cuda"
         key: unit_rectilinear_cuda
-        command: "julia --color=yes --check-bounds=yes --project=test test/Operators/spectralelement/rectilinear_cuda.jl"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Operators/spectralelement/rectilinear_cuda.jl CUDA"
         agents:
           slurm_gpus: 1
 
       - label: "Unit: hybrid operators cuda"
         key: unit_ops_cuda
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Operators/hybrid/cuda.jl CUDA"
         agents:
           slurm_gpus: 1
 
       - label: "Unit: extruded sphere cuda"
         key: unit_extruded_sphere_cuda
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Operators/hybrid/extruded_sphere_cuda.jl CUDA"
         agents:
           slurm_gpus: 1
 
       - label: "Unit: extruded 3dbox cuda"
         key: unit_extruded_3dbox_cuda
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Operators/hybrid/extruded_3dbox_cuda.jl CUDA"
         agents:
           slurm_gpus: 1
@@ -726,8 +731,8 @@ steps:
       - label: "Perf: SEM operator benchmarks (cuda Float32)"
         key: "perf_gpu_spectral_ops_cuda_float32"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
-          - "julia --color=yes --project=test test/Operators/spectralelement/benchmark_ops.jl --device CUDA --float-type Float32"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test test/Operators/spectralelement/benchmark_ops.jl --device CUDA --float-type Float32"
         agents:
           slurm_gpus: 1
 
@@ -738,8 +743,8 @@ steps:
       - label: "Perf: SEM operator benchmarks (cuda Float64)"
         key: "perf_gpu_spectral_ops_cuda_float64"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
-          - "julia --color=yes --project=test test/Operators/spectralelement/benchmark_ops.jl --device CUDA --float-type Float64"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test test/Operators/spectralelement/benchmark_ops.jl --device CUDA --float-type Float64"
         agents:
           slurm_gpus: 1
 
@@ -754,8 +759,8 @@ steps:
       - label: "Perf: SEM operator benchmarks"
         key: "perf_gpu_spectral_ops"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
-          - "julia --color=yes --project=test test/Operators/spectralelement/benchmark_ops.jl --device CUDA"
+          - "julia --color=yes --check-bounds=yes --project=test -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --color=yes --check-bounds=yes --project=test test/Operators/spectralelement/benchmark_ops.jl --device CUDA"
         agents:
           slurm_gpus: 1
 

diff --git a/src/Operators/finitedifference.jl b/src/Operators/finitedifference.jl
@@ -240,11 +240,16 @@ StencilBroadcasted{Style}(
 
 Adapt.adapt_structure(to, sbc::StencilBroadcasted{Style}) where {Style} =
     StencilBroadcasted{Style}(
-        sbc.op,
+        Adapt.adapt(to, sbc.op),
         Adapt.adapt(to, sbc.args),
         Adapt.adapt(to, sbc.axes),
     )
 
+function Adapt.adapt_structure(to, op::FiniteDifferenceOperator)
+    op
+end
+
+
 
 function Base.Broadcast.instantiate(sbc::StencilBroadcasted)
     op = sbc.op
@@ -2690,6 +2695,15 @@ Base.@propagate_inbounds function stencil_right_boundary(
     stencil_interior(op, loc, space, idx - 1, hidx, arg)
 end
 
+function Adapt.adapt_structure(to, op::DivergenceF2C)
+    DivergenceF2C(map(bc -> Adapt.adapt_structure(to, bc), op.bcs))
+end
+
+function Adapt.adapt_structure(to, bc::SetValue)
+    SetValue(Adapt.adapt_structure(to, bc.val))
+end
+
+
 """
     D = DivergenceC2F(;boundaryname=boundarycondition...)
     D.(v)

diff --git a/test/Operators/hybrid/3d.jl b/test/Operators/hybrid/3d.jl
@@ -18,6 +18,8 @@ import ClimaCore.Geometry: WVector
 import ClimaCore.Utilities: half
 import ClimaCore.DataLayouts: level
 
+device = ClimaComms.device()
+
 @testset "sphere divergence" begin
     FT = Float64
     vertdomain = Domains.IntervalDomain(
@@ -26,12 +28,16 @@ import ClimaCore.DataLayouts: level
         boundary_tags = (:bottom, :top),
     )
     vertmesh = Meshes.IntervalMesh(vertdomain, nelems = 10)
-    vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh)
+    verttopology = Topologies.IntervalTopology(
+        ClimaComms.SingletonCommsContext(device),
+        vertmesh,
+    )
+    vert_center_space = Spaces.CenterFiniteDifferenceSpace(verttopology)
 
     horzdomain = Domains.SphereDomain(30.0)
     horzmesh = Meshes.EquiangularCubedSphere(horzdomain, 4)
     horztopology = Topologies.Topology2D(
-        ClimaComms.SingletonCommsContext(ClimaComms.CPUSingleThreaded()),
+        ClimaComms.SingletonCommsContext(device),
         horzmesh,
     )
     quad = Spaces.Quadratures.GLL{3 + 1}()
@@ -53,7 +59,8 @@ function hvspace_3D(
     xelem = 4,
     yelem = 4,
     zelem = 16,
-    npoly = 7,
+    npoly = 7;
+    device = ClimaComms.device(),
 )
     FT = Float64
     vertdomain = Domains.IntervalDomain(
@@ -62,7 +69,11 @@ function hvspace_3D(
         boundary_tags = (:bottom, :top),
     )
     vertmesh = Meshes.IntervalMesh(vertdomain, nelems = zelem)
-    vert_center_space = Spaces.CenterFiniteDifferenceSpace(vertmesh)
+    verttopology = Topologies.IntervalTopology(
+        ClimaComms.SingletonCommsContext(device),
+        vertmesh,
+    )
+    vert_center_space = Spaces.CenterFiniteDifferenceSpace(verttopology)
 
     horzdomain = Domains.RectangleDomain(
         Geometry.XPoint{FT}(xlim[1]) .. Geometry.XPoint{FT}(xlim[2]),
@@ -72,7 +83,7 @@ function hvspace_3D(
     )
     horzmesh = Meshes.RectilinearMesh(horzdomain, xelem, yelem)
     horztopology = Topologies.Topology2D(
-        ClimaComms.SingletonCommsContext(ClimaComms.CPUSingleThreaded()),
+        ClimaComms.SingletonCommsContext(device),
         horzmesh,
     )
 
@@ -105,7 +116,8 @@ end
 
 @testset "2D SE, 1D FV Extruded Domain ∇ ODE Solve vertical" begin
 
-    hv_center_space, hv_face_space = hvspace_3D()
+    hv_center_space, hv_face_space =
+        hvspace_3D(; device = ClimaComms.CPUSingleThreaded())
     V =
         Geometry.UVWVector.(
             zeros(Float64, hv_face_space),
@@ -181,8 +193,12 @@ end
     #
     # NOTE: the equation setup is only correct for Cartesian domains!
 
-    hv_center_space, hv_face_space =
-        hvspace_3D((-1.0, 1.0), (-1.0, 1.0), (-1.0, 1.0))
+    hv_center_space, hv_face_space = hvspace_3D(
+        (-1.0, 1.0),
+        (-1.0, 1.0),
+        (-1.0, 1.0);
+        device = ClimaComms.CPUSingleThreaded(),
+    )
 
     abstract type BCtag end
     struct ZeroFlux <: BCtag end