NCDataset memory leaks fixed

CliMA · Dec 12, 2022 · d9c40cc · d9c40cc
1 parent 4a57b78
commit d9c40cc
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 26 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -77,17 +77,17 @@ steps:
           slurm_nodes: 3
           slurm_tasks_per_node: 1
 
-      # - label: "MPI BCReader unit tests"
-      #   key: "bcreader_mpi_tests"
-      #   command: "mpiexec julia --color=yes --project=test/ test/mpi_tests/bcreader_mpi_tests.jl"
-      #   timeout_in_minutes: 20
-      #   env:
-      #     CLIMACORE_DISTRIBUTED: "MPI"
-      #   agents:
-      #     config: cpu
-      #     queue: central
-      #     slurm_nodes: 3
-      #     slurm_tasks_per_node: 1
+      - label: "MPI BCReader unit tests"
+        key: "bcreader_mpi_tests"
+        command: "mpiexec julia --color=yes --project=test/ test/mpi_tests/bcreader_mpi_tests.jl"
+        timeout_in_minutes: 20
+        env:
+          CLIMACORE_DISTRIBUTED: "MPI"
+        agents:
+          config: cpu
+          queue: central
+          slurm_nodes: 3
+          slurm_tasks_per_node: 1
 
   - group: "Integration Tests"
     steps:

diff --git a/test/bcreader_tests.jl b/test/bcreader_tests.jl
@@ -294,11 +294,15 @@ for FT in (Float32, Float64)
             outfile = hd_outfile_root * ".nc"
             outfile_root = mono ? outfile[1:(end - 3)] * "_mono" : outfile[1:(end - 3)]
             weightfile = joinpath(regrid_dir, outfile_root * "_remap_weights.nc")
-            weights = NCDataset(weightfile)["S"]
-
+
             # test monotone remapping (all weights in [0, 1])
-            max_weight = maximum(weights)
-            min_weight = minimum(weights)
+            nt = NCDataset(weightfile) do weights
+                max_weight = maximum(weights["S"])
+                min_weight = minimum(weights["S"])
+                (; max_weight, min_weight)
+            end
+            (; max_weight, min_weight) = nt
+
             @test max_weight <= FT(1.0) || isapprox(max_weight, FT(1.0), atol = 1e-16)
             @test min_weight >= FT(0.0) || isapprox(min_weight, FT(0.0), atol = 1e-16)
 

diff --git a/test/mpi_tests/bcreader_mpi_tests.jl b/test/mpi_tests/bcreader_mpi_tests.jl
@@ -63,11 +63,15 @@ ClimaComms.barrier(comms_ctx)
         outfile = hd_outfile_root * ".nc"
         outfile_root = mono ? outfile[1:(end - 3)] * "_mono" : outfile[1:(end - 3)]
         weightfile = joinpath(regrid_dir, outfile_root * "_remap_weights.nc")
-        weights = NCDataset(weightfile)["S"]
-
+
         # test monotone remapping (all weights in [0, 1])
-        max_weight = maximum(weights)
-        min_weight = minimum(weights)
+        nt = NCDataset(weightfile) do weights
+            max_weight = maximum(weights["S"])
+            min_weight = minimum(weights["S"])
+            (; max_weight, min_weight)
+        end
+        (; max_weight, min_weight) = nt
+
         @test max_weight <= FT(1.0) || isapprox(max_weight, FT(1.0), atol = 1e-16)
         @test min_weight >= FT(0.0) || isapprox(min_weight, FT(0.0), atol = 1e-16)
 

diff --git a/test/regridder_tests.jl b/test/regridder_tests.jl
@@ -106,9 +106,16 @@ for FT in (Float32, Float64)
 
             Regridder.remap_field_cgll_to_rll(name, field, remap_tmpdir, datafile_rll)
 
-            ncdataset_rll = NCDataset(datafile_rll)
-            @test maximum(ncdataset_rll[name]) == maximum(field)
-            @test minimum(ncdataset_rll[name]) == minimum(field)
+            # Test no new extrema are introduced in monotone remapping
+            nt = NCDataset(datafile_rll) do ds
+                max_remapped = maximum(ds[name])
+                min_remapped = minimum(ds[name])
+                (; max_remapped, min_remapped)
+            end
+            (; max_remapped, min_remapped) = nt
+
+            @test max_remapped <= maximum(field)
+            @test min_remapped >= minimum(field)
 
             # Delete testing directory and files
             rm(REGRID_DIR; recursive = true, force = true)
@@ -132,9 +139,16 @@ for FT in (Float32, Float64)
                 Regridder.land_sea_mask(FT, REGRID_DIR, comms_ctx, data_path, varname, test_space, mono = true)
 
             # Test no new extrema are introduced in monotone remapping
-            dataset = NCDataset(data_path)
-            @test maximum(dataset[varname]) >= maximum(land_mask_mono) &&
-                  minimum(dataset[varname]) <= minimum(land_mask_mono)
+            nt = NCDataset(data_path) do ds
+                max_val = maximum(ds[varname])
+                min_val = minimum(ds[varname])
+                (; max_val, min_val)
+            end
+            (; max_val, min_val) = nt
+
+            @test maximum(land_mask_mono) <= max_val
+            @test minimum(land_mask_mono) >= min_val
+
             # Test that monotone remapping a dataset of all ones conserves surface area
             @test sum(land_mask_mono) - 4 * π * (R^2) < 10e-14
 
@@ -153,7 +167,6 @@ for FT in (Float32, Float64)
             # Test non-monotone masking
             land_mask_halves =
                 Regridder.land_sea_mask(FT, REGRID_DIR, comms_ctx, data_path, varname, test_space, mono = false)
-            dataset = NCDataset(data_path)
 
             # Masking of values below threshold should result in 0
             @test all(parent(land_mask_halves) .== FT(0))