Wrap repro tests in testset, debug

CliMA · Dec 23, 2024 · df626de · df626de
1 parent b2b3e7f
commit df626de
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 13 deletions.
diff --git a/reproducibility_tests/mse_summary.jl b/reproducibility_tests/mse_summary.jl
@@ -10,6 +10,7 @@ include(joinpath(@__DIR__, "reproducibility_test_job_ids.jl"))
 job_ids = reproducibility_test_job_ids()
 
 computed_mses = get_computed_mses(; job_ids)
+isempty(computed_mses) && @warn "No MSEs were computed"
 print_mse_summary(; mses = computed_mses)
 print_skipped_jobs(; mses = computed_mses)
 

diff --git a/reproducibility_tests/test_mse.jl b/reproducibility_tests/test_mse.jl
@@ -10,13 +10,17 @@ include(joinpath(@__DIR__, "reproducibility_tools.jl"))
 (; job_id, out_dir, test_broken_report_flakiness) =
     reproducibility_test_params()
 
+debug = true
 repro_dir = joinpath(out_dir, "reproducibility_bundle")
 computed_mse_filenames =
     map(filter(default_is_mse_file, readdir(repro_dir))) do x
         joinpath(repro_dir, x)
     end
-
 if isempty(computed_mse_filenames)
+    @warn "No reproducibility tests performed, due to non-existent comparable data."
+    debug && @show readdir(out_dir)
+    debug && @show readdir(repro_dir)
+    debug && @show filter(default_is_mse_file, readdir(repro_dir))
     dirs = latest_comparable_dirs()
     if isempty(dirs) # no comparable references
         bins = compute_bins() # all reproducible bins, may or may not be comparable
@@ -46,19 +50,21 @@ if isempty(computed_mse_filenames)
         error(msg)
     end
 else
-    commit_hashes =
-        map(x -> basename(dirname(dirname(x))), computed_mse_filenames)
-    results = report_reproducibility_results(
-        commit_hashes,
-        computed_mse_filenames;
-        test_broken_report_flakiness,
-    )
+    @testset "Reproducibility tests" begin
+        commit_hashes =
+            map(x -> basename(dirname(dirname(x))), computed_mse_filenames)
+        results = report_reproducibility_results(
+            commit_hashes,
+            computed_mse_filenames;
+            test_broken_report_flakiness,
+        )
 
-    if test_broken_report_flakiness
-        @test results == :not_yet_reproducible
-        @test_broken results == :now_reproducible
-    else
-        @test results == :reproducible
+        if test_broken_report_flakiness
+            @test results == :not_yet_reproducible
+            @test_broken results == :now_reproducible
+        else
+            @test results == :reproducible
+        end
     end
 end