diff --git a/reproducibility_tests/reproducibility_tools.jl b/reproducibility_tests/reproducibility_tools.jl index 0ecb8844b5..bb11a881e1 100644 --- a/reproducibility_tests/reproducibility_tools.jl +++ b/reproducibility_tests/reproducibility_tools.jl @@ -180,7 +180,8 @@ function reproducibility_results( :no_comparable_dirs, ) - data_file_references = map(p -> joinpath(p, reference_filename), dirs) + data_file_references = + map(p -> joinpath(p, job_id, reference_filename), dirs) # foreach(x->maybe_extract(x), data_file_references) @@ -203,6 +204,11 @@ function reproducibility_results( dict_reference = dict_reference_solution, ) end + if debug_reproducibility() + println("------ end of reproducibility_results") + @show computed_mses + println("------") + end return (dirs, computed_mses, :successful_comparison) end @@ -275,8 +281,9 @@ function export_reproducibility_results( skip, ) + commit_shas = readdir(save_dir) for (computed_mse, dir) in zip(computed_mses, dirs) - commit_hash = basename(dirname(dir)) + commit_hash = commit_sha_from_dir(commit_shas, dir) computed_mse_file = joinpath(repro_dir, "computed_mse_$commit_hash.json") @@ -287,6 +294,15 @@ function export_reproducibility_results( return (data_file_computed, computed_mses, dirs, how) end +function commit_sha_from_json_file(file) + filename = basename(file) + if startswith(filename, "computed_mse_") && endswith(filename, ".json") + return replace(filename, "computed_mse_" => "", ".json" => "") + else + error("File $file does not follow correct format.") + end +end + import ClimaReproducibilityTests as CRT using Test import PrettyTables @@ -367,6 +383,11 @@ function report_reproducibility_results( for computed_mse in computed_mses all_reproducible = true + if debug_reproducibility() + println("---- in report_reproducibility_results") + @show computed_mse + println("----") + end for (var, reproducible) in CRT.test_mse(; computed_mse) if !reproducible all_reproducible = false @@ -420,8 +441,9 @@ function report_reproducibility_results( table_data = hcat(sources, summary_statuses) PrettyTables.pretty_table(io, table_data; header, crop = :none) + n_comparisons = length(computed_mses) println(io, "Summary:") - println(io, " n_comparisons = $(length(computed_mses))") + println(io, " n_comparisons = $n_comparisons") println(io, " n_times_reproducible = $n_times_reproducible") println(io, " n_times_not_reproducible = $n_times_not_reproducible") println(io, " n_passes = $n_passes") @@ -440,7 +462,7 @@ function report_reproducibility_results( elseif test_broken_report_flakiness return :not_yet_reproducible else - if n_passes ≥ n_pass_limit + if n_passes ≥ n_pass_limit || n_passes == n_comparisons return :reproducible else return :not_reproducible diff --git a/reproducibility_tests/reproducibility_utils.jl b/reproducibility_tests/reproducibility_utils.jl index 3e08beb0bc..306e38bae4 100644 --- a/reproducibility_tests/reproducibility_utils.jl +++ b/reproducibility_tests/reproducibility_utils.jl @@ -51,10 +51,9 @@ assist our understanding and reasoning, we let's assume that there are two state ## state 2: data is saved for future reference - - `commit_hash/job_id/output_dir/` - - `commit_hash/job_id/output_dir/reproducibility_bundle/` - - `commit_hash/job_id/output_dir/reproducibility_bundle/ref_counter.jl` - - `commit_hash/job_id/output_dir/reproducibility_bundle/prog_state.hdf5` + - `commit_hash/job_id/reproducibility_bundle/` + - `commit_hash/job_id/reproducibility_bundle/ref_counter.jl` + - `commit_hash/job_id/reproducibility_bundle/prog_state.hdf5` - `commit_hash/reproducibility_bundle/ref_counter.jl` - `commit_hash/reproducibility_bundle/job_id/` @@ -458,12 +457,12 @@ print_dir_tree(dir) = print_dir_tree(stdout, dir) print_dir_tree(io::IO, dir) = println(io, string_dir_tree(dir)) function string_dir_tree(dir) - s = "Files in `$dir`\n:" + s = "Files in `$dir`:\n" for (root, _, files) in walkdir(dir) for file in files f = joinpath(root, file) isfile(f) || continue # rm symlink folders (included but not files) - s *= " $f\n" + s *= " $(replace(f, dir => ""))\n" end end return s @@ -480,8 +479,7 @@ end ref_counter_PR = read_ref_counter(ref_counter_file_PR), skip = get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) != "climaatmos-ci", dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", strip_folder = strip_output_active_path, ) @@ -519,8 +517,7 @@ function move_data_to_save_dir(; ref_counter_PR = read_ref_counter(ref_counter_file_PR), skip = get(ENV, "BUILDKITE_PIPELINE_SLUG", nothing) != "climaatmos-ci", dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", strip_folder = strip_output_active_path, ) @@ -541,36 +538,20 @@ function move_data_to_save_dir(; dirs_src, dest_root, commit, - n_hash_characters, repro_folder, strip_folder, ) if debug_reproducibility() - @show repro_folder - @show dirs_src - @show dest_root - @show files_dest - @show files_src - @show isfile.(files_src) println("******") foreach(print_dir_tree, dirs_src) println("******") - print_dir_tree(dest_root) - println("******") end for (src, dest) in zip(files_src, files_dest) - @show src - @show dest @assert isfile(src) mkpath(dirname(dest)) mv(src, dest; force = true) end - dest_repro = destination_directory(; - dest_root, - commit, - n_hash_characters, - repro_folder, - ) + dest_repro = destination_directory(; dest_root, commit, repro_folder) ref_counter_file_main = joinpath(dest_repro, "ref_counter.jl") debug_reproducibility() && @info "Repro: moving $ref_counter_file_PR to $ref_counter_file_main" @@ -590,14 +571,40 @@ function move_data_to_save_dir(; end end +""" + get_commit_sha(; + n_hash_characters = 7, + commit = get(ENV, "BUILDKITE_COMMIT", nothing) + ) + +Returns a string of the commit hash. +""" +get_commit_sha(; + n_hash_characters = 7, + commit = get(ENV, "BUILDKITE_COMMIT", nothing), +) = return commit[1:min(n_hash_characters, length(commit))] + +function commit_sha_from_dir(commit_shas, dir) + while true + if isempty(dir) + error("Unfound commit sha.") + else + b = basename(dir) + if b in commit_shas || any(x -> occursin(b, x), commit_shas) + return b + else + dir = dirname(dir) + end + end + end +end """ save_dir_transform( src; job_id, dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", strip_folder = strip_output_active_path, ) @@ -607,7 +614,6 @@ Returns the output file, to be saved, given: - `job_id` the job ID - `dest_root` the destination root directory - `commit` the commit hash - - `n_hash_characters` truncates the commit hash to given number of characters - `repro_folder` reproducibility folder - `strip_folder` function to strip folders in output path """ @@ -615,17 +621,11 @@ function save_dir_transform( src; job_id, dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", strip_folder = strip_output_active_path, ) - dest_repro = destination_directory(; - dest_root, - commit, - n_hash_characters, - repro_folder, - ) + dest_repro = destination_directory(; dest_root, commit, repro_folder) src_filename = basename(src) dst = joinpath(dest_repro, job_id, src_filename) return strip_output_active_path(dst) @@ -634,8 +634,7 @@ end """ destination_directory(; dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", ) @@ -643,17 +642,14 @@ Return the reproducibility destination directory: `root/commit_sha/repro_folder`, given: - `dest_root` the destination root directory - `commit` the commit hash - - `n_hash_characters` truncates the commit hash to given number of characters - `repro_folder` reproducibility folder """ function destination_directory(; dest_root = "/central/scratch/esm/slurm-buildkite/climaatmos-main", - commit = get(ENV, "BUILDKITE_COMMIT", nothing), - n_hash_characters = 7, + commit = get_commit_sha(), repro_folder = "reproducibility_bundle", ) - commit_sha = commit[1:min(n_hash_characters, length(commit))] - return joinpath(dest_root, commit_sha, repro_folder) + return joinpath(dest_root, commit, repro_folder) end """ diff --git a/reproducibility_tests/test_mse.jl b/reproducibility_tests/test_mse.jl index 7e586b2059..f1b35cceea 100644 --- a/reproducibility_tests/test_mse.jl +++ b/reproducibility_tests/test_mse.jl @@ -12,11 +12,10 @@ include(joinpath(@__DIR__, "reproducibility_tools.jl")) debug = true repro_dir = joinpath(out_dir, "reproducibility_bundle") -computed_mse_filenames = - map(filter(default_is_mse_file, readdir(repro_dir))) do x - joinpath(repro_dir, x) - end -if isempty(computed_mse_filenames) +computed_mse_files = map(filter(default_is_mse_file, readdir(repro_dir))) do x + joinpath(repro_dir, x) +end +if isempty(computed_mse_files) @warn "No reproducibility tests performed, due to non-existent comparable data." debug && @show readdir(out_dir) debug && @show readdir(repro_dir) @@ -61,10 +60,16 @@ if isempty(computed_mse_filenames) else @testset "Reproducibility tests" begin commit_hashes = - map(x -> basename(dirname(dirname(x))), computed_mse_filenames) + map(x -> commit_sha_from_json_file(x), computed_mse_files) + computed_mses = map(x -> parse_file_json(x), computed_mse_files) + if debug_reproducibility() + println("------ in test_mse.jl") + @show computed_mses + println("------") + end results = report_reproducibility_results( commit_hashes, - computed_mse_filenames; + computed_mses; test_broken_report_flakiness, ) diff --git a/test/unit_reproducibility_infra.jl b/test/unit_reproducibility_infra.jl index 5afe803d3d..ae58f5826d 100644 --- a/test/unit_reproducibility_infra.jl +++ b/test/unit_reproducibility_infra.jl @@ -687,7 +687,6 @@ end make_and_cd() do dir job_id = "job_id" commit = "commit_sha" - n_hash_characters = 10 output = "output_active" strip_folder = output repro_folder = "rbundle" @@ -698,14 +697,12 @@ end dest_root = dir, job_id, commit, - n_hash_characters, repro_folder, strip_folder, ) == joinpath(dir, dst) job_id = "job_id" commit = "commit_sha" - n_hash_characters = 10 output = "output_active" strip_folder = output repro_folder = "rbundle" @@ -716,7 +713,6 @@ end dest_root = dir, job_id, commit, - n_hash_characters, repro_folder, strip_folder, ) == joinpath(dir, dst) @@ -734,6 +730,15 @@ end joinpath("a", "output_1A34", "c") end +@testset "Reproducibility infrastructure: commit_sha_from_json_file" begin + @test_throws ErrorException commit_sha_from_json_file( + joinpath("a", "b", "c"), + ) + @test commit_sha_from_json_file( + joinpath("a", "b", "computed_mse_H123.json"), + ) == "H123" +end + @testset "Reproducibility infrastructure: save_dir_in_out_list" begin mktempdir2_cd_computed() do (save_dir, computed_dir) hash1 = joinpath(save_dir, "hash1") @@ -772,7 +777,6 @@ end dirs_src = [job_id_1, job_id_2], dest_root = save_dir, commit = "commit_sha", - n_hash_characters = 10, repro_folder, strip_folder = "output_active", ) @@ -847,7 +851,6 @@ end dest_root = save_dir, buildkite_ci = true, commit = "hash_new", - n_hash_characters = length("hash_new"), branch = "unit_test_move_data_to_save_dir", in_merge_queue = true, dirs_src = [job_id_1, job_id_2], @@ -905,7 +908,6 @@ end dest_root = save_dir, buildkite_ci = true, commit = "hash_new", - n_hash_characters = length("hash_new"), branch = "unit_test_move_data_to_save_dir", in_merge_queue = true, dirs_src = [job_id_1, job_id_2], @@ -964,7 +966,6 @@ end dest_root = save_dir, buildkite_ci = true, commit = "hash_new", - n_hash_characters = length("hash_new"), branch = "unit_test_move_data_to_save_dir", in_merge_queue = true, dirs_src = [job_id_1, job_id_2], @@ -982,6 +983,41 @@ end @test isfile(joinpath(repro_dir, "ref_counter.jl")) end end +@testset "Reproducibility infrastructure: commit_sha_from_dir" begin + @test commit_sha_from_dir( + ["CH1", "CH2", "CH3"], + joinpath("a", "b", "c", "CH1", "e", "f"), + ) == "CH1" + @test commit_sha_from_dir( + ["CH1", "CH2", "CH3"], + joinpath("a", "b", "c", "CH2", "e", "f"), + ) == "CH2" + @test commit_sha_from_dir( + ["CH1", "CH2", "CH3"], + joinpath("a", "b", "c", "e", "f", "CH2"), + ) == "CH2" + @test commit_sha_from_dir( + ["CH1", "CH2", "CH3"], + joinpath("CH2", "a", "b", "c", "e", "f"), + ) == "CH2" + + @test commit_sha_from_dir( + ["CH1_xyz", "CH2_xyz", "CH3_xyz"], + joinpath("a", "b", "c", "CH1", "e", "f"), + ) == "CH1" + @test commit_sha_from_dir( + ["CH1_xyz", "CH2_xyz", "CH3_xyz"], + joinpath("a", "b", "c", "CH2", "e", "f"), + ) == "CH2" + @test commit_sha_from_dir( + ["CH1_xyz", "CH2_xyz", "CH3_xyz"], + joinpath("a", "b", "c", "e", "f", "CH2"), + ) == "CH2" + @test commit_sha_from_dir( + ["CH1_xyz", "CH2_xyz", "CH3_xyz"], + joinpath("CH2", "a", "b", "c", "e", "f"), + ) == "CH2" +end using ClimaComms using ClimaCore: Spaces, Fields, Grids, InputOutput @@ -1019,8 +1055,18 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" end end + # ## state 1: end of simulation, folder structure + # - `job_id/output_dir/` + # - `job_id/output_dir/reproducibility_bundle/` + # - `job_id/output_dir/reproducibility_bundle/ref_counter.jl` + # - `job_id/output_dir/reproducibility_bundle/prog_state.hdf5` + # ## state 2: data is saved for future reference + # - `commit_hash/job_id/reproducibility_bundle/` + # - `commit_hash/job_id/reproducibility_bundle/ref_counter.jl` + # - `commit_hash/job_id/reproducibility_bundle/prog_state.hdf5` + @testset "Reproducibility infrastructure: reproducibility_results - legacy folder structure" begin - make_and_cd() do dir + mktempdir2_cd_computed() do (save_dir, computed_dir) grid = ExtrudedCubedSphereGrid(; z_elem = 5, z_min = 0, @@ -1033,19 +1079,25 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) comms_ctx = ClimaComms.context(space) + # Folder structure: + job_id = "unit_test" + rfolder = "rbundle" + output = "output_active" + repro_dir = joinpath(computed_dir, job_id, output, rfolder) + mkpath(repro_dir) + fv = Fields.FieldVector(; x = ones(space), y = ones(space)) - file = joinpath(dir, "my_prog_state.hdf5") + file = joinpath(repro_dir, "my_prog_state.hdf5") hdfwriter = InputOutput.HDF5Writer(file, comms_ctx) InputOutput.write!(hdfwriter, fv, "Y") Base.close(hdfwriter) # Not on buildkite - job_id = "unit_test" (d, v, how) = reproducibility_results( comms_ctx; job_id, name = "Y", - save_dir = dir, + save_dir = save_dir, ref_counter_PR = 1, reference_filename = "my_prog_state.hdf5", data_file_computed = file, @@ -1066,7 +1118,7 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" comms_ctx; job_id, name = "Y", - save_dir = dir, + save_dir = save_dir, reference_filename = "my_prog_state.hdf5", ref_counter_PR = 1, data_file_computed = file, @@ -1083,26 +1135,111 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" # Successful comparison - d01 = make_ref_file_counter(1, dir, "01") - d02 = make_ref_file_counter(2, dir, "02") - d03 = make_ref_file_counter(3, dir, "03") - d04 = make_ref_file_counter(3, dir, "04") - d05 = make_ref_file_counter(3, dir, "05") - - put_data_file(d01, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d02, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d03, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d04, fv, comms_ctx; filename = "ref_prog_state.hdf5") + commit_sha_01 = "commit_hash_01" + commit_sha_02 = "commit_hash_02" + commit_sha_03 = "commit_hash_03" + commit_sha_04 = "commit_hash_04" + commit_sha_05 = "commit_hash_05" + d01 = make_ref_file_counter(1, save_dir, commit_sha_01, rfolder) + d02 = make_ref_file_counter(2, save_dir, commit_sha_02, rfolder) + d03 = make_ref_file_counter(3, save_dir, commit_sha_03, rfolder) + d04 = make_ref_file_counter(3, save_dir, commit_sha_04, rfolder) + d05 = make_ref_file_counter(3, save_dir, commit_sha_05, rfolder) + + put_data_file( + joinpath(d01, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d02, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d03, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d04, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) fv.x .= 200 fv.y .= 300 - put_data_file(d05, fv, comms_ctx; filename = "ref_prog_state.hdf5") + put_data_file( + joinpath(d05, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + + # Test folder structure + @test isfile( + joinpath( + computed_dir, + job_id, + output, + rfolder, + "my_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_01, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_02, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_03, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_04, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_05, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) - job_id = "unit_test" (d, v, how) = reproducibility_results( comms_ctx; job_id, name = "Y", - save_dir = dir, + save_dir = save_dir, ref_counter_PR = 3, reference_filename = "ref_prog_state.hdf5", data_file_computed = file, @@ -1138,18 +1275,21 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" comms_ctx = ClimaComms.context(space) fv = Fields.FieldVector(; x = ones(space), y = ones(space)) - file = joinpath( - computed_dir, - "reproducibility_bundle", - "computed_prog_state.hdf5", - ) - mkpath(dirname(file)) + + # Folder structure: + job_id = "unit_test" + rfolder = "rbundle" + output = "output_active" + repro_dir = joinpath(computed_dir, job_id, output, rfolder) + mkpath(repro_dir) + + file = joinpath(repro_dir, "computed_prog_state.hdf5") + mkpath(repro_dir) hdfwriter = InputOutput.HDF5Writer(file, comms_ctx) InputOutput.write!(hdfwriter, fv, "Y") Base.close(hdfwriter) # Not on buildkite - job_id = "unit_test" (d, v, how) = reproducibility_results( comms_ctx; job_id, @@ -1170,7 +1310,6 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" @test how == :skipped # Empty comparable dirs - job_id = "unit_test" (d, v, how) = reproducibility_results( comms_ctx; job_id, @@ -1192,19 +1331,49 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" # Successful comparison - d01 = make_ref_file_counter(1, save_dir, rbundle("01")) - d02 = make_ref_file_counter(2, save_dir, rbundle("02")) - d03 = make_ref_file_counter(3, save_dir, rbundle("03")) - d04 = make_ref_file_counter(3, save_dir, rbundle("04")) - d05 = make_ref_file_counter(3, save_dir, rbundle("05")) - - put_data_file(d01, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d02, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d03, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d04, fv, comms_ctx; filename = "ref_prog_state.hdf5") + commit_sha_01 = "commit_hash_01" + commit_sha_02 = "commit_hash_02" + commit_sha_03 = "commit_hash_03" + commit_sha_04 = "commit_hash_04" + commit_sha_05 = "commit_hash_05" + d01 = make_ref_file_counter(1, save_dir, commit_sha_01, rfolder) + d02 = make_ref_file_counter(2, save_dir, commit_sha_02, rfolder) + d03 = make_ref_file_counter(3, save_dir, commit_sha_03, rfolder) + d04 = make_ref_file_counter(3, save_dir, commit_sha_04, rfolder) + d05 = make_ref_file_counter(3, save_dir, commit_sha_05, rfolder) + + put_data_file( + joinpath(d01, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d02, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d03, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d04, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) fv.x .= 200 fv.y .= 300 - put_data_file(d05, fv, comms_ctx; filename = "ref_prog_state.hdf5") + put_data_file( + joinpath(d05, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) job_id = "unit_test" (d, v, how) = reproducibility_results( @@ -1229,6 +1398,62 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" @test d == [d05, d04, d03] @test how == :successful_comparison + + # Test folder structure + @test isfile( + joinpath( + computed_dir, + job_id, + output, + rfolder, + "computed_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_01, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_02, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_03, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_04, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) + @test isfile( + joinpath( + save_dir, + commit_sha_05, + rfolder, + job_id, + "ref_prog_state.hdf5", + ), + ) end end @@ -1246,7 +1471,12 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) comms_ctx = ClimaComms.context(space) + # Folder structure: job_id = "unit_test_export_reproducibility_results" + rfolder = "rbundle" + output = "output_active" + repro_dir = joinpath(computed_dir, job_id, output, rfolder) + mkpath(repro_dir) fv = Fields.FieldVector(; x = ones(space), y = ones(space)) @@ -1283,25 +1513,55 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" @test isempty(dirs) # Successful comparisons, legacy path configuration - d01 = make_ref_file_counter(1, save_dir, "01") - d02 = make_ref_file_counter(2, save_dir, "02") - d03 = make_ref_file_counter(3, save_dir, "03") - d04 = make_ref_file_counter(3, save_dir, "04") - d05 = make_ref_file_counter(3, save_dir, "05") - - put_data_file(d01, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d02, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d03, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d04, fv, comms_ctx; filename = "ref_prog_state.hdf5") + commit_sha_01 = "commit_hash_01" + commit_sha_02 = "commit_hash_02" + commit_sha_03 = "commit_hash_03" + commit_sha_04 = "commit_hash_04" + commit_sha_05 = "commit_hash_05" + d01 = make_ref_file_counter(1, save_dir, commit_sha_01) + d02 = make_ref_file_counter(2, save_dir, commit_sha_02) + d03 = make_ref_file_counter(3, save_dir, commit_sha_03) + d04 = make_ref_file_counter(3, save_dir, commit_sha_04) + d05 = make_ref_file_counter(3, save_dir, commit_sha_05) + + put_data_file( + joinpath(d01, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d02, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d03, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d04, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) fv.x .= 200 fv.y .= 300 - put_data_file(d05, fv, comms_ctx; filename = "ref_prog_state.hdf5") + put_data_file( + joinpath(d05, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) - @test isfile(joinpath(d01, "ref_prog_state.hdf5")) - @test isfile(joinpath(d02, "ref_prog_state.hdf5")) - @test isfile(joinpath(d03, "ref_prog_state.hdf5")) - @test isfile(joinpath(d04, "ref_prog_state.hdf5")) - @test isfile(joinpath(d05, "ref_prog_state.hdf5")) + @test isfile(joinpath(d01, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d02, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d03, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d04, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d05, job_id, "ref_prog_state.hdf5")) @test isfile(joinpath(d01, "ref_counter.jl")) @test isfile(joinpath(d02, "ref_counter.jl")) @test isfile(joinpath(d03, "ref_counter.jl")) @@ -1341,7 +1601,12 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) comms_ctx = ClimaComms.context(space) + # Folder structure: job_id = "unit_test_export_reproducibility_results" + rfolder = "rbundle" + output = "output_active" + repro_dir = joinpath(computed_dir, job_id, output, rfolder) + mkpath(repro_dir) fv = Fields.FieldVector(; x = ones(space), y = ones(space)) @@ -1359,6 +1624,7 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" n = 10, ref_counter_PR = 1, skip = true, + repro_folder = rfolder, ) @test how == :skipped @test isempty(dirs) @@ -1377,30 +1643,61 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" n = 10, ref_counter_PR = 1, skip = false, + repro_folder = rfolder, ) @test how == :no_comparable_dirs @test isempty(dirs) - # Successful comparisons, legacy path configuration - d01 = make_ref_file_counter(1, save_dir, rbundle("CH01")) - d02 = make_ref_file_counter(2, save_dir, rbundle("CH02")) - d03 = make_ref_file_counter(3, save_dir, rbundle("CH03")) - d04 = make_ref_file_counter(3, save_dir, rbundle("CH04")) - d05 = make_ref_file_counter(3, save_dir, rbundle("CH05")) - - put_data_file(d01, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d02, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d03, fv, comms_ctx; filename = "ref_prog_state.hdf5") - put_data_file(d04, fv, comms_ctx; filename = "ref_prog_state.hdf5") + # Successful comparisons, legacy path configuration (no repro folder) + commit_sha_01 = "sha_01" + commit_sha_02 = "sha_02" + commit_sha_03 = "sha_03" + commit_sha_04 = "sha_04" + commit_sha_05 = "sha_05" + d01 = make_ref_file_counter(1, save_dir, commit_sha_01) + d02 = make_ref_file_counter(2, save_dir, commit_sha_02) + d03 = make_ref_file_counter(3, save_dir, commit_sha_03) + d04 = make_ref_file_counter(3, save_dir, commit_sha_04) + d05 = make_ref_file_counter(3, save_dir, commit_sha_05) + + put_data_file( + joinpath(d01, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d02, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d03, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) + put_data_file( + joinpath(d04, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) fv.x .= 200 fv.y .= 300 - put_data_file(d05, fv, comms_ctx; filename = "ref_prog_state.hdf5") + put_data_file( + joinpath(d05, job_id), + fv, + comms_ctx; + filename = "ref_prog_state.hdf5", + ) - @test isfile(joinpath(d01, "ref_prog_state.hdf5")) - @test isfile(joinpath(d02, "ref_prog_state.hdf5")) - @test isfile(joinpath(d03, "ref_prog_state.hdf5")) - @test isfile(joinpath(d04, "ref_prog_state.hdf5")) - @test isfile(joinpath(d05, "ref_prog_state.hdf5")) + @test isfile(joinpath(d01, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d02, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d03, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d04, job_id, "ref_prog_state.hdf5")) + @test isfile(joinpath(d05, job_id, "ref_prog_state.hdf5")) @test isfile(joinpath(d01, "ref_counter.jl")) @test isfile(joinpath(d02, "ref_counter.jl")) @test isfile(joinpath(d03, "ref_counter.jl")) @@ -1420,14 +1717,21 @@ if pkgversion(ClimaCore) ≥ v"0.14.18" n = 10, ref_counter_PR = 3, skip = false, + repro_folder = rfolder, ) @test how == :successful_comparison @test dirs == [d05, d04, d03] - repro_dir = joinpath(computed_dir, "reproducibility_bundle") + repro_dir = joinpath(computed_dir, rfolder) @test isfile(joinpath(repro_dir, "computed_prog_state.hdf5")) - @test isfile(joinpath(repro_dir, "computed_mse_CH05.json")) - @test isfile(joinpath(repro_dir, "computed_mse_CH04.json")) - @test isfile(joinpath(repro_dir, "computed_mse_CH03.json")) + @test isfile( + joinpath(repro_dir, "computed_mse_$commit_sha_05.json"), + ) + @test isfile( + joinpath(repro_dir, "computed_mse_$commit_sha_04.json"), + ) + @test isfile( + joinpath(repro_dir, "computed_mse_$commit_sha_03.json"), + ) end end end