Compression loading (#369)

* update project and ci * add test for bug #368 * add fix * move modules test to subfolder * fix module related tests Co-authored-by: Jonas Isensee <[email protected]>
JuliaIO · Jan 26, 2022 · 4499f87 · 4499f87 · JonasIsensee · Jan 26, 2022
1 parent eebc008
commit 4499f87
Show file tree

Hide file tree

Showing 9 changed files with 55 additions and 46 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.0'
+          - '1.6'
           - '1'
           - 'nightly'
         os:  [ubuntu-latest, windows-latest, macos-latest] # adjust according to need, e.g. os: [ubuntu-latest] if testing only on linux
@@ -25,9 +25,6 @@ jobs:
           - os: ubuntu-latest
             version: '1'
             arch: x86
-        exclude:
-          - os: macos-latest
-            version: '1.0'
     steps:
       # Cancel ongoing CI test runs if pushing to branch again before the previous tests
       # have finished

diff --git a/Project.toml b/Project.toml
@@ -1,6 +1,6 @@
 name = "JLD2"
 uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-version = "0.4.17"
+version = "0.4.18"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
@@ -19,18 +19,4 @@ FileIO = "1"
 MacroTools = "0.5"
 Reexport = "1"
 TranscodingStreams = "0.9"
-julia = "0.7, 1"
-
-[extras]
-Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
-CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd"
-CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
-CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
-FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
-LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
-Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[targets]
-test = ["FileIO", "LinearAlgebra", "Printf", "Random", "Test", "Base64", "CodecLz4", "CodecZlib", "CodecBzip2"]
+julia = "1.6"
diff --git a/src/compression.jl b/src/compression.jl
@@ -23,7 +23,7 @@ function verify_compressor(compressor)
         "\n\t"*string(val[1])*"."*string(val[2])
     end
 
-    
+
     throw(ArgumentError("""Unsupported Compressor
     Supported Compressors are $(crs...)"""))
 end
@@ -76,7 +76,7 @@ function Base.write(g::Group, name::AbstractString, obj, wsession::JLDWriteSessi
     f = g.f
     prewrite(f)
     (g, name) = pathize(g, name, true)
-    if !(compress === nothing) 
+    if !(compress === nothing)
         verify_compressor(compress)
         if obj isa Array
             g[name] = write_dataset(f, obj, wsession, compress)
@@ -91,9 +91,14 @@ end
 
 get_compressor(compressor) = false, COMPRESSOR_TO_ID[nameof(typeof(compressor))], compressor
 
-function get_compressor(::Bool) 
+function get_compressor(::Bool)
    call_again, m = checked_import(:CodecZlib)
-    if call_again
+    if call_again || !applicable(m.ZlibCompressor)
+        # Reinvoke with latest world age if
+        # - we just loaded the CodecZlib
+        # - we didn't just load it but the constructor is still not `applicable`.
+        #   This happens when a save call wants to compress multiple datasets
+        #   and loaded CodecZlib for the first one.
         return true, Base.invokelatest(get_compressor, true)[2:3]...
     end
     false, COMPRESSOR_TO_ID[:ZlibCompressor], m.ZlibCompressor()
@@ -117,9 +122,9 @@ function write_filter_pipeline_message(io, filter_id::UInt16)
     jlwrite(io, UInt8(2))                 # Version
     jlwrite(io, UInt8(1))                 # Number of Filters
     jlwrite(io, filter_id)                # Filter Identification Value
-    filter_id > 255 && jlwrite(io, UInt16(length(filter_name))) 
+    filter_id > 255 && jlwrite(io, UInt16(length(filter_name)))
                                         # Length of Filter Name
-    jlwrite(io, UInt16(0))                # Flags 
+    jlwrite(io, UInt16(0))                # Flags
     jlwrite(io, UInt16(1))                # Number of Client Data Values
     filter_id > 255 && jlwrite(io, filter_name) # Filter Name
     jlwrite(io, UInt32(5))                # Client Data (Compression Level)
@@ -176,7 +181,7 @@ function write_compressed_data(cio, f, data, odr, wsession, filter_id, compresso
 
     write_chunked_storage_message(cio, odr_sizeof(odr), size(data), length(deflated), h5offset(f, f.end_of_data))
     jlwrite(f.io, end_checksum(cio))
-   
+
     f.end_of_data += length(deflated)
     jlwrite(f.io, deflated)
 end
@@ -192,7 +197,7 @@ function read_compressed_array!(v::Array{T}, f::JLDFile{MmapIO},
     invoke_again, decompressor = get_decompressor(filter_id)
     if invoke_again
         return Base.invokelatest(read_compressed_array!, v, f, rr, data_length, filter_id)
-    end                                    
+    end
     io = f.io
     inptr = io.curptr
     TranscodingStreams.initialize(decompressor)

diff --git a/test/Project.toml b/test/Project.toml
@@ -0,0 +1,11 @@
+[deps]
+Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd"
+CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
diff --git a/test/compression.jl b/test/compression.jl
@@ -1,5 +1,5 @@
 using JLD2, Test, FileIO
-
+using Pkg: Pkg
 using CodecZlib, CodecBzip2, CodecLz4
 
 
@@ -40,7 +40,7 @@ end
 
     randomdata = repeat(rand(2000), 10)
     @save fn {compress=LZ4FrameCompressor()} randomdata
-    
+
     r = jldopen(f -> f["randomdata"], fn, "r")
     @test r == randomdata
 end
@@ -53,4 +53,22 @@ end
     jldopen(fn, "w") do f
         @test_throws ArgumentError write(f, "x", zeros(10); compress = π)
     end
-end
+end
+
+
+@testset "issue #368 - dynamically loaded CodecZlib" begin
+    testprojectpath = Pkg.project().path
+    # Simply test if this fails
+    code = """
+    using JLD2
+
+    N = 100
+    a = (rand(N, N), rand(N, N))
+    save("test.jld2", "a", a; compress = true)
+    """
+
+    cd(mktempdir()) do
+        my_cmd = `$(Base.julia_cmd()) --project=$(testprojectpath) -e $(code)`
+        @test better_success(my_cmd)
+    end
+end
diff --git a/test/modules-nested.jl b/test/modules-nested.jl
@@ -253,13 +253,6 @@
         rm(tmpdir; force = true, recursive = true)
     end
     @testset "issue #154 - type exists in child module but not in parent module" begin
-        original_directory = pwd()
-        tmpdir = mktempdir()
-        atexit(() -> rm(tmpdir; force = true, recursive = true))
-
-        cd(tmpdir)
-        rm("test.jld"; force = true, recursive = true)
-
         code = """
         using JLD2
 
@@ -325,11 +318,9 @@
         Test.@test result[1] isa LinearAlgebra.Symmetric{Float64,Array{Float64,2}}
         Test.@test result[2] isa Pkg.Types.PackageSpec
         """
-
-        my_cmd = `$(Base.julia_cmd()) -e $(code)`
-        @test better_success(my_cmd)
-
-        cd(original_directory)
-        rm(tmpdir; force = true, recursive = true)
+        my_cmd = `$(Base.julia_cmd()) --project=$(pwd()) -e $(code)`
+        cd(mktempdir()) do
+            @test better_success(my_cmd)
+        end
     end
 end
diff --git a/test/modules.jl b/test/modules.jl
@@ -1,4 +1,5 @@
-push!(LOAD_PATH, pwd())
+push!(LOAD_PATH, joinpath(pwd(),"testmodules/"))
+println(LOAD_PATH)
 
 module TestModule
 
@@ -14,7 +15,7 @@ S = BType(x)
 @testset "save and load" begin
 	T = deepcopy(S)
 	@test T == S
-	fn = joinpath(mktempdir(), "test_out.jld")	
+	fn = joinpath(mktempdir(), "test_out.jld")
 	@save fn T
 	T = nothing
 	@load fn T

diff --git a/test/A.jl → test/testmodules/A.jl b/test/A.jl → test/testmodules/A.jl
diff --git a/test/B.jl → test/testmodules/B.jl b/test/B.jl → test/testmodules/B.jl