don't inline long ntuples (#370)

* don't inline long ntuples

* fix edge case of Tuple{}

* @nospecialize to help compiler + NTuple repr

* add fallback for empty tuple

* remove spurious print statement

* bump patch version

Co-authored-by: Jonas Isensee <[email protected]>

Project.toml

@@ -1,6 +1,6 @@
 name = "JLD2"
 uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
-version = "0.4.18"
+version = "0.4.19"
 
 [deps]
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"

src/data/specialcased_types.jl

@@ -104,7 +104,7 @@ rconvert(::Type{BigFloat}, x::String) = parse(BigFloat, x)
 
 # Previously it was disallowed to serialize pointers.
 # Due to popular demand and in particular to not error on serializing complex structures
-# that contain non-essential pointers this has been changed to instead 
+# that contain non-essential pointers this has been changed to instead
 # return null pointers.
 writeas(::Type{Ptr{T}}) where {T} = Nothing
 rconvert(::Type{Ptr{T}}, ::Nothing) where {T} = Ptr{T}(0)
@@ -187,3 +187,23 @@ function rconvert(::Type{<:Base.ImmutableDict}, x::Vector{Pair{K,V}}) where {K,V
     end
     d
 end
+
+## NTuples
+# Immutable objects are stored as HDF5 structs and inlined into
+# parent structures. HDF5 only allows typemax(Unt16) bytes
+# for struct description. NTuples are the most common offender for
+# exploding struct size. (e.g. in the form of large StaticArrays)
+# The definitions below prevent inlining of large NTuples and
+# convert to an array instead.
+const NTUPLE_INLINE_THRESHOLD = 10
+
+function writeas(NT::Type{NTuple{N,T}}) where {N,T}
+    if N > NTUPLE_INLINE_THRESHOLD
+        return Vector{T}
+    else
+        return NT
+    end
+end
+
+wconvert(::Type{Vector{T}}, x::NTuple{N,T}) where {N,T} = collect(x)
+rconvert(::Type{NTuple{N,T}}, x::Vector{T}) where {N,T} = NTuple{N,T}(x)

src/data/writing_datatypes.jl

@@ -413,6 +413,22 @@ function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::DataType, wsess
 end
 
 
+# This is a trick to compactly write long NTuple
+# This uses that NTuple{N,T} == Tuple{T,T,T,T,...,T}
+function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::Type{NTuple{N,ET}}, wsession::JLDWriteSession) where {N,ET}
+    if isempty(T.parameters)
+        store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, "Tuple"), f.datatype_wsession)
+        h5convert_uninitialized!(out+odr_sizeof(Vlen{UInt8}), Vlen{UInt8})
+    else
+        store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, "NTuple"), f.datatype_wsession)
+        refs = refs_from_types(f, Any[N,ET], wsession)
+        store_vlen!(out+odr_sizeof(Vlen{UInt8}), RelOffset, f, refs, f.datatype_wsession)
+    end
+    nothing
+end
+
+
+
 ## Union Types
 
 const H5TYPE_UNION = CompoundDatatype(

src/datasets.jl

@@ -160,7 +160,7 @@ end
 
 # Most types can only be scalars or arrays
 function read_data(f::JLDFile,
-     rr,
+     @nospecialize(rr),
      read_dataspace::Tuple{ReadDataspace,RelOffset,Int,UInt16},
      attributes::Union{Vector{ReadAttribute},Nothing}=nothing)
 
@@ -315,7 +315,7 @@ function read_array(f::JLDFile, dataspace::ReadDataspace,
     io = f.io
     data_offset = position(io)
     ndims, offset = get_ndims_offset(f, dataspace, attributes)
-    
+
     seek(io, offset)
     v = construct_array(io, T, Int(ndims))
     n = length(v)
@@ -351,11 +351,11 @@ function write_dataset(
     io = f.io
     datasz = odr_sizeof(odr) * numel(dataspace)
     #layout_class
-    if datasz < 8192 
+    if datasz < 8192
         layout_class = LC_COMPACT_STORAGE
-    elseif compress != false 
-        layout_class = LC_CHUNKED_STORAGE 
-    else 
+    elseif compress != false
+        layout_class = LC_CHUNKED_STORAGE
+    else
         layout_class = LC_CONTIGUOUS_STORAGE
     end
     psz = payload_size_without_storage_message(dataspace, datatype)
@@ -565,7 +565,7 @@ function Base.delete!(g::Group, name::AbstractString)
 
     # Dataset must already exist in the file
     # Retrieve offset of group in file
-    offset = group_offset(g)    
+    offset = group_offset(g)
     offset == NULL_REFERENCE && throw(InternalError("Group could not be found."))
     delete_written_link!(g.f, offset, name)
     delete!(g.written_links, name)
@@ -615,7 +615,7 @@ function delete_written_link!(f::JLDFile, roffset::RelOffset, name::AbstractStri
         while (curpos = position(io)) <= chunk_checksum_offset - 4
             msg = jlread(io, HeaderMessage)
             endpos = curpos + jlsizeof(HeaderMessage) + msg.size
-            
+
             if msg.msg_type == HM_LINK_MESSAGE
                 dataset_name, loffset = read_link(io)
                 if dataset_name == name

test/loadsave.jl

@@ -464,3 +464,12 @@ end
 
     rm(tmpdir; force = true, recursive = true)
 end
+
+# Test for saving long NTuples
+@testset "Long NTuples" begin
+    cd(mktempdir()) do
+        tup = ntuple(i->i^2, 5000)
+        jldsave("test.jld2"; tup)
+        @test tup == load_object("test.jld2")
+    end
+end

test/modules.jl

@@ -1,6 +1,4 @@
 push!(LOAD_PATH, joinpath(pwd(),"testmodules/"))
-println(LOAD_PATH)
-
 module TestModule
 
 using A