diff --git a/Project.toml b/Project.toml index e231fdaa..ec4df9cf 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "JLD2" uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -version = "0.4.18" +version = "0.4.19" [deps] DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" diff --git a/src/data/specialcased_types.jl b/src/data/specialcased_types.jl index 925f7603..9d0a7b22 100644 --- a/src/data/specialcased_types.jl +++ b/src/data/specialcased_types.jl @@ -104,7 +104,7 @@ rconvert(::Type{BigFloat}, x::String) = parse(BigFloat, x) # Previously it was disallowed to serialize pointers. # Due to popular demand and in particular to not error on serializing complex structures -# that contain non-essential pointers this has been changed to instead +# that contain non-essential pointers this has been changed to instead # return null pointers. writeas(::Type{Ptr{T}}) where {T} = Nothing rconvert(::Type{Ptr{T}}, ::Nothing) where {T} = Ptr{T}(0) @@ -187,3 +187,23 @@ function rconvert(::Type{<:Base.ImmutableDict}, x::Vector{Pair{K,V}}) where {K,V end d end + +## NTuples +# Immutable objects are stored as HDF5 structs and inlined into +# parent structures. HDF5 only allows typemax(Unt16) bytes +# for struct description. NTuples are the most common offender for +# exploding struct size. (e.g. in the form of large StaticArrays) +# The definitions below prevent inlining of large NTuples and +# convert to an array instead. +const NTUPLE_INLINE_THRESHOLD = 10 + +function writeas(NT::Type{NTuple{N,T}}) where {N,T} + if N > NTUPLE_INLINE_THRESHOLD + return Vector{T} + else + return NT + end +end + +wconvert(::Type{Vector{T}}, x::NTuple{N,T}) where {N,T} = collect(x) +rconvert(::Type{NTuple{N,T}}, x::Vector{T}) where {N,T} = NTuple{N,T}(x) diff --git a/src/data/writing_datatypes.jl b/src/data/writing_datatypes.jl index 7f03dfc0..3d9de4fc 100644 --- a/src/data/writing_datatypes.jl +++ b/src/data/writing_datatypes.jl @@ -413,6 +413,22 @@ function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::DataType, wsess end +# This is a trick to compactly write long NTuple +# This uses that NTuple{N,T} == Tuple{T,T,T,T,...,T} +function h5convert!(out::Pointers, ::DataTypeODR, f::JLDFile, T::Type{NTuple{N,ET}}, wsession::JLDWriteSession) where {N,ET} + if isempty(T.parameters) + store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, "Tuple"), f.datatype_wsession) + h5convert_uninitialized!(out+odr_sizeof(Vlen{UInt8}), Vlen{UInt8}) + else + store_vlen!(out, UInt8, f, unsafe_wrap(Vector{UInt8}, "NTuple"), f.datatype_wsession) + refs = refs_from_types(f, Any[N,ET], wsession) + store_vlen!(out+odr_sizeof(Vlen{UInt8}), RelOffset, f, refs, f.datatype_wsession) + end + nothing +end + + + ## Union Types const H5TYPE_UNION = CompoundDatatype( diff --git a/src/datasets.jl b/src/datasets.jl index e20645f9..87c1c0a3 100644 --- a/src/datasets.jl +++ b/src/datasets.jl @@ -160,7 +160,7 @@ end # Most types can only be scalars or arrays function read_data(f::JLDFile, - rr, + @nospecialize(rr), read_dataspace::Tuple{ReadDataspace,RelOffset,Int,UInt16}, attributes::Union{Vector{ReadAttribute},Nothing}=nothing) @@ -315,7 +315,7 @@ function read_array(f::JLDFile, dataspace::ReadDataspace, io = f.io data_offset = position(io) ndims, offset = get_ndims_offset(f, dataspace, attributes) - + seek(io, offset) v = construct_array(io, T, Int(ndims)) n = length(v) @@ -351,11 +351,11 @@ function write_dataset( io = f.io datasz = odr_sizeof(odr) * numel(dataspace) #layout_class - if datasz < 8192 + if datasz < 8192 layout_class = LC_COMPACT_STORAGE - elseif compress != false - layout_class = LC_CHUNKED_STORAGE - else + elseif compress != false + layout_class = LC_CHUNKED_STORAGE + else layout_class = LC_CONTIGUOUS_STORAGE end psz = payload_size_without_storage_message(dataspace, datatype) @@ -565,7 +565,7 @@ function Base.delete!(g::Group, name::AbstractString) # Dataset must already exist in the file # Retrieve offset of group in file - offset = group_offset(g) + offset = group_offset(g) offset == NULL_REFERENCE && throw(InternalError("Group could not be found.")) delete_written_link!(g.f, offset, name) delete!(g.written_links, name) @@ -615,7 +615,7 @@ function delete_written_link!(f::JLDFile, roffset::RelOffset, name::AbstractStri while (curpos = position(io)) <= chunk_checksum_offset - 4 msg = jlread(io, HeaderMessage) endpos = curpos + jlsizeof(HeaderMessage) + msg.size - + if msg.msg_type == HM_LINK_MESSAGE dataset_name, loffset = read_link(io) if dataset_name == name diff --git a/test/loadsave.jl b/test/loadsave.jl index f3930037..623695aa 100644 --- a/test/loadsave.jl +++ b/test/loadsave.jl @@ -464,3 +464,12 @@ end rm(tmpdir; force = true, recursive = true) end + +# Test for saving long NTuples +@testset "Long NTuples" begin + cd(mktempdir()) do + tup = ntuple(i->i^2, 5000) + jldsave("test.jld2"; tup) + @test tup == load_object("test.jld2") + end +end diff --git a/test/modules.jl b/test/modules.jl index 58658030..1389c69b 100644 --- a/test/modules.jl +++ b/test/modules.jl @@ -1,6 +1,4 @@ push!(LOAD_PATH, joinpath(pwd(),"testmodules/")) -println(LOAD_PATH) - module TestModule using A