Merge pull request #32 from simonster/sjk/fix31

Don't put >65535 strings in a heap (fixes #31)

src/global_heaps.jl

@@ -25,11 +25,11 @@ function write_heap_object(f::JLDFile, odr, data, wsession::JLDWriteSession)
 
     # This is basically a memory allocation problem. Right now we do it
     # in a pretty naive way. We:
-
+    #
     # 1. Put the object in the last created global heap if it fits
     # 2. Extend the last global heap if it's at the end of the file
     # 3. Create a new global heap if we can't do 1 or 2
-
+    #
     # This is not a great approach if we're writing objects of
     # different sizes interspersed with new datasets. The torture case
     # would be a Vector{Any} of mutable objects, some of which contain
@@ -38,10 +38,12 @@ function write_heap_object(f::JLDFile, odr, data, wsession::JLDWriteSession)
     # strings into existing heaps, rather than writing new ones. This
     # should be revisited at a later date.
 
-    if objsz + 8 + sizeof(Length) < f.global_heap.free
+    # Can only fit up to typemax(UInt16) items in a single heap
+    heap_filled = length(f.global_heap.objects) >= typemax(UInt16)
+    if objsz + 8 + sizeof(Length) < f.global_heap.free && !heap_filled
         # Fits in existing global heap
         gh = f.global_heap
-    elseif isatend(f, f.global_heap)
+    elseif isatend(f, f.global_heap) && !heap_filled
         # Global heap is at end and can be extended
         gh = f.global_heap
         delta = objsz - gh.free + 8 + sizeof(Length)

test/rw.jl

@@ -265,14 +265,17 @@ cyclicobject.x = cyclicobject
 simplevec = Core.svec(1, 2, Int64, "foo")
 iseq(x::SimpleVector, y::SimpleVector) = collect(x) == collect(y)
 
-# Issue #243
+# JLD issue #243
 # Type that overloads != so that it is not boolean
 mutable struct NALikeType; end
 Base.:!=(::NALikeType, ::NALikeType) = NALikeType()
 Base.:!=(::NALikeType, ::Void) = NALikeType()
 Base.:!=(::Void, ::NALikeType) = NALikeType()
 natyperef = Any[NALikeType(), NALikeType()]
 
+# JLD2 issue #31 (lots of strings)
+lotsastrings = fill("a", 100000)
+
 iseq(x,y) = isequal(x,y)
 function iseq(x::Array{EmptyType}, y::Array{EmptyType})
     size(x) != size(y) && return false
@@ -346,14 +349,7 @@ function checkexpr(a::Expr, b::Expr)
     @assert i >= length(a.args) && j >= length(b.args)
 end
 
-fn = joinpath(tempdir(),"test.jld")
-
-# Issue #106
-module Mod106
-primitive type Typ{T} 64 end
-typ(x::Int64, ::Type{T}) where {T} = Base.box(Typ{T}, Base.unbox(Int64,x))
-abstract type UnexportedT end
-end
+fn = joinpath(tempdir(), "test.jld")
 
 println(fn)
 for ioty in [JLD2.MmapIO, IOStream], compress in [false, true]
@@ -468,6 +464,7 @@ for ioty in [JLD2.MmapIO, IOStream], compress in [false, true]
     @write fid cyclicobject
     @write fid simplevec
     @write fid natyperef
+    @write fid lotsastrings
     end
     close(fid)
 
@@ -604,6 +601,7 @@ for ioty in [JLD2.MmapIO, IOStream], compress in [false, true]
 
     @check fidr simplevec
     @check fidr natyperef
+    @check fidr lotsastrings
     end
     close(fidr)
 end