Skip to content

Commit

Permalink
Add GC-safe regions around some ccalls
Browse files Browse the repository at this point in the history
Specifically, around `LZ4_compress_fast`, `LZ4_compress_destSize` and
`LZ4_decompress_safe`. If these are called on large data (on the order
of GBs), the calls could take multiple seconds during which GC cannot
run without these GC-safe regions.
  • Loading branch information
kpamnany committed Nov 3, 2023
1 parent c1aca1e commit f1cf8f8
Showing 1 changed file with 28 additions and 3 deletions.
31 changes: 28 additions & 3 deletions src/headers/lz4.jl
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,15 @@ This function never writes outside `dst` buffer, nor read outside `source` buffe
Returns the number of bytes written into buffer `dst` (necessarily <= dstcapacity)
"""
function LZ4_compress_fast(src, dst, srcsize, dstcapacity, acceleration=1)
ret = ccall((:LZ4_compress_fast, liblz4), Cint, (Ptr{UInt8}, Ptr{UInt8}, Cint, Cint, Cint), src, dst, srcsize, dstcapacity, acceleration)
csrc = Base.cconvert(Ptr{UInt8}, src)
cdst = Base.cconvert(Ptr{UInt8}, dst)
GC.@preserve csrc cdst begin
# Allow Julia to GC while compressing
gc_state = @ccall(jl_gc_safe_enter()::Int8)
ret = ccall((:LZ4_compress_fast, liblz4), Cint, (Ptr{UInt8}, Ptr{UInt8}, Cint, Cint, Cint), Base.unsafe_convert(Ptr{UInt8}, csrc)::Ptr{UInt8}, Base.unsafe_convert(Ptr{UInt8}, cdst)::Ptr{UInt8}, srcsize, dstcapacity, acceleration)
# Leave GC-safe region, waiting for GC to complete if it's running
@ccall(jl_gc_safe_leave(gc_state::Int8)::Cvoid)
end
check_compression_error(ret, "LZ4_compress_fast")
end

Expand All @@ -63,7 +71,16 @@ or fill `dst` buffer completely with as much data as possible from `src`.
Returns number of bytes written into `dst` (necessarily <= dstcapacity)
"""
function LZ4_compress_destSize(src, dst, srcsize, dstcapacity)
ret = ccall((:LZ4_compress_destSize, liblz4), Cint, (Ptr{UInt8}, Ptr{UInt8}, Ptr{Cint}, Cint), src, dst, srcsize, dstcapacity)
csrc = Base.cconvert(Ptr{UInt8}, src)
cdst = Base.cconvert(Ptr{UInt8}, dst)
csrcsize = Base.cconvert(Ptr{Cint}, srcsize)
GC.@preserve csrc cdst csrcsize begin
# Allow Julia to GC while compressing
gc_state = @ccall(jl_gc_safe_enter()::Int8)
ret = ccall((:LZ4_compress_destSize, liblz4), Cint, (Ptr{UInt8}, Ptr{UInt8}, Ptr{Cint}, Cint), Base.unsafe_convert(Ptr{UInt8}, csrc)::Ptr{UInt8}, Base.unsafe_convert(Ptr{UInt8}, cdst)::Ptr{UInt8}, Base.unsafe_convert(Ptr{Cint}, csrcsize)::Ptr{Cint}, dstcapacity)
# Leave GC-safe region, waiting for GC to complete if it's running
@ccall(jl_gc_safe_leave(gc_state::Int8)::Cvoid)
end
check_compression_error(ret, "LZ4_compress_destSize")
end

Expand Down Expand Up @@ -150,7 +167,15 @@ dstcapacity : is the size of destination buffer, which must be already allocated
Returns the number of bytes decompressed into destination buffer (necessarily <= dstcapacity)
"""
function LZ4_decompress_safe(src, dst, cmpsize, dstcapacity)
ret = ccall((:LZ4_decompress_safe, liblz4), Cint, (Ptr{UInt8}, Ptr{UInt8}, Cint, Cint), src, dst, cmpsize, dstcapacity)
csrc = Base.cconvert(Ptr{UInt8}, src)
cdst = Base.cconvert(Ptr{UInt8}, dst)
GC.@preserve csrc cdst begin
# Allow Julia to GC while decompressing
gc_state = @ccall(jl_gc_safe_enter()::Int8)
ret = ccall((:LZ4_decompress_safe, liblz4), Cint, (Ptr{UInt8}, Ptr{UInt8}, Cint, Cint), Base.unsafe_convert(Ptr{UInt8}, csrc)::Ptr{UInt8}, Base.unsafe_convert(Ptr{UInt8}, cdst)::Ptr{UInt8}, cmpsize, dstcapacity)
# Leave GC-safe region, waiting for GC to complete if it's running
@ccall(jl_gc_safe_leave(gc_state::Int8)::Cvoid)
end
check_decompression_error(ret, "LZ4_decompress_safe")
end

Expand Down

0 comments on commit f1cf8f8

Please sign in to comment.