From f7b9a9d583f8049e63f389ee8655f9cf8f365b58 Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 30 Mar 2024 18:47:48 -0400 Subject: [PATCH 1/5] fix skip breaking mark --- src/stream.jl | 74 ++++++++++++++++++++++-------------------- test/codecquadruple.jl | 37 +++++++++++++++++++++ 2 files changed, 75 insertions(+), 36 deletions(-) diff --git a/src/stream.jl b/src/stream.jl index f175ae8d..2f9b34f0 100644 --- a/src/stream.jl +++ b/src/stream.jl @@ -229,50 +229,26 @@ end end end -function Base.ismarked(stream::TranscodingStream) +function Base.ismarked(stream::TranscodingStream)::Bool checkmode(stream) - return ismarked(stream.buffer1) + ismarked(stream.buffer1) end -function Base.mark(stream::TranscodingStream) - checkmode(stream) - return mark!(stream.buffer1) +function Base.mark(stream::TranscodingStream)::Int64 + ready_to_read!(stream) + mark!(stream.buffer1) + position(stream) end -function Base.unmark(stream::TranscodingStream) +function Base.unmark(stream::TranscodingStream)::Bool checkmode(stream) - return unmark!(stream.buffer1) + unmark!(stream.buffer1) end -function Base.reset(stream::TranscodingStream) - checkmode(stream) - return reset!(stream.buffer1) -end - -function Base.skip(stream::TranscodingStream, offset::Integer) - checkmode(stream) - if offset < 0 - throw(ArgumentError("negative offset")) - end - mode = stream.state.mode - buffer1 = stream.buffer1 - skipped = 0 - if mode === :read || mode === :stop - while !eof(stream) && buffersize(buffer1) < offset - skipped - n = buffersize(buffer1) - emptybuffer!(buffer1) - skipped += n - end - if eof(stream) - emptybuffer!(buffer1) - else - skipbuffer!(buffer1, offset - skipped) - end - else - # TODO: support skip in write mode - throw(ArgumentError("not in read mode")) - end - return stream +function Base.reset(stream::T) where T<:TranscodingStream + Base.ismarked(stream) || throw(ArgumentError("$T not marked")) + reset!(stream.buffer1) + position(stream) end """ @@ -389,6 +365,32 @@ function Base.readuntil(stream::TranscodingStream, delim::UInt8; keep::Bool=fals return ret end +""" + skip(stream::TranscodingStream, offset) + +Read bytes from `stream` until `offset` bytes have been read or `eof(stream)` is reached. + +Return `stream`, discarding read bytes. + +This function will not throw an `EOFError` if `eof(stream)` is reached before +`offset` bytes can be read. +""" +function Base.skip(stream::TranscodingStream, offset::Integer) + if offset < 0 + # TODO support negative offset if stream is marked + throw(ArgumentError("negative offset")) + end + ready_to_read!(stream) + buffer1 = stream.buffer1 + skipped = 0 + while skipped < offset && !eof(stream) + n = min(buffersize(buffer1), offset - skipped) + skipbuffer!(buffer1, n) + skipped += n + end + return stream +end + function Base.unsafe_read(stream::TranscodingStream, output::Ptr{UInt8}, nbytes::UInt) ready_to_read!(stream) buffer = stream.buffer1 diff --git a/test/codecquadruple.jl b/test/codecquadruple.jl index a604a0fd..5c23c85f 100644 --- a/test/codecquadruple.jl +++ b/test/codecquadruple.jl @@ -110,6 +110,43 @@ end end end + @testset "skip" begin + @testset "position" begin + # make sure position is updated correctly by skip + iob = IOBuffer(repeat("x", 400)) + source = IOBuffer(repeat("x", 100)) + stream = TranscodingStream(QuadrupleCodec(), source, bufsize=16) + @test position(stream) == position(iob) + for len in 0:10:100 + skip(stream, len) + skip(iob, len) + @test position(stream) == position(iob) + end + close(stream) + end + @testset "mark" begin + iob = IOBuffer(repeat("x", 400)) + source = IOBuffer(repeat("x", 100)) + stream = TranscodingStream(QuadrupleCodec(), source, bufsize=16) + @test position(stream) == position(iob) + @test mark(stream) == mark(iob) + for len in 0:10:100 + skip(stream, len) + skip(iob, len) + @test position(stream) == position(iob) + end + @test Base.reset(stream) == Base.reset(iob) + for len in 0:10:100 + skip(stream, len) + skip(iob, len) + @test position(stream) == position(iob) + end + @test mark(stream) == mark(iob) + @test Base.reset(stream) == Base.reset(iob) + close(stream) + end + end + @testset "seekstart" begin data = Vector(b"abracadabra") source = IOBuffer(data) From 13dec86bdd9c3759c03774872a81a4660fa3ee40 Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 30 Mar 2024 19:50:14 -0400 Subject: [PATCH 2/5] add more tests --- ext/TestExt.jl | 6 +++++- test/codecnoop.jl | 10 ++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ext/TestExt.jl b/ext/TestExt.jl index 71aa1226..b7c7de36 100644 --- a/ext/TestExt.jl +++ b/ext/TestExt.jl @@ -88,11 +88,15 @@ function TranscodingStreams.test_chunked_read(Encoder, Decoder) ok = true for chunk in chunks stream = TranscodingStream(Decoder(), buffer, stop_on_end=true) - ok &= hash(read(stream)) == hash(chunk) + ok &= read(stream) == chunk ok &= eof(stream) ok &= isreadable(stream) close(stream) end + # read without stop_on_end should read the full data. + stream = TranscodingStream(Decoder(), IOBuffer(data)) + ok &= read(stream) == reduce(vcat, chunks) + close(stream) Test.@test ok end finalize(encoder) diff --git a/test/codecnoop.jl b/test/codecnoop.jl index 2acd28a7..5a9d371f 100644 --- a/test/codecnoop.jl +++ b/test/codecnoop.jl @@ -59,16 +59,18 @@ data = collect(0x00:0x0f) stream = TranscodingStream(Noop(), IOBuffer(data)) @test !ismarked(stream) - mark(stream) + @test mark(stream) == 0 @test ismarked(stream) @test [read(stream, UInt8) for _ in 1:3] == data[1:3] - reset(stream) + @test reset(stream) == 0 + @test_throws ArgumentError reset(stream) @test !ismarked(stream) @test [read(stream, UInt8) for _ in 1:3] == data[1:3] - mark(stream) + @test mark(stream) == 3 @test ismarked(stream) - unmark(stream) + @test unmark(stream) @test !ismarked(stream) + @test !unmark(stream) close(stream) stream = TranscodingStream(Noop(), IOBuffer(b"foobarbaz")) From d514eb89f4f9f7c66ac198136632f9487c10bb61 Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 30 Mar 2024 20:05:11 -0400 Subject: [PATCH 3/5] test closing removes mark --- src/stream.jl | 4 ++-- test/codecnoop.jl | 2 ++ test/codecquadruple.jl | 4 ++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/stream.jl b/src/stream.jl index 2f9b34f0..0eae4982 100644 --- a/src/stream.jl +++ b/src/stream.jl @@ -231,7 +231,7 @@ end function Base.ismarked(stream::TranscodingStream)::Bool checkmode(stream) - ismarked(stream.buffer1) + isopen(stream) && ismarked(stream.buffer1) end function Base.mark(stream::TranscodingStream)::Int64 @@ -242,7 +242,7 @@ end function Base.unmark(stream::TranscodingStream)::Bool checkmode(stream) - unmark!(stream.buffer1) + isopen(stream) && unmark!(stream.buffer1) end function Base.reset(stream::T) where T<:TranscodingStream diff --git a/test/codecnoop.jl b/test/codecnoop.jl index 5a9d371f..3583cbdc 100644 --- a/test/codecnoop.jl +++ b/test/codecnoop.jl @@ -71,7 +71,9 @@ @test unmark(stream) @test !ismarked(stream) @test !unmark(stream) + @test mark(stream) == 3 close(stream) + @test !ismarked(stream) stream = TranscodingStream(Noop(), IOBuffer(b"foobarbaz")) @test stream == seek(stream, 2) diff --git a/test/codecquadruple.jl b/test/codecquadruple.jl index 5c23c85f..1e17db94 100644 --- a/test/codecquadruple.jl +++ b/test/codecquadruple.jl @@ -143,7 +143,11 @@ end end @test mark(stream) == mark(iob) @test Base.reset(stream) == Base.reset(iob) + @test mark(stream) == mark(iob) close(stream) + close(iob) + @test !ismarked(stream) + @test !ismarked(iob) end end From 84930cea33657255564cf4f376a0f1721d0d3a5e Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 30 Mar 2024 20:24:05 -0400 Subject: [PATCH 4/5] add docs --- docs/src/reference.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/reference.md b/docs/src/reference.md index 0773c338..610e23fa 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -19,6 +19,7 @@ TranscodingStreams.unsafe_read TranscodingStreams.unread TranscodingStreams.unsafe_unread Base.position(stream::TranscodingStream) +Base.skip ``` Statistics From 7e603834699a71e8485231253028a68682d0fb4d Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 30 Mar 2024 20:54:49 -0400 Subject: [PATCH 5/5] test skip 0 behavior --- test/codecdoubleframe.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/codecdoubleframe.jl b/test/codecdoubleframe.jl index 672dbff3..f3b5076d 100644 --- a/test/codecdoubleframe.jl +++ b/test/codecdoubleframe.jl @@ -287,7 +287,7 @@ DoubleFrameDecoderStream(stream::IO; kwargs...) = TranscodingStream(DoubleFrameD @testset "reading zero bytes from invalid stream" begin # This behavior is required to avoid breaking JLD2.jl # `s` must go into read mode, but not actually call `eof` - for readnone in (io -> read!(io, UInt8[]), io -> read(io, 0)) + for readnone in (io -> read!(io, UInt8[]), io -> read(io, 0), io -> skip(io, 0)) for invalid_data in (b"", b"asdf") s = DoubleFrameDecoderStream(IOBuffer(invalid_data;read=true,write=true)) @test iswritable(s)