Add basic Supposition.jl tests #190
Conversation
No, thank YOU for using the package! Really great to see that it helped you find a bug, that's awesome! If you want to try your hand at some more complex/higher-level encoder/decoder properties, this article by the authors of Hypothesis might be insightful.
Hmm.. nicer in what sense exactly? Do you mean as a nicer printed list of "these steps are needed to wrap an The printing of the results is really ugly and I do want to make that prettier (see Seelengrab/Supposition.jl#23 for the basic target I'm going to aim for), but that's currently a bit blocked by StyledStrings.jl not yet being released. I could expand the printing minimally for now, so that it's not rendered as one big blob and rather have each argument on its own line? There's also Seelengrab/Supposition.jl#26, which I think would be a bit of a better fit here; basically everything you have here is an application of either a noop-operation, an enc-dec-operation or a dec-enc-opration on an The invariant you'd then test for after applying all "operations" is what you have right now in e.g. using Supposition, TranscodingStreams
include("../test/codecdoubleframe.jl")
const TS_kwarg = @composed (
bufsize=Data.Integers(1, 2^10),
stop_on_end=Data.Booleans(),
sharedbuf=Data.Booleans(),
) -> (
if sharedbuf
# default sharedbuf
(;bufsize, stop_on_end)
else
# sharedbuf = false
(;bufsize, stop_on_end, sharedbuf)
end
)
function noop_wrapper(io, kw)
event!("noop", kw)
NoopStream(io; kw...)
end
function r_enc_dec_wrapper(io, kw_enc, kw_dec)
event!("encoder", kw_enc)
event!("decoder", kw_dec)
DoubleFrameDecoderStream(DoubleFrameEncoderStream(io; kw_enc...); kw_dec...)
end
const datas = Data.Vectors(Data.Integers{UInt8}())
const read_ops = Data.Vectors(Data.SampledFrom((noop_wrapper, r_enc_dec_wrapper)); max_size=5)
function prepare_kws(ops)
res = []
sizehint!(res, length(ops))
for op in ops
kw = Data.produce!(TS_kwarg)
if op === noop_wrapper
push!(res, (op, (kw,)))
else
kw2 = Data.produce!(TS_kwarg)
push!(res, (op, (kw,kw2)))
end
end
res
end
function prepare_stream(ops, data::Vector{UInt8})::IO
stream::IO = IOBuffer(data)
foldl(ops; init=stream) do stream, (op, args)
op(stream, args...)
end
end
@time @check db=false function read_data(
kws=map(prepare_kws, read_ops),
data=datas)
stream = prepare_stream(kws, data)
for i in eachindex(data)
read(stream, UInt8) == data[i] || return false
end
eof(stream)
end;which is reasonably fast, shrinks well and is still pretty readable when it comes to the example: Events occured: 4
encoder
(bufsize = 1, stop_on_end = false, sharedbuf = false)
decoder
(bufsize = 1, stop_on_end = true, sharedbuf = false)
encoder
(bufsize = 1, stop_on_end = false)
decoder
(bufsize = 1, stop_on_end = false, sharedbuf = false)
┌ Error: Property errored!
│ Description = "read_data"
│ Example = (kws = Any[(r_enc_dec_wrapper, ((bufsize = 1, stop_on_end = false, sharedbuf = false), (bufsize = 1, stop_on_end = true, sharedbuf = false))), (r_enc_dec_wrapper, ((bufsize = 1, stop_on_end = false), (bufsize = 1, stop_on_end = false, sharedbuf = false)))], data = UInt8[])
│ exception =
│ ArgumentError: cannot change the mode from stop to read
│ Stacktrace:
[...]
Test Summary: | Error Total Time
read_data | 1 1 2.9s
3.037167 seconds (724.76 k allocations: 84.920 MiB, 0.45% gc time, 6.48% compilation time: 55% of which was recompilation)(I get varying runtimes of the same order for your original fuzzing code on my machine) The core difference is that you don't have to generate functions that do the wrapping, but instead have a function that does the wrapping and just call it during generation :) That'll make it much easier to debug, since you can then see in the output which operations led to the result, without having to explicitly call |
|
With a small patch to Supposition.jl, the final output could be made to look like this (omitting the stacktraces here for brevity) for your fuzzer: Events occured: 4
wrapping IOBuffer with:
nothing
encoder
(bufsize = 1, stop_on_end = false, sharedbuf = false)
decoder
(bufsize = 1, stop_on_end = true, sharedbuf = false)
noop
(bufsize = 1, stop_on_end = false, sharedbuf = true)
┌ Error: Property errored!
│ Description = "read_data"
│ w = (::var"#read_wrapper#12"{Vector{Union{var"#noop_wrapper#9", var"#r_enc_dec_wrapper#10"}}}) (generic function with 1 method)
│ data = UInt8[]
│ exception =
│ ArgumentError: cannot change the mode from stop to read
│ Stacktrace:and like this for the one I posted above: Events occured: 4
encoder
(bufsize = 1, stop_on_end = false, sharedbuf = false)
decoder
(bufsize = 1, stop_on_end = true, sharedbuf = false)
encoder
(bufsize = 1, stop_on_end = false)
decoder
(bufsize = 1, stop_on_end = false, sharedbuf = false)
┌ Error: Property errored!
│ Description = "read_data"
│ kws =
│ 2-element Vector{Any}:
│ (r_enc_dec_wrapper, ((bufsize = 1, stop_on_end = false, sharedbuf = false), (bufsize = 1, stop_on_end = true, sharedbuf = false)))
│ (r_enc_dec_wrapper, ((bufsize = 1, stop_on_end = false), (bufsize = 1, stop_on_end = false, sharedbuf = false)))
│ data = UInt8[]
│ exception =
│ ArgumentError: cannot change the mode from stop to read
│ Stacktrace: |
|
Thanks for the suggestions. Using |
These tests found a bug with reading data from nested streams if
stop_on_end=trueis set somewhere in the stack.To run the
fuzz.jltests run:@Seelengrab thank you for making Supposition.jl, this is my first attempt at using it. I think I'm using the
@composedmacro correctly, but is there something obvious I missed to make these tests nicer?Currently, I get the following messages showing counterexample nested streams: