Copy testing utilities into separate package (#235)

.github/workflows/CI.yml

@@ -39,9 +39,25 @@ jobs:
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
+          show-versioninfo: true
       - uses: julia-actions/cache@v2
-      - uses: julia-actions/julia-buildpkg@v1
-      - uses: julia-actions/julia-runtest@v1
+      - name: Develop packages
+        run: |
+            using Pkg
+            Pkg.Registry.update()
+            Pkg.develop([
+                PackageSpec(path="."),
+                PackageSpec(path="./lib/TestsForCodecPackages"),
+            ])
+            Pkg.update()
+            Pkg.status(;mode=Pkg.PKGMODE_PROJECT)
+            Pkg.status(;mode=Pkg.PKGMODE_MANIFEST)
+        shell: julia --project=test --color=yes --check-bounds=yes {0}
+      - name: Run tests
+        env:
+          JULIA_LOAD_PATH: "@"
+        run: |
+          julia --project=test --color=yes --depwarn=yes --warn-overwrite=yes --check-bounds=yes --startup-file=no --code-coverage=user test/runtests.jl
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v4
         with:

.github/workflows/Downstream.yml

@@ -37,21 +37,31 @@ jobs:
         with:
           version: ${{ matrix.julia-version }}
           arch: x64
-      - uses: julia-actions/julia-buildpkg@latest
       - name: Clone Downstream
         uses: actions/checkout@v4
         with:
           repository: ${{ matrix.package.user }}/${{ matrix.package.repo }}
           path: downstream
       - name: Load this and run the downstream tests
-        shell: julia --color=yes --project=downstream {0}
+        shell: julia --color=yes {0}
         run: |
           using Pkg
+          using TOML
+          Pkg.Registry.update()
+          Pkg.activate(;temp=true)
           try
-            # force it to use this PR's version of the package
-            Pkg.develop(PackageSpec(path="."))  # resolver may fail with main deps
+            # force it to use this PR's version of the package and test package
+            Pkg.develop([
+              PackageSpec(path="downstream"),
+              PackageSpec(path="."),
+              PackageSpec(path="./lib/TestsForCodecPackages"),
+            ])
+            # resolver may fail with main deps
             Pkg.update()
-            Pkg.test()  # resolver may fail with test time deps
+            p1 = joinpath("downstream", "JuliaProject.toml")
+            p2 = joinpath("downstream", "Project.toml")
+            proj_toml = isfile(p1) ? p1 : p2
+            Pkg.test(TOML.parsefile(proj_toml)["name"])  # resolver may fail with test time deps
           catch err
             err isa Pkg.Resolve.ResolverError || rethrow()
             # If we can't resolve that means this is incompatible by SemVer and this is fine.

.gitignore

@@ -13,3 +13,4 @@
 /docs/Manifest.toml
 /fuzz/test/
 /fuzz/Manifest.toml
+/lib/TestsForCodecPackages/Manifest.toml

lib/TestsForCodecPackages/Project.toml

@@ -0,0 +1,15 @@
+name = "TestsForCodecPackages"
+uuid = "c2e61002-3542-480d-8b3c-5f05cc4f8554"
+authors = ["nhz2 <[email protected]>"]
+version = "0.1.0"
+
+[deps]
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
+
+[compat]
+Random = "1"
+Test = "1"
+TranscodingStreams = "0.11"
+julia = "1.6"

lib/TestsForCodecPackages/src/TestsForCodecPackages.jl

@@ -0,0 +1,160 @@
+module TestsForCodecPackages
+
+using Test: Test
+using Random: seed!, randstring
+
+using TranscodingStreams: TranscodingStreams, initialize, finalize, transcode, 
+    TranscodingStream, NoopStream, TOKEN_END
+
+TEST_RANDOM_SEED = 12345
+
+export
+    test_roundtrip_read,
+    test_roundtrip_write,
+    test_roundtrip_transcode,
+    test_roundtrip_lines,
+    test_roundtrip_seekstart,
+    test_roundtrip_fileio,
+    test_chunked_read,
+    test_chunked_write
+
+function test_roundtrip_read(encoder, decoder)
+    seed!(TEST_RANDOM_SEED)
+    for n in vcat(0:30, sort!(rand(500:100_000, 30))), alpha in (0x00:0xff, 0x00:0x0f)
+        data = rand(alpha, n)
+        file = IOBuffer(data)
+        stream = decoder(encoder(file))
+        Test.@test hash(read(stream)) == hash(data)
+        close(stream)
+    end
+end
+
+function test_roundtrip_write(encoder, decoder)
+    seed!(TEST_RANDOM_SEED)
+    for n in vcat(0:30, sort!(rand(500:100_000, 30))), alpha in (0x00:0xff, 0x00:0x0f)
+        data = rand(alpha, n)
+        sink = IOBuffer()
+        decode_sink = decoder(sink)
+        stream = encoder(decode_sink)
+        write(stream, data)
+        write(stream, TranscodingStreams.TOKEN_END)
+        flush(stream)
+        write(decode_sink, TranscodingStreams.TOKEN_END)
+        flush(decode_sink)
+        Test.@test take!(sink) == data
+        close(stream)
+    end
+end
+
+function test_roundtrip_transcode(encode, decode)
+    seed!(TEST_RANDOM_SEED)
+    encoder = encode()
+    initialize(encoder)
+    decoder = decode()
+    initialize(decoder)
+    for n in vcat(0:30, sort!(rand(500:100_000, 30))), alpha in (0x00:0xff, 0x00:0x0f)
+        data = rand(alpha, n)
+        Test.@test hash(transcode(decode, transcode(encode, data))) == hash(data)
+        Test.@test hash(transcode(decoder, transcode(encoder, data))) == hash(data)
+    end
+    finalize(encoder)
+    finalize(decoder)
+end
+
+function test_roundtrip_lines(encoder, decoder)
+    seed!(TEST_RANDOM_SEED)
+    lines = String[]
+    buf = IOBuffer()
+    stream = encoder(buf)
+    for i in 1:100_000
+        line = randstring(rand(0:1000))
+        println(stream, line)
+        push!(lines, line)
+    end
+    write(stream, TOKEN_END)
+    flush(stream)
+    seekstart(buf)
+    Test.@test hash(lines) == hash(readlines(decoder(buf)))
+end
+
+function test_roundtrip_seekstart(encoder, decoder)
+    seed!(TEST_RANDOM_SEED)
+    for n in vcat(0:30, sort!(rand(500:100_000, 30))), alpha in (0x00:0xff, 0x00:0x0f)
+        data = rand(alpha, n)
+        file = IOBuffer(data)
+        stream = decoder(encoder(file))
+        for m in vcat(0:min(n,20), rand(0:n, 10))
+            Test.@test read(stream, m) == @view(data[1:m])
+            seekstart(stream)
+        end
+        seekstart(stream)
+        Test.@test read(stream) == data
+        seekstart(stream)
+        Test.@test read(stream) == data
+        close(stream)
+    end
+end
+
+function test_roundtrip_fileio(Encoder, Decoder)
+    data = b"""
+    Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla sit amet tempus felis. Etiam molestie urna placerat iaculis pellentesque. Maecenas porttitor et dolor vitae posuere. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc eget nibh quam. Nullam aliquet interdum fringilla. Duis facilisis, lectus in consectetur varius, lorem sem tempor diam, nec auctor tellus nibh sit amet sapien. In ex nunc, elementum eget facilisis ut, luctus eu orci. Sed sapien urna, accumsan et elit non, auctor pretium massa. Phasellus consectetur nisi suscipit blandit aliquam. Nulla facilisi. Mauris pellentesque sem sit amet mi vestibulum eleifend. Nulla faucibus orci ac lorem efficitur, et blandit orci interdum. Aenean posuere ultrices ex sed rhoncus. Donec malesuada mollis sem, sed varius nunc sodales sed. Curabitur lobortis non justo non tristique.
+    """
+    mktemp() do filename, file
+        stream = TranscodingStream(Encoder(), file)
+        write(stream, data)
+        close(stream)
+        stream = TranscodingStream(Decoder(), open(filename))
+        Test.@test hash(read(stream)) == hash(data)
+        close(stream)
+    end
+end
+
+function test_chunked_read(Encoder, Decoder)
+    seed!(TEST_RANDOM_SEED)
+    alpha = b"色即是空"
+    encoder = Encoder()
+    initialize(encoder)
+    for sharedbuf in false:true
+        for _ in 1:500
+            chunks = [rand(alpha, rand(0:100)) for _ in 1:rand(1:100)]
+            data = mapfoldl(x->transcode(encoder, x), vcat, chunks, init=UInt8[])
+            buffer = NoopStream(IOBuffer(data))
+            ok = true
+            for chunk in chunks
+                stream = TranscodingStream(Decoder(), buffer; stop_on_end=true, sharedbuf)
+                ok &= read(stream) == chunk
+                ok &= position(stream) == length(chunk)
+                ok &= eof(stream)
+                ok &= isreadable(stream)
+                close(stream)
+            end
+            # read without stop_on_end should read the full data.
+            stream = TranscodingStream(Decoder(), IOBuffer(data))
+            ok &= read(stream) == reduce(vcat, chunks)
+            close(stream)
+            Test.@test ok
+        end
+    end
+    finalize(encoder)
+end
+
+function test_chunked_write(Encoder, Decoder)
+    seed!(TEST_RANDOM_SEED)
+    alpha = b"空即是色"
+    encoder = Encoder()
+    initialize(encoder)
+    for _ in 1:500
+        chunks = [rand(alpha, rand(0:100)) for _ in 1:2]
+        data = map(x->transcode(encoder, x), chunks)
+        buffer = IOBuffer()
+        stream = TranscodingStream(Decoder(), buffer, stop_on_end=true)
+        write(stream, vcat(data...))
+        close(stream)
+        ok = true
+        ok &= hash(take!(buffer)) == hash(vcat(chunks...))
+        Test.@test ok
+    end
+    finalize(encoder)
+end
+
+end # module TestsForCodecPackages

test/Project.toml

@@ -4,4 +4,5 @@ FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b"
 OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TestsForCodecPackages = "c2e61002-3542-480d-8b3c-5f05cc4f8554"
 TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"

test/codecdoubleframe.jl

@@ -4,15 +4,16 @@ using Test
 using TranscodingStreams:
     TranscodingStreams,
     TranscodingStream,
+    Error
+using TestsForCodecPackages:
     test_roundtrip_read,
     test_roundtrip_write,
+    test_roundtrip_transcode,
     test_roundtrip_lines,
     test_roundtrip_seekstart,
-    test_roundtrip_transcode,
     test_roundtrip_fileio,
     test_chunked_read,
-    test_chunked_write,
-    Error
+    test_chunked_write
 
 # An insane codec for testing the codec APIs.
 struct DoubleFrameEncoder <: TranscodingStreams.Codec 

test/codecnoop.jl

@@ -1,5 +1,14 @@
 using OffsetArrays: OffsetArray
 using FillArrays: Zeros
+using TestsForCodecPackages:
+    test_roundtrip_read,
+    test_roundtrip_write,
+    test_roundtrip_transcode,
+    test_roundtrip_lines,
+    test_roundtrip_seekstart,
+    test_roundtrip_fileio,
+    test_chunked_read,
+    test_chunked_write
 
 @testset "Noop Codec" begin
     source = IOBuffer("")
@@ -294,11 +303,11 @@ using FillArrays: Zeros
     data = "foo"
     @test String(transcode(Noop, data)) == data
 
-    TranscodingStreams.test_roundtrip_transcode(Noop, Noop)
-    TranscodingStreams.test_roundtrip_read(NoopStream, NoopStream)
-    TranscodingStreams.test_roundtrip_write(NoopStream, NoopStream)
-    TranscodingStreams.test_roundtrip_lines(NoopStream, NoopStream)
-    TranscodingStreams.test_roundtrip_seekstart(NoopStream, NoopStream)
+    test_roundtrip_transcode(Noop, Noop)
+    test_roundtrip_read(NoopStream, NoopStream)
+    test_roundtrip_write(NoopStream, NoopStream)
+    test_roundtrip_lines(NoopStream, NoopStream)
+    test_roundtrip_seekstart(NoopStream, NoopStream)
 
     @testset "switch write => read" begin
         stream = NoopStream(IOBuffer(collect(b"foobar"), read=true, write=true))

test/runtests.jl

@@ -1,9 +1,11 @@
 using TranscodingStreams
+using TestsForCodecPackages: TestsForCodecPackages
 using Random
 using Test
 using Aqua: Aqua
 
 Aqua.test_all(TranscodingStreams)
+Aqua.test_all(TestsForCodecPackages)
 
 @test isempty(detect_unbound_args(TranscodingStreams; recursive=true))
 @test isempty(detect_ambiguities(TranscodingStreams; recursive=true))