init

.gitattributes

@@ -0,0 +1,3 @@
+* text=auto eol=lf
+*.{cmd,[cC][mM][dD]} text eol=crlf
+*.{bat,[bB][aA][tT]} text eol=crlf

.github/workflows/CI.yml

@@ -0,0 +1,60 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1.6'
+          - '1'
+          - '~1.11.0-0'
+        os:
+          - ubuntu-latest
+          - macOS-13 # intel
+          - macOS-14 # arm
+          - windows-latest
+        arch:
+          - 'x64'
+          - 'x86'
+          - 'aarch64'
+        exclude:
+          - os: ubuntu-latest
+            arch: aarch64
+          - os: windows-latest
+            arch: aarch64
+          - os: macOS-13
+            arch: x86
+          - os: macOS-13
+            arch: aarch64
+          - os: macOS-14
+            arch: x86
+          - os: macOS-14
+            arch: x64
+          - os: macOS-14
+            version: '1.6'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+          show-versioninfo: true
+      - uses: julia-actions/cache@v2
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }} # required
+          file: lcov.info

.github/workflows/CompatHelper.yml

@@ -0,0 +1,16 @@
+name: CompatHelper
+on:
+  schedule:
+    - cron: 0 0 * * *
+  workflow_dispatch:
+jobs:
+  CompatHelper:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Pkg.add("CompatHelper")
+        run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+      - name: CompatHelper.main()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
+        run: julia -e 'using CompatHelper; CompatHelper.main()'

.github/workflows/TagBot.yml

@@ -0,0 +1,15 @@
+name: TagBot
+on:
+  issue_comment:
+    types:
+      - created
+  workflow_dispatch:
+jobs:
+  TagBot:
+    if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ssh: ${{ secrets.DOCUMENTER_KEY }}

.gitignore

@@ -0,0 +1,12 @@
+#Mac OS X
+*.DS_Store
+
+#VS Code
+/.vscode/
+
+/test/Manifest.toml
+/Manifest.toml
+fixture.tar.gz
+fixture
+.CondaPkg
+*.cov

Project.toml

@@ -0,0 +1,11 @@
+name = "CodecInflate64"
+uuid = "6309b1aa-fc58-479c-8956-599a07234577"
+authors = ["nhz2 <[email protected]>"]
+version = "0.1.0"
+
+[deps]
+TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
+
+[compat]
+TranscodingStreams = "0.9, 0.10, 0.11"
+julia = "1.6"

README.md

@@ -1,2 +1,32 @@
-# CodecInflate64.jl
-(WIP) Julia implementation of deflate64 decompression
+# (WIP) CodecInflate64.jl
+
+[![CI](https://github.com/nhz2/CodecInflate64.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/nhz2/CodecInflate64.jl/actions/workflows/CI.yml)
+[![codecov](https://codecov.io/gh/nhz2/CodecInflate64.jl/branch/main/graph/badge.svg?token=K3J0T9BZ42)](https://codecov.io/gh/nhz2/CodecInflate64.jl)
+[![Aqua QA](https://raw.githubusercontent.com/JuliaTesting/Aqua.jl/master/badge.svg)](https://github.com/JuliaTesting/Aqua.jl)
+
+CodecInflate64 implements deflate64 decompression for the [TranscodingStream.jl](https://github.com/JuliaIO/TranscodingStreams.jl) interface.
+
+This library's goal is reading entries of ZIP files created by the default Windows archiver.
+
+Deflate64 is a incompatible variant of deflate that Windows sometimes uses when making ZIP files.
+
+This package is a work in progress, and may not be able to read all deflate64 data correctly.
+
+Check all decompressed data with a checksum.
+
+The deflate algorithm is described in [RFC 1951](https://www.ietf.org/rfc/rfc1951.txt).
+
+Deflate64 has a reference implementation in [dotnet](https://github.com/dotnet/runtime/tree/e5efd8010e19593298dc2c3ee15106d5aec5a924/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateManaged)
+
+It is also describe unofficially in https://libzip.org/specifications/appnote_iz.txt
+
+Some of the code from [Inflate.jl](https://github.com/GunnarFarneback/Inflate.jl) is used here, but modified to work with deflate64.
+
+This package exports following codecs and streams:
+
+| Codec                   | Stream                        |
+| ----------------------- | ----------------------------- |
+| `DeflateDecompressor`   | `DeflateDecompressorStream`   |
+| `Deflate64Decompressor` | `Deflate64DecompressorStream` |
+
+See docstrings and [TranscodingStreams.jl](https://github.com/bicycle1885/TranscodingStreams.jl) for details.

src/CodecInflate64.jl

@@ -0,0 +1,12 @@
+module CodecInflate64
+
+export DeflateDecompressor
+export DeflateDecompressorStream
+export Deflate64Decompressor
+export Deflate64DecompressorStream
+
+include("huffmantree.jl")
+include("stream.jl")
+include("codecs.jl")
+
+end

src/codecs.jl

@@ -0,0 +1,76 @@
+# Glue code for TranscodingStreams
+
+abstract type DecompressorCodec <: TranscodingStreams.Codec end
+
+"""
+    DeflateDecompressor()
+
+Create a deflate decompression codec.
+"""
+struct DeflateDecompressor <: DecompressorCodec
+    s::StreamState
+end
+DeflateDecompressor() = DeflateDecompressor(StreamState(deflate64=false))
+
+const DeflateDecompressorStream{S} = TranscodingStream{DeflateDecompressor,S} where S<:IO
+
+"""
+    DeflateDecompressorStream(stream::IO; kwargs...)
+
+Create a deflate decompression stream.
+"""
+DeflateDecompressorStream(stream::IO; kwargs...) = TranscodingStream(DeflateDecompressor(), stream; kwargs...)
+
+"""
+    Deflate64Decompressor()
+
+Create a deflate64 decompression codec.
+"""
+struct Deflate64Decompressor <: DecompressorCodec
+    s::StreamState
+end
+Deflate64Decompressor() = Deflate64Decompressor(StreamState(deflate64=true))
+
+const Deflate64DecompressorStream{S} = TranscodingStream{Deflate64Decompressor,S} where S<:IO
+
+"""
+    Deflate64DecompressorStream(stream::IO; kwargs...)
+
+Create a deflate64 decompression stream.
+"""
+Deflate64DecompressorStream(stream::IO; kwargs...) = TranscodingStream(Deflate64Decompressor(), stream; kwargs...)
+
+function Base.show(io::IO, codec::DecompressorCodec)
+    print(io, summary(codec), "()")
+end
+
+function TranscodingStreams.startproc(codec::DecompressorCodec, ::Symbol, error_ref::TranscodingStreams.Error)
+    reset!(codec.s)
+    :ok
+end
+
+function TranscodingStreams.process(
+        codec     :: DecompressorCodec,
+        input     :: TranscodingStreams.Memory,
+        output    :: TranscodingStreams.Memory,
+        error_ref :: TranscodingStreams.Error,
+    )
+    all_in = iszero(input.size)
+    # @show "proccall" input.size output.size
+    # done, in2, out2 = main_run!(all_in, input, output, codec.s) 
+    done, in2, out2 = try
+        main_run!(all_in, input, output, codec.s)
+    catch e
+        # rethrow()
+        e isa InterruptException && rethrow()
+        error_ref[] = e
+        return 0, 0, :error
+    end
+    Δin::Int = input.size - in2.size
+    Δout::Int = output.size - out2.size
+    if done
+        return Δin, Δout, :end
+    else
+        return Δin, Δout, :ok
+    end
+end

src/huffmantree.jl

@@ -0,0 +1,67 @@
+struct HuffmanTree
+    sorted_ops::Vector{UInt16}
+    num_ops_per_num_bit::Vector{UInt16}
+    op_offset_per_num_bit::Vector{UInt16}
+end
+function HuffmanTree(num_ops::Integer, max_num_bits::Integer)
+    HuffmanTree(fill(0xFFFF,num_ops), zeros(UInt16,max_num_bits), zeros(UInt16,max_num_bits+1))
+end
+
+function reset!(tree::HuffmanTree)
+    tree.sorted_ops .= 0xFFFF
+    tree.num_ops_per_num_bit .= 0x0000
+    tree.op_offset_per_num_bit .= 0x0000
+end
+
+# Non allocating version of https://github.com/GunnarFarneback/Inflate.jl/blob/cc77be73388f4160d187ab0c3fdaa3df13aa7f3b/src/Inflate.jl#L174-L186
+function parse_huffman!(
+        tree::HuffmanTree,
+        num_bits_per_op::AbstractVector{UInt8}, # in
+    )
+    # @show length(num_bits_per_op)
+    # TODO Validate produced tree
+    sorted_ops = tree.sorted_ops
+    num_ops_per_num_bit = tree.num_ops_per_num_bit
+    op_offset_per_num_bit = tree.op_offset_per_num_bit
+    num_ops_per_num_bit .= 0x0000
+    sorted_ops .= 0xFFFF
+    op_offset_per_num_bit .= 0x0000
+    max_num_bits = length(num_ops_per_num_bit)
+    @assert max_num_bits ≥ maximum(num_bits_per_op)
+    @assert length(op_offset_per_num_bit) == max_num_bits + 1
+    @assert length(sorted_ops) ≥ length(num_bits_per_op)
+    for n in num_bits_per_op
+        if !iszero(n)
+            num_ops_per_num_bit[n] += 1
+        end
+    end
+    op_offset_per_num_bit[1] = 1
+    op_offset_per_num_bit[2] = 1
+    for n in 2:max_num_bits
+        op_offset_per_num_bit[n+1] = op_offset_per_num_bit[n+1-1] + num_ops_per_num_bit[n-1]
+    end
+    # display(op_offset_per_num_bit)
+    for (c, n) in enumerate(num_bits_per_op)
+        # @show c, n
+        if !iszero(n)
+            off = op_offset_per_num_bit[n+1]
+            sorted_ops[off] = c-1
+            op_offset_per_num_bit[n+1] = off + 1
+        end
+    end
+end
+
+# Using algorithm from https://github.com/GunnarFarneback/Inflate.jl/blob/cc77be73388f4160d187ab0c3fdaa3df13aa7f3b/src/Inflate.jl#L134-L145
+function get_op(bits::UInt16, tree::HuffmanTree)::Tuple{UInt16, UInt8}
+    v = 0x0000
+    for nbits in 1:length(tree.num_ops_per_num_bit)
+        bit = bits & 0x01
+        bits >>= 1
+        v = (v << 1) | bit
+        if v < tree.num_ops_per_num_bit[nbits]
+            return tree.sorted_ops[tree.op_offset_per_num_bit[nbits]+v], UInt8(nbits)
+        end
+        v -= tree.num_ops_per_num_bit[nbits]
+    end
+    error("incomplete code table")
+end