[EXTREMELY WIP] fsspec compatible storage v2

src/Filters/Filters.jl

@@ -86,4 +86,5 @@ include("vlenfilters.jl")
 include("fletcher32.jl")
 include("fixedscaleoffset.jl")
 include("shuffle.jl")
-include("quantize.jl")
+include("quantize.jl")
+include("delta.jl")

src/Filters/delta.jl

@@ -0,0 +1,45 @@
+#=
+# Delta compression
+
+
+=#
+
+"""
+    DeltaFilter(; DecodingType, [EncodingType = DecodingType])
+
+Delta-based compression for Zarr arrays.  (Delta encoding is Julia `diff`, decoding is Julia `cumsum`).
+"""
+struct DeltaFilter{T, TENC} <: Filter{T, TENC}
+end
+
+function DeltaFilter(; DecodingType = Float16, EncodingType = DecodingType)
+    return DeltaFilter{DecodingType, EncodingType}()
+end
+
+DeltaFilter{T}() where T = DeltaFilter{T, T}()
+
+function zencode(data::AbstractArray, filter::DeltaFilter{DecodingType, EncodingType}) where {DecodingType, EncodingType}
+    arr = reinterpret(DecodingType, vec(data))
+
+    enc = similar(arr, EncodingType)
+    # perform the delta operation
+    enc[begin] = arr[begin]
+    enc[begin+1:end] .= diff(arr)
+    return enc
+end
+
+function zdecode(data::AbstractArray, filter::DeltaFilter{DecodingType, EncodingType}) where {DecodingType, EncodingType}
+    encoded = reinterpret(EncodingType, vec(data))
+    decoded = DecodingType.(cumsum(encoded))
+    return decoded
+end
+
+function JSON.lower(filter::DeltaFilter{T, Tenc}) where {T, Tenc}
+    return Dict("type" => "delta", "dtype" => typestring(T), "atype" => typestring(Tenc))
+end
+
+function getfilter(::Type{<: DeltaFilter}, d)
+    return DeltaFilter{typestr(d["dtype"], haskey(d, "atype") ? typestr(d["atype"]) : d["dtype"])}()
+end
+
+filterdict["delta"] = DeltaFilter

src/Filters/fixedscaleoffset.jl

@@ -27,7 +27,7 @@ function zdecode(a::AbstractArray, c::FixedScaleOffsetFilter{ScaleOffsetType, T,
 end
 
 
-function getFilter(::Type{<: FixedScaleOffsetFilter}, d::Dict)
+function getfilter(::Type{<: FixedScaleOffsetFilter}, d::Dict)
     scale = d["scale"]
     offset = d["offset"]
     # Types must be converted from strings to the actual Julia types they represent.

src/Filters/fletcher32.jl

@@ -19,7 +19,7 @@ the checksum and cropping the last 4 bytes of the data during decoding.
 struct Fletcher32Filter <: Filter{UInt8, UInt8}
 end
 
-getFilter(::Type{<: Fletcher32Filter}, d::Dict) = Fletcher32Filter()
+getfilter(::Type{<: Fletcher32Filter}, d::Dict) = Fletcher32Filter()
 JSON.lower(::Fletcher32Filter) = Dict("id" => "fletcher32")
 filterdict["fletcher32"] = Fletcher32Filter
 

src/Filters/quantize.jl

@@ -49,7 +49,7 @@ function JSON.lower(filter::QuantizeFilter{T, Tenc}) where {T, Tenc}
     return Dict("type" => "quantize", "digits" => filter.digits, "dtype" => typestring(T), "atype" => typestring(Tenc))
 end
 
-function getFilter(::Type{<: QuantizeFilter}, d)
+function getfilter(::Type{<: QuantizeFilter}, d)
     return QuantizeFilter{typestr(d["dtype"], typestr(d["atype"]))}(; digits = d["digits"])
 end
 

src/Filters/shuffle.jl

@@ -52,7 +52,7 @@ function zdecode(a::AbstractArray, c::ShuffleFilter)
     return dest
 end
 
-function getFilter(::Type{ShuffleFilter}, d::Dict)
+function getfilter(::Type{ShuffleFilter}, d::Dict)
     return ShuffleFilter(d["elementsize"])
 end
 

test/Filters.jl

@@ -2,7 +2,7 @@ using Test
 using Zarr: DateTime64 # for datetime reinterpret
 
 using Zarr: zencode, zdecode
-using Zarr: Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter
+using Zarr: Fletcher32Filter, FixedScaleOffsetFilter, ShuffleFilter, QuantizeFilter, DeltaFilter
 
 @testset "Fletcher32Filter" begin
     # These tests are copied exactly from the [`numcodecs`](https://github.com/zarr-developers/numcodecs/) Python package,
@@ -133,4 +133,31 @@ end
             end
         end
     end
+end
+
+@testset "DeltaFilter" begin
+
+    arrays = [
+        Int32.(collect(0:999)),  # np.arange(1000, dtype='<i4')
+        Float32.(reshape(LinRange(1000, 1001, 1000), (100, 10))),  # np.linspace(1000, 1001, 1000, dtype='<f4').reshape(100, 10)
+        Float64.(reshape(randn(1000) .* 1 .+ 1000, (10, 10, 10))),  # np.random.normal(loc=1000, scale=1, size=(10, 10, 10)).astype('<f8')
+        UInt16.(permutedims(reshape(rand(UInt16, 1000) .% 200, (100, 10))))  # np.random.randint(0, 200, size=1000, dtype='u2').astype('<u2').reshape(100, 10, order='F')
+    ]
+
+    for array in arrays
+        encoded = Zarr.zencode(array, DeltaFilter{eltype(array)}())
+        decoded = Zarr.zdecode(encoded, DeltaFilter{eltype(array)}())
+        @test decoded == array
+    end
+
+    @testset "DeltaFilter with different dtypes" begin
+        dtype = Int64
+        astype = Int32
+        codec = Zarr.DeltaFilter{dtype, astype}()
+        arr = collect(Int64, 10:19)
+        expect = Int32[10; fill(1, 9)]
+        actual = Zarr.zencode(arr, codec)
+        @test Int64.(expect) == Int64.(actual)
+        @test eltype(actual) == astype
+    end
 end