Merge pull request #7 from Sean1708/extensions

Extension types.

README.md

@@ -40,3 +40,93 @@ NOTE: The standard method for encoding integers in msgpack is to use the most co
 For compatibility with other implementations, I'm following this convention.  On the unpacking side, every integer type becomes an Int64 in Julia, unless it doesn't fit (ie. values greater than 2^63 are unpacked as Uint64).
 
 I might change this at some point, and/or provide a way to control the unpacked types.
+
+### The Extension Type
+
+The MsgPack spec [defines](https://github.com/msgpack/msgpack/blob/master/spec.md#formats-ext) the [extension type](https://github.com/msgpack/msgpack/blob/master/spec.md#types-extension-type) to be a tuple of `(typecode, bytearray)` where `typecode` is an application-specific identifier for the data in `bytearray`. MsgPack.jl provides support for the extension type through the `Ext` immutable.
+
+It is defined like so
+
+```julia
+immutable Ext
+    typecode::Int8
+    data::Vector{Uint8}
+end
+```
+
+and used like this
+
+```julia
+julia> a = [0x34, 0xff, 0x76, 0x22, 0xd3, 0xab]
+6-element Array{UInt8,1}:
+ 0x34
+ 0xff
+ 0x76
+ 0x22
+ 0xd3
+ 0xab
+
+julia> b = Ext(22, a)
+MsgPack.Ext(22,UInt8[0x34,0xff,0x76,0x22,0xd3,0xab])
+
+julia> p = pack(b)
+9-element Array{UInt8,1}:
+ 0xc7
+ 0x06
+ 0x16
+ 0x34
+ 0xff
+ 0x76
+ 0x22
+ 0xd3
+ 0xab
+
+julia> c = unpack(p)
+MsgPack.Ext(22,UInt8[0x34,0xff,0x76,0x22,0xd3,0xab])
+
+julia> c == b
+true
+```
+
+MsgPack reserves typecodes in the range `[-128, -1]` for future types specified by the MsgPack spec. MsgPack.jl enforces this when creating an `Ext` but if you are packing an implementation defined extension type (currently there are none) you can pass `impltype=true`.
+
+```julia
+julia> Ext(-43, Uint8[1, 5, 3, 9])
+ERROR: MsgPack Ext typecode -128 through -1 reserved by implementation
+ in call at /Users/sean/.julia/v0.4/MsgPack/src/MsgPack.jl:48 
+
+julia> Ext(-43, Uint8[1, 5, 3, 9], impltype=true)
+MsgPack.Ext(-43,UInt8[0x01,0x05,0x03,0x09])
+```
+
+#### Serialization
+
+MsgPack.jl also defines the `extserialize` and `extdeserialize` convenience functions. These functions can turn an arbitrary object into an `Ext` and vice-versa.
+
+```julia
+julia> type Point{T}
+        x::T
+        y::T
+       end
+
+julia> r = Point(2.5, 7.8)
+Point{Float64}(2.5,7.8)
+
+julia> e = MsgPack.extserialize(123, r)
+MsgPack.Ext(123,UInt8[0x11,0x01,0x02,0x05,0x50,0x6f,0x69,0x6e,0x74,0x23  …  0x40,0x0e,0x33,0x33,0x33,0x33,0x33,0x33,0x1f,0x40])
+
+julia> s = MsgPack.extdeserialize(e)
+(123,Point{Float64}(2.5,7.8))
+
+julia> s[2]
+Point{Float64}(2.5,7.8)
+
+julia> r
+Point{Float64}(2.5,7.8)
+```
+
+Since these functions use [`serialize`](http://docs.julialang.org/en/latest/stdlib/base/#Base.serialize) under the hood they are subject to the following caveat.
+
+> In general, this process will not work if the reading and writing are done by
+> different versions of Julia, or an instance of Julia with a different system
+> image.

src/MsgPack.jl

@@ -1,11 +1,12 @@
 module MsgPack
 
-export pack, unpack
+export pack, unpack, Ext
 
 const INT_FP   = 0x00 # - 0xf7
 const MAP_F    = 0x80 # - 0x8f
 const ARR_F    = 0x90 # - 0x9f
 const STR_F    = 0xa0 # - 0xbf
+const EXT_F    = 0xd4 # - 0xd8
 
 const NIL      = 0xc0
 const UNUSED   = 0xc1
@@ -37,6 +38,33 @@ const MAP_32   = 0xdf
 
 const INT_FN   = 0xe0 # - 0xff
 
+immutable Ext
+    typecode::Int8
+    data::Vector{Uint8}
+
+    function Ext(t::Integer, d::Vector{Uint8}; impltype=false)
+        # -128 to -1 reserved for implementation
+        if -128 <= t <= -1
+            impltype || error("MsgPack Ext typecode -128 through -1 reserved by implementation")
+        elseif !(0 <= t <= 127)
+            error("MsgPack Ext typecode must be in the range [-128, 127]")
+        end
+
+        new(t, d)
+    end
+end
+==(a::Ext, b::Ext) = a.typecode == b.typecode && a.data == b.data
+
+# return Ext where Ext.data is a serialized object
+function extserialize(t::Integer, d)
+    i = IOBuffer()
+    serialize(i, d)
+    return Ext(t, takebuf_array(i))
+end
+
+# return (typecode, object) from an Ext where Ext.data is a serialized object
+extdeserialize(e::Ext) = (e.typecode, deserialize(IOBuffer(e.data)))
+
 
 readn(s, t) = ntoh(read(s, t))
 readi(s, t) = int64(readn(s, t))
@@ -58,9 +86,9 @@ const DISPATCH =
      ,BIN_8    => s -> unpack_bin(s, readn(s, Uint8))
      ,BIN_16   => s -> unpack_bin(s, readn(s, Uint16))
      ,BIN_32   => s -> unpack_bin(s, readn(s, Uint32))
-     ,EXT_8    => s -> nothing
-     ,EXT_16   => s -> nothing
-     ,EXT_32   => s -> nothing
+     ,EXT_8    => s -> unpack_ext(s, readn(s, Uint8))
+     ,EXT_16   => s -> unpack_ext(s, readn(s, Uint16))
+     ,EXT_32   => s -> unpack_ext(s, readn(s, Uint32))
      ,FLOAT_32 => s -> readn(s, Float32)
      ,FLOAT_64 => s -> readn(s, Float64)
      ,UINT_8   => s -> readi(s, Uint8)
@@ -100,6 +128,10 @@ unpack(s::IO) = begin
         # fixstr
         unpack_str(s, b $ STR_F)
 
+    elseif 0xd4 <= b <= 0xd8
+        # fixext
+        unpack_ext(s, 2^(b - EXT_F))
+
     elseif b <= 0xdf
         DISPATCH[b](s)
 
@@ -128,6 +160,7 @@ unpack_arr(s, n) = begin
 end
 
 unpack_str(s, n) = utf8(readbytes(s, n))
+unpack_ext(s, n) = Ext(read(s, Int8), readbytes(s, n), impltype=true)
 unpack_bin(s, n) = readbytes(s, n)
 
 wh(io, head, v) = begin
@@ -202,6 +235,32 @@ pack(s, v::String) = begin
     write(s, v)
 end
 
+# ext format
+pack(s, v::Ext) = begin
+    n = sizeof(v.data)
+    if n == 1
+        write(s, 0xd4)
+    elseif n == 2
+        write(s, 0xd5)
+    elseif n == 4
+        write(s, 0xd6)
+    elseif n == 8
+        write(s, 0xd7)
+    elseif n == 16
+        write(s, 0xd8)
+    elseif n < 2^8
+        wh(s, 0xc7, uint8(n))
+    elseif n < 2^16
+        wh(s, 0xc8, uint16(n))
+    elseif n < 2^32
+        wh(s, 0xc9, uint32(n))
+    else
+        error("MsgPack ext overflow: ", n)
+    end
+    write(s, v.typecode)
+    write(s, v.data)
+end
+
 # bin format
 pack(s, v::Vector{Uint8}) = begin
     n = length(v)

test/runtests.jl

@@ -82,3 +82,30 @@ b = []
 @test ck_pack({1=>2, "hi"=>"mom"},
               [0x82,0xa2,0x68,0x69,0xa3,0x6d,0x6f,0x6d,0x01,0x02])
 
+# fixext 1
+@test ck_pack(Ext(4, [0xbb]), [0xd4, 0x04, 0xbb])
+# fixext 2
+@test ck_pack(Ext(-32, [0x56, 0x8d], impltype=true), [0xd5, 0xe0, 0x56, 0x8d])
+# fixext 4
+@test ck_pack(Ext(-123, [0x80, 0x7c, 0x8b, 0xf8], impltype=true),
+              [0xd6, 0x85, 0x80, 0x7c, 0x8b, 0xf8])
+# fixext 8
+@test ck_pack(Ext(111, [0x04, 0x16, 0x94, 0x13, 0x0a, 0x7d, 0x6f, 0x0c]),
+              [0xd7, 0x6f, 0x04, 0x16, 0x94, 0x13, 0x0a, 0x7d, 0x6f, 0x0c])
+# fixext 16
+@test ck_pack(Ext(79, [0x00, 0x30, 0xd5, 0x64, 0x0f, 0x8d, 0x92, 0x90,
+                       0x98, 0x99, 0x14, 0x57, 0x0e, 0x8d, 0xf1, 0x3a]),
+              [0xd8, 0x4f, 0x00, 0x30, 0xd5, 0x64, 0x0f, 0x8d, 0x92,
+               0x90, 0x98, 0x99, 0x14, 0x57, 0x0e, 0x8d, 0xf1, 0x3a])
+# no elements
+@test ck_pack(Ext(-118, Uint8[], impltype=true), [0xc7, 0x00, 0x8a])
+# ext 8
+@test ck_pack(Ext(50, [0x62, 0x4c, 0x7c, 0x0f, 0x86, 0x04]),
+              [0xc7, 0x06, 0x32, 0x62, 0x4c, 0x7c, 0x0f, 0x86, 0x04])
+# ext 16
+b = rand(Uint8, 2^14)
+@test ck_pack(Ext(78, b), vcat(0xc8, 0x40, 0x00, 0x4e, b))
+# ext 32
+b = rand(Uint8, 2^19)
+@test ck_pack(Ext(-123, b, impltype=true),
+              vcat(0xc9, 0x00, 0x08, 0x00, 0x00, 0x85, b))