Update alpha

CHANGELOG.md

@@ -30,8 +30,6 @@ The `Unreleased` section name is replaced by the expected version of next releas
 
 - `eqx stats`: `-A` (all stats) is now the default unless you specify >=1 of the individual stats via `ESDNO` flags [#459](https://github.com/jet/equinox/pull/459)
 
-### Fixed
-
 <a name="4.0.4"></a>
 ## [4.0.4] - 2024-05-08
 
@@ -109,7 +107,7 @@ The `Unreleased` section name is replaced by the expected version of next releas
 - `CosmosStore`: Only log `bytes` when log level is `Debug` [#305](https://github.com/jet/equinox/pull/305)
 - `CosmosStore.AccessStrategy.MultiSnapshot`,`Custom`: Change `list` and `seq` types to `array` [#338](https://github.com/jet/equinox/pull/338)
 - `CosmosStore.CosmosStoreCategory`: Generalize `compressUnfolds` to `shouldCompress` predicate [#436](https://github.com/jet/equinox/pull/436)
-- `CosmosStore.CosmosStoreCategory.TryHydrateTip`: Generates a Stream State Momento based on externally loaded `u`nfold state [#434](https://github.com/jet/equinox/pull/434)
+- `CosmosStore.CosmosStoreCategory.TryHydrateTip`: Generates a Stream State Memento based on externally loaded `u`nfold state [#434](https://github.com/jet/equinox/pull/434)
 - `CosmosStore.CosmosStoreCategory.TryLoad`: Renders a `'state` based on an Unfold [#434](https://github.com/jet/equinox/pull/434)
 - `CosmosStore.Core.Initialization.initAux`: Replace hard-coded manual 400 RU with `mode` parameter [#328](https://github.com/jet/equinox/pull/328) :pray: [@brihadish](https://github.com/brihadish)
 - `CosmosStore.CosmosClientFactory`: Moved to Core [#430](https://github.com/jet/equinox/pull/430)

README.md

@@ -170,7 +170,7 @@ The components within this repository are delivered as multi-targeted Nuget pack
 
 - `Equinox.Core` [![NuGet](https://img.shields.io/nuget/v/Equinox.Core.svg)](https://www.nuget.org/packages/Equinox.Core/): Hosts generic utility types frequently useful alongside Equinox: [`TaskCell`](https://github.com/jet/equinox/blob/master/src/Equinox.Core/TaskCell.fs#L36), [`Batcher`, `BatcherCache`, `BatcherDictionary`](https://github.com/jet/equinox/blob/master/src/Equinox.Core/Batching.fs#L44). ([depends](https://www.fuget.org/packages/Equinox.Core) on `System.Runtime.Caching`)
 - `Equinox.MemoryStore` [![MemoryStore NuGet](https://img.shields.io/nuget/v/Equinox.MemoryStore.svg)](https://www.nuget.org/packages/Equinox.MemoryStore/): In-memory store for integration testing/performance base-lining/providing out-of-the-box zero dependency storage for examples. ([depends](https://www.fuget.org/packages/Equinox.MemoryStore) on `Equinox`)
-- `Equinox.CosmosStore` [![CosmosStore NuGet](https://img.shields.io/nuget/v/Equinox.CosmosStore.svg)](https://www.nuget.org/packages/Equinox.CosmosStore/): Azure CosmosDB Adapter with integrated 'unfolds' feature, facilitating optimal read performance in terms of latency and RU costs, instrumented to meet Jet's production monitoring requirements. ([depends](https://www.fuget.org/packages/Equinox.CosmosStore) on `Equinox`, `Equinox`, `Microsoft.Azure.Cosmos` >= `3.43.0`, `System.Text.Json`, `FSharp.Control.TaskSeq`)
+- `Equinox.CosmosStore` [![CosmosStore NuGet](https://img.shields.io/nuget/v/Equinox.CosmosStore.svg)](https://www.nuget.org/packages/Equinox.CosmosStore/): Azure CosmosDB Adapter with integrated 'unfolds' feature, facilitating optimal read performance in terms of latency and RU costs, instrumented to meet Jet's production monitoring requirements. ([depends](https://www.fuget.org/packages/Equinox.CosmosStore) on `Equinox`, `Equinox`, `Microsoft.Azure.Cosmos` >= `3.43.1`, `System.Text.Json`, `FSharp.Control.TaskSeq`)
 - `Equinox.CosmosStore.Prometheus` [![CosmosStore.Prometheus NuGet](https://img.shields.io/nuget/v/Equinox.CosmosStore.Prometheus.svg)](https://www.nuget.org/packages/Equinox.CosmosStore.Prometheus/): Integration package providing a `Serilog.Core.ILogEventSink` that extracts detailed metrics information attached to the `LogEvent`s and feeds them to the `prometheus-net`'s `Prometheus.Metrics` static instance. ([depends](https://www.fuget.org/packages/Equinox.CosmosStore.Prometheus) on `Equinox.CosmosStore`, `prometheus-net >= 3.6.0`)
 - `Equinox.DynamoStore` [![DynamoStore NuGet](https://img.shields.io/nuget/v/Equinox.DynamoStore.svg)](https://www.nuget.org/packages/Equinox.DynamoStore/): Amazon DynamoDB Adapter with integrated 'unfolds' feature, facilitating optimal read performance in terms of latency and RC costs, patterned after `Equinox.CosmosStore`. ([depends](https://www.fuget.org/packages/Equinox.DynamoStore) on `Equinox`, `FSharp.AWS.DynamoDB` >= `0.12.0-beta`, `FSharp.Control.TaskSeq`)
 - `Equinox.DynamoStore.Prometheus` [![DynamoStore.Prometheus NuGet](https://img.shields.io/nuget/v/Equinox.DynamoStore.Prometheus.svg)](https://www.nuget.org/packages/Equinox.DynamoStore.Prometheus/): Integration package providing a `Serilog.Core.ILogEventSink` that extracts detailed metrics information attached to the `LogEvent`s and feeds them to the `prometheus-net`'s `Prometheus.Metrics` static instance. ([depends](https://www.fuget.org/packages/Equinox.CosmosStore.Prometheus) on `Equinox.DynamoStore`, `prometheus-net >= 3.6.0`)

samples/Infrastructure/Services.fs

@@ -17,7 +17,7 @@ type Store(store) =
             MemoryStore.MemoryStoreCategory(store, name, codec, fold, initial)
         | Store.Config.Cosmos (store, caching, unfolds) ->
             let accessStrategy = if unfolds then CosmosStore.AccessStrategy.Snapshot snapshot else CosmosStore.AccessStrategy.Unoptimized
-            CosmosStore.CosmosStoreCategory<'event,'state,_>(store, name, FsCodec.SystemTextJson.Compression.EncodeTryCompress codec, fold, initial, accessStrategy, caching)
+            CosmosStore.CosmosStoreCategory<'event,'state,_>(store, name, FsCodec.SystemTextJson.EncodedBody.EncodeTryCompress codec, fold, initial, accessStrategy, caching)
         | Store.Config.Dynamo (store, caching, unfolds) ->
             let accessStrategy = if unfolds then DynamoStore.AccessStrategy.Snapshot snapshot else DynamoStore.AccessStrategy.Unoptimized
             DynamoStore.DynamoStoreCategory<'event,'state,_>(store, name, FsCodec.Compression.EncodeTryCompress codec, fold, initial, accessStrategy, caching)

samples/Infrastructure/Store.fs

@@ -71,7 +71,7 @@ module Cosmos =
                 | ArchiveContainer _ -> "specify a container name for Archive store. Default: use same as Primary Container"
                 | TipMaxEvents _ ->     "specify maximum number of events to hold in Tip before calving off to a frozen Batch. Default: 256"
                 | TipMaxJsonLength _ -> "specify maximum length of JSON (as measured by JSON.stringify) to hold in Tip before calving off to a frozen Batch. Default: 30,000"
-                | QueryMaxItems _ ->    "specify maximum number of batches of events to retrieve in per query response. Default: 10"
+                | QueryMaxItems _ ->    "specify maximum number of batches of events to retrieve in per query response. Default: 10/9999"
     type Arguments(p : ParseResults<Parameters>) =
         member val Verbose =            p.Contains StoreVerbose
         member val Mode =               p.TryGetResult ConnectionMode
@@ -90,7 +90,8 @@ module Cosmos =
         member _.MaxRetryWaitTime =     p.GetResult(RetriesWaitTimeS, 5.) |> TimeSpan.FromSeconds
         member _.TipMaxEvents =         p.GetResult(TipMaxEvents, 256)
         member _.TipMaxJsonLength =     p.GetResult(TipMaxJsonLength, 30_000)
-        member x.QueryMaxItems =        p.GetResult(QueryMaxItems, 10)
+        member _.QueryMaxItemsOr(def: int) = p.GetResult(QueryMaxItems, def)
+        member x.QueryMaxItems =        x.QueryMaxItemsOr 10
 
     let logContainer (log: ILogger) role (mode, endpoint, db, container) =
         log.Information("CosmosDB {role:l} {mode} {connection} Database {database} Container {container}",

samples/Store/Domain/Domain.fsproj

@@ -19,7 +19,7 @@
     <PackageReference Include="FSharp.Core" Version="6.0.7" ExcludeAssets="contentfiles" />
 
     <PackageReference Include="FsCodec.NewtonsoftJson" Version="3.0.0" />
-    <PackageReference Include="FsCodec.SystemTextJson" Version="3.0.4-alpha.0.2" />
+    <PackageReference Include="FsCodec.SystemTextJson" Version="3.0.4-alpha.0.5" />
 
     <ProjectReference Include="..\..\..\src\Equinox\Equinox.fsproj" />
   </ItemGroup>

samples/Store/Domain/Infrastructure.fs

@@ -92,7 +92,7 @@ module EventCodec =
 
     /// For CosmosStore - we encode to JsonElement as that's what the store talks
     let genJsonElement<'t when 't :> TypeShape.UnionContract.IUnionContract> =
-        FsCodec.SystemTextJson.CodecJsonElement.Create<'t>() |> FsCodec.SystemTextJson.Compression.EncodeUncompressed
+        FsCodec.SystemTextJson.CodecJsonElement.Create<'t>() |> FsCodec.SystemTextJson.EncodedBody.EncodeUncompressed
 
     /// For stores other than CosmosStore, we encode to UTF-8 and have the store do the right thing
     let gen<'t when 't :> TypeShape.UnionContract.IUnionContract> =

samples/Tutorial/Infrastructure.fs

@@ -23,7 +23,7 @@ module EventCodec =
 
     /// For CosmosStore - we encode to JsonElement as that's what the store talks
     let genJsonElement<'t when 't :> TypeShape.UnionContract.IUnionContract> =
-        FsCodec.SystemTextJson.CodecJsonElement.Create<'t>() |> FsCodec.SystemTextJson.Compression.EncodeUncompressed
+        FsCodec.SystemTextJson.CodecJsonElement.Create<'t>() |> FsCodec.SystemTextJson.EncodedBody.EncodeUncompressed
 
     /// For stores other than CosmosStore, we encode to UTF-8 and have the store do the right thing
     let gen<'t when 't :> TypeShape.UnionContract.IUnionContract> =

samples/Tutorial/Tutorial.fsproj

@@ -28,7 +28,7 @@
 
   <ItemGroup>
     <PackageReference Include="MinVer" Version="5.0.0" PrivateAssets="All" />
-    <PackageReference Include="FsCodec.SystemTextJson" Version="3.0.4-alpha.0.2" />
+    <PackageReference Include="FsCodec.SystemTextJson" Version="3.0.4-alpha.0.5" />
     <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
     <PackageReference Include="Serilog.Sinks.Seq" Version="7.0.0" />
   </ItemGroup>

src/Equinox.CosmosStore/CosmosStore.fs

@@ -17,6 +17,10 @@ type EncodedBody = (struct (int * JsonElement))
 module internal EncodedBody =
     let internal jsonRawText: EncodedBody -> string = ValueTuple.snd >> _.GetRawText()
     let internal jsonUtf8Bytes = jsonRawText >> System.Text.Encoding.UTF8.GetByteCount
+    let [<Literal>] deflateEncoding = 1
+    // prior to the addition of the `D` field in 4.1.0, the integrated compression support
+    // was predicated entirely on a JSON String `d` value in the Unfold as implying it was UTF8->Deflate->Base64 encoded
+    let parseUnfold = function struct (0, e: JsonElement) when e.ValueKind = JsonValueKind.String -> struct (deflateEncoding, e) | x -> x
 
 /// A single Domain Event from the array held in a Batch
 [<NoEquality; NoComparison>]
@@ -28,12 +32,14 @@ type Event =
         c: string
 
         /// Event body
+        [<Serialization.JsonIgnore(Condition = Serialization.JsonIgnoreCondition.WhenWritingDefault)>]
         d: JsonElement // Can be Json Null for Nullary cases
         /// The encoding scheme used for `d`
         [<Serialization.JsonIgnore(Condition = Serialization.JsonIgnoreCondition.WhenWritingDefault)>]
         D: int
 
         /// Optional metadata
+        [<Serialization.JsonIgnore(Condition = Serialization.JsonIgnoreCondition.WhenWritingDefault)>]
         m: JsonElement
         /// The encoding scheme used for `m`
         [<Serialization.JsonIgnore(Condition = Serialization.JsonIgnoreCondition.WhenWritingDefault)>]
@@ -89,12 +95,14 @@ type Unfold =
         /// The Case (Event Type) of this snapshot, used to drive deserialization
         c: string // required
 
+        [<Serialization.JsonIgnore(Condition = Serialization.JsonIgnoreCondition.WhenWritingDefault)>]
         d: JsonElement // required
         /// The encoding scheme used for `d`
         [<Serialization.JsonIgnore(Condition = Serialization.JsonIgnoreCondition.WhenWritingDefault)>]
         D: int
 
         /// Optional metadata, same encoding as `d` (can be null; not written if missing)
+        [<Serialization.JsonIgnore(Condition = Serialization.JsonIgnoreCondition.WhenWritingDefault)>]
         m: JsonElement
         /// The encoding scheme used for `m`
         [<Serialization.JsonIgnore(Condition = Serialization.JsonIgnoreCondition.WhenWritingDefault)>]
@@ -169,7 +177,7 @@ type internal Enum private () =
     static member internal Events(b: Batch, ?minIndex, ?maxIndex) =
         Enum.Events(b.i, b.e, ?minIndex = minIndex, ?maxIndex = maxIndex)
     static member Unfolds(xs: Unfold[]): ITimelineEvent<EncodedBody> seq = seq {
-        for x in xs -> FsCodec.Core.TimelineEvent.Create(x.i, x.c, (x.D, x.d), (x.M, x.m), Guid.Empty, null, null, x.t, isUnfold = true) }
+        for x in xs -> FsCodec.Core.TimelineEvent.Create(x.i, x.c, EncodedBody.parseUnfold (x.D, x.d), (x.M, x.m), Guid.Empty, null, null, x.t, isUnfold = true) }
     static member EventsAndUnfolds(x: Tip, ?minIndex, ?maxIndex): ITimelineEvent<EncodedBody> seq =
         Enum.Events(x, ?minIndex = minIndex, ?maxIndex = maxIndex)
         |> Seq.append (Enum.Unfolds x.u)

tests/Equinox.CosmosStore.Integration/AccessStrategies.fs

@@ -44,7 +44,7 @@ module SequenceCheck =
 #if STORE_DYNAMO
         let codec = FsCodec.SystemTextJson.Codec.Create<Event>() |> FsCodec.Compression.EncodeTryCompress
 #else
-        let codec = FsCodec.SystemTextJson.CodecJsonElement.Create<Event>() |> FsCodec.SystemTextJson.Compression.EncodeTryCompress
+        let codec = FsCodec.SystemTextJson.CodecJsonElement.Create<Event>() |> FsCodec.SystemTextJson.EncodedBody.EncodeTryCompress
 #endif
 
     module Fold =

tests/Equinox.CosmosStore.Integration/CosmosCoreIntegration.fs

@@ -10,7 +10,7 @@ open System
 
 type TestEvents() =
     static member private Create(i, ?eventType, ?json) =
-        let enc = System.Text.Json.JsonSerializer.SerializeToElement >> FsCodec.SystemTextJson.Compression.Encode
+        let enc = System.Text.Json.JsonSerializer.SerializeToElement >> FsCodec.SystemTextJson.EncodedBody.Uncompressed
         FsCodec.Core.EventData.Create
             (   sprintf "%s:%d" (defaultArg eventType "test_event") i,
                 enc (defaultArg json "{\"d\":\"d\"}"),
@@ -65,7 +65,7 @@ type Tests(testOutputHelper) =
         test <@ match res with Choice2Of2 (:? InvalidOperationException as ex) -> ex.Message.StartsWith "Must write either events or unfolds." | x -> failwith $"%A{x}" @>
     }
 
-    let stringOfEncodedBody (x: Equinox.CosmosStore.Core.EncodedBody) = FsCodec.SystemTextJson.Compression.DecodeToJsonElement(x).GetRawText()
+    let stringOfEncodedBody (x: Equinox.CosmosStore.Core.EncodedBody) = FsCodec.SystemTextJson.EncodedBody.ToJsonElement(x).GetRawText()
     let jsonDiff (x: string) (y: string) =
         match JsonDiffPatchDotNet.JsonDiffPatch().Diff(JToken.Parse x, JToken.Parse y) with
         | null -> ""

tests/Equinox.CosmosStore.Integration/DocumentStoreIntegration.fs

@@ -29,8 +29,9 @@ module Cart =
         StoreCategory(context, Cart.CategoryName, codec, fold, initial, AccessStrategy.MultiSnapshot unfArgs, Equinox.CachingStrategy.NoCaching)
         |> Equinox.Decider.forStream log
         |> Cart.create
+    let createCategorySnapshot context = StoreCategory(context, Cart.CategoryName, codec, fold, initial, AccessStrategy.Snapshot Cart.Fold.Snapshot.config, Equinox.CachingStrategy.NoCaching)
     let createServiceWithSnapshotStrategy log context =
-        StoreCategory(context, Cart.CategoryName, codec, fold, initial, AccessStrategy.Snapshot Cart.Fold.Snapshot.config, Equinox.CachingStrategy.NoCaching)
+        createCategorySnapshot context
         |> Equinox.Decider.forStream log
         |> Cart.create
     let createServiceWithSnapshotStrategyAndCaching log context cache =