From 6c8ebe2e623002835c8b0090deb82222f64e8617 Mon Sep 17 00:00:00 2001 From: Samer Boshra Date: Mon, 18 Nov 2024 13:41:22 -0800 Subject: [PATCH] [Internal] JSON Binary Encoding: Adds support for writing unsigned 64-bit integer values (#4883) ## Description This PR adds support for serializing unsigned Int64 values in Binary encoding, ensuring that values exceeding the maximum signed Int64 limit are preserved without conversion to floating-point doubles. This enhancement enables full-fidelity roundtrips between Text and Binary encoding formats. For values outside the signed Int64 range, retrieval from either the JSON Reader or Navigator (across all encoding formats) will return them as floating-point doubles. ## Type of change - [x] New feature (non-breaking change which adds functionality) ## Closing issues None --- .../src/CosmosElements/CosmosElement.cs | 2 +- .../CosmosNumber64.LazyCosmosNumber64.cs | 6 +- .../CosmosElements/Numbers/CosmosNumber64.cs | 2 +- .../src/Json/IJsonNavigator.cs | 10 +- .../src/Json/IJsonWriter.cs | 8 +- .../Interop/CosmosDBToNewtonsoftWriter.cs | 4 +- .../Interop/NewtonsoftToCosmosDBReader.cs | 7 + .../Interop/NewtonsoftToCosmosDBWriter.cs | 7 +- .../src/Json/JsonBinaryEncoding.NodeTypes.cs | 2 +- .../src/Json/JsonBinaryEncoding.Numbers.cs | 24 ++ .../src/Json/JsonBinaryEncoding.Strings.cs | 14 +- .../src/Json/JsonBinaryEncoding.TypeMarker.cs | 20 +- .../Json/JsonNavigator.JsonBinaryNavigator.cs | 30 ++- .../Json/JsonNavigator.JsonTextNavigator.cs | 23 +- .../src/Json/JsonNavigator.cs | 56 +++-- .../src/Json/JsonNodeType.cs | 2 +- .../src/Json/JsonReader.JsonBinaryReader.cs | 27 ++- .../src/Json/JsonReader.JsonTextReader.cs | 8 + Microsoft.Azure.Cosmos/src/Json/JsonReader.cs | 18 +- .../src/Json/JsonSerializer.cs | 2 +- .../src/Json/JsonTextParser.cs | 5 + .../src/Json/JsonWriteOptions.cs | 2 +- .../src/Json/JsonWriter.JsonBinaryWriter.cs | 137 +++++++---- .../src/Json/JsonWriter.JsonTextWriter.cs | 27 ++- Microsoft.Azure.Cosmos/src/Json/JsonWriter.cs | 20 +- .../src/Serializer/CosmosElementSerializer.cs | 2 +- .../ClientConfigurationTraceDatum.cs | 6 +- .../Tracing/TraceData/SummaryDiagnostics.cs | 6 +- .../Tracing/TraceWriter.TraceJsonWriter.cs | 34 +-- .../Query/CosmosUndefinedQueryTests.cs | 4 +- .../Query/DistributedQueryClientTests.cs | 4 +- .../Json/JsonMicroBenchmarksBase.cs | 4 +- .../Json/Utils.cs | 2 +- .../CosmosElements/LazyCosmosElementTests.cs | 2 +- .../Json/DeserializerTests.cs | 16 +- .../Json/JsonNavigatorTests.cs | 2 +- .../Json/JsonNewtonsoftNavigator.cs | 41 ++-- .../Json/JsonReaderTests.cs | 82 +++---- .../Json/JsonRoundtripTests.cs | 24 +- .../Json/JsonTestUtils.cs | 47 +++- .../Json/JsonTokenInfo.cs | 36 ++- .../Json/JsonWriterTests.cs | 226 +++++++++++++++++- 42 files changed, 760 insertions(+), 241 deletions(-) diff --git a/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosElement.cs b/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosElement.cs index 24d1fa78d5..b06f7a6823 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosElement.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosElements/CosmosElement.cs @@ -227,7 +227,7 @@ public static CosmosElement Dispatch( JsonNodeType.Null => CosmosNull.Create(), JsonNodeType.False => CosmosBoolean.Create(false), JsonNodeType.True => CosmosBoolean.Create(true), - JsonNodeType.Number64 => CosmosNumber64.Create(jsonNavigator, jsonNavigatorNode), + JsonNodeType.Number => CosmosNumber64.Create(jsonNavigator, jsonNavigatorNode), JsonNodeType.FieldName => CosmosString.Create(jsonNavigator, jsonNavigatorNode), JsonNodeType.String => CosmosString.Create(jsonNavigator, jsonNavigatorNode), JsonNodeType.Array => CosmosArray.Create(jsonNavigator, jsonNavigatorNode), diff --git a/Microsoft.Azure.Cosmos/src/CosmosElements/Numbers/CosmosNumber64.LazyCosmosNumber64.cs b/Microsoft.Azure.Cosmos/src/CosmosElements/Numbers/CosmosNumber64.LazyCosmosNumber64.cs index cb6d66a388..8e4a29ca77 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosElements/Numbers/CosmosNumber64.LazyCosmosNumber64.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosElements/Numbers/CosmosNumber64.LazyCosmosNumber64.cs @@ -39,12 +39,12 @@ public LazyCosmosNumber64( } JsonNodeType type = jsonNavigator.GetNodeType(jsonNavigatorNode); - if (type != JsonNodeType.Number64) + if (type != JsonNodeType.Number) { - throw new ArgumentOutOfRangeException($"{nameof(jsonNavigatorNode)} must be a {JsonNodeType.Number64} node. Got {type} instead."); + throw new ArgumentOutOfRangeException($"{nameof(jsonNavigatorNode)} must be a {JsonNodeType.Number} node. Got {type} instead."); } - this.lazyNumber = new Lazy(() => jsonNavigator.GetNumber64Value(jsonNavigatorNode)); + this.lazyNumber = new Lazy(() => jsonNavigator.GetNumberValue(jsonNavigatorNode)); } public override Number64 GetValue() diff --git a/Microsoft.Azure.Cosmos/src/CosmosElements/Numbers/CosmosNumber64.cs b/Microsoft.Azure.Cosmos/src/CosmosElements/Numbers/CosmosNumber64.cs index 2cb027355f..affcdba677 100644 --- a/Microsoft.Azure.Cosmos/src/CosmosElements/Numbers/CosmosNumber64.cs +++ b/Microsoft.Azure.Cosmos/src/CosmosElements/Numbers/CosmosNumber64.cs @@ -67,7 +67,7 @@ public int CompareTo(CosmosNumber64 cosmosNumber64) public override void WriteTo(IJsonWriter jsonWriter) { - jsonWriter.WriteNumber64Value(this.GetValue()); + jsonWriter.WriteNumberValue(this.GetValue()); } public static CosmosNumber64 Create( diff --git a/Microsoft.Azure.Cosmos/src/Json/IJsonNavigator.cs b/Microsoft.Azure.Cosmos/src/Json/IJsonNavigator.cs index 58065addf9..4c87f70223 100644 --- a/Microsoft.Azure.Cosmos/src/Json/IJsonNavigator.cs +++ b/Microsoft.Azure.Cosmos/src/Json/IJsonNavigator.cs @@ -40,7 +40,7 @@ interface IJsonNavigator /// /// The node you want the number value from. /// A double that represents the number value in the node. - Number64 GetNumber64Value(IJsonNavigatorNode numberNode); + Number64 GetNumberValue(IJsonNavigatorNode numberNode); /// /// Tries to get the buffered string value from a node. @@ -178,15 +178,15 @@ bool TryGetBufferedBinaryValue( /// /// Creates an that is able to read the supplied . /// - /// The node to create a reader from.. + /// The node to create a reader from.. /// The that is able to read the supplied . - public IJsonReader CreateReader(IJsonNavigatorNode jsonNavigatorNode); + public IJsonReader CreateReader(IJsonNavigatorNode node); /// /// Writes a to a . /// - /// The to write. + /// The to write. /// The to write to. - void WriteNode(IJsonNavigatorNode jsonNavigatorNode, IJsonWriter jsonWriter); + void WriteNode(IJsonNavigatorNode node, IJsonWriter jsonWriter); } } diff --git a/Microsoft.Azure.Cosmos/src/Json/IJsonWriter.cs b/Microsoft.Azure.Cosmos/src/Json/IJsonWriter.cs index d7d0cb71ec..94228f9776 100644 --- a/Microsoft.Azure.Cosmos/src/Json/IJsonWriter.cs +++ b/Microsoft.Azure.Cosmos/src/Json/IJsonWriter.cs @@ -75,7 +75,13 @@ interface IJsonWriter /// Writes a number to the internal buffer. /// /// The value of the number to write. - void WriteNumber64Value(Number64 value); + void WriteNumberValue(Number64 value); + + /// + /// Writes an unsigned 64-bit integer to the internal buffer. + /// + /// The unsigned 64-bit integer value to write. + void WriteNumberValue(ulong value); /// /// Writes a boolean to the internal buffer. diff --git a/Microsoft.Azure.Cosmos/src/Json/Interop/CosmosDBToNewtonsoftWriter.cs b/Microsoft.Azure.Cosmos/src/Json/Interop/CosmosDBToNewtonsoftWriter.cs index d81f505b12..589040e6f4 100644 --- a/Microsoft.Azure.Cosmos/src/Json/Interop/CosmosDBToNewtonsoftWriter.cs +++ b/Microsoft.Azure.Cosmos/src/Json/Interop/CosmosDBToNewtonsoftWriter.cs @@ -212,7 +212,7 @@ public override void WriteValue(uint value) public override void WriteValue(long value) { base.WriteValue(value); - this.jsonWriter.WriteNumber64Value(value); + this.jsonWriter.WriteNumberValue(value); } /// @@ -247,7 +247,7 @@ public override void WriteValue(float value) public override void WriteValue(double value) { base.WriteValue(value); - this.jsonWriter.WriteNumber64Value(value); + this.jsonWriter.WriteNumberValue(value); } /// diff --git a/Microsoft.Azure.Cosmos/src/Json/Interop/NewtonsoftToCosmosDBReader.cs b/Microsoft.Azure.Cosmos/src/Json/Interop/NewtonsoftToCosmosDBReader.cs index 4bc702bd09..1d6eaa47fc 100644 --- a/Microsoft.Azure.Cosmos/src/Json/Interop/NewtonsoftToCosmosDBReader.cs +++ b/Microsoft.Azure.Cosmos/src/Json/Interop/NewtonsoftToCosmosDBReader.cs @@ -194,5 +194,12 @@ public override bool TryGetBufferedStringValue(out Utf8Memory bufferedUtf8String bufferedUtf8StringValue = default; return false; } + + /// + protected override bool TryGetUInt64NumberValue(out ulong value) + { + value = 0; + return false; + } } } diff --git a/Microsoft.Azure.Cosmos/src/Json/Interop/NewtonsoftToCosmosDBWriter.cs b/Microsoft.Azure.Cosmos/src/Json/Interop/NewtonsoftToCosmosDBWriter.cs index fde91c616f..7e8d162fd8 100644 --- a/Microsoft.Azure.Cosmos/src/Json/Interop/NewtonsoftToCosmosDBWriter.cs +++ b/Microsoft.Azure.Cosmos/src/Json/Interop/NewtonsoftToCosmosDBWriter.cs @@ -102,7 +102,7 @@ public override void WriteNullValue() this.writer.WriteNull(); } - public override void WriteNumber64Value(Number64 value) + public override void WriteNumberValue(Number64 value) { if (value.IsInteger) { @@ -114,6 +114,11 @@ public override void WriteNumber64Value(Number64 value) } } + public override void WriteNumberValue(ulong value) + { + this.writer.WriteValue(value); + } + public override void WriteObjectEnd() { this.writer.WriteEndObject(); diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.NodeTypes.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.NodeTypes.cs index e5e08321c8..613d793baf 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.NodeTypes.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.NodeTypes.cs @@ -21,7 +21,7 @@ public static class NodeTypes private const JsonNodeType Int64 = JsonNodeType.Int64; private const JsonNodeType Int8 = JsonNodeType.Int8; private const JsonNodeType Null = JsonNodeType.Null; - private const JsonNodeType Number = JsonNodeType.Number64; + private const JsonNodeType Number = JsonNodeType.Number; private const JsonNodeType Object = JsonNodeType.Object; private const JsonNodeType String = JsonNodeType.String; private const JsonNodeType True = JsonNodeType.True; diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Numbers.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Numbers.cs index 96faefdb2b..9e8df5648b 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Numbers.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Numbers.cs @@ -168,6 +168,16 @@ public static bool TryGetNumberValue(ReadOnlySpan numberToken, UniformArra bytesConsumed = 1 + 8; break; + case JsonBinaryEncoding.TypeMarker.NumberUInt64: + if (numberToken.Length < (1 + 8)) + { + return false; + } + + number64 = MemoryMarshal.Read(numberToken.Slice(1)); + bytesConsumed = 1 + 8; + break; + case JsonBinaryEncoding.TypeMarker.NumberDouble: if (numberToken.Length < (1 + 8)) { @@ -187,6 +197,20 @@ public static bool TryGetNumberValue(ReadOnlySpan numberToken, UniformArra return true; } + public static bool TryGetUInt64Value(ReadOnlySpan numberToken, UniformArrayInfo uniformArrayInfo, out ulong value) + { + const int RequiredLength = 1 + sizeof(ulong); + + if ((numberToken.Length >= RequiredLength) && (uniformArrayInfo == null) && (numberToken[0] == TypeMarker.NumberUInt64)) + { + value = MemoryMarshal.Read(numberToken.Slice(1)); + return true; + } + + value = 0; + return false; + } + public static sbyte GetInt8Value(ReadOnlySpan int8Token) { if (!JsonBinaryEncoding.TryGetInt8Value(int8Token, out sbyte int8Value)) diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Strings.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Strings.cs index d3c866fbf0..f0af32b43e 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Strings.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.Strings.cs @@ -433,7 +433,7 @@ private static bool TryGetBufferedLengthPrefixedString( { switch (typeMarker) { - case JsonBinaryEncoding.TypeMarker.String1ByteLength: + case JsonBinaryEncoding.TypeMarker.StrL1: if (stringTokenSpan.Length < JsonBinaryEncoding.OneByteLength) { value = default; @@ -444,7 +444,7 @@ private static bool TryGetBufferedLengthPrefixedString( length = stringTokenSpan[0]; break; - case JsonBinaryEncoding.TypeMarker.String2ByteLength: + case JsonBinaryEncoding.TypeMarker.StrL2: if (stringTokenSpan.Length < JsonBinaryEncoding.TwoByteLength) { value = default; @@ -455,7 +455,7 @@ private static bool TryGetBufferedLengthPrefixedString( length = MemoryMarshal.Read(stringTokenSpan); break; - case JsonBinaryEncoding.TypeMarker.String4ByteLength: + case JsonBinaryEncoding.TypeMarker.StrL4: if (stringTokenSpan.Length < JsonBinaryEncoding.FourByteLength) { value = default; @@ -466,7 +466,7 @@ private static bool TryGetBufferedLengthPrefixedString( length = MemoryMarshal.Read(stringTokenSpan); break; - case JsonBinaryEncoding.TypeMarker.ReferenceString1ByteOffset: + case JsonBinaryEncoding.TypeMarker.StrR1: if (stringTokenSpan.Length < JsonBinaryEncoding.OneByteOffset) { value = default; @@ -478,7 +478,7 @@ private static bool TryGetBufferedLengthPrefixedString( buffer.Slice(start: stringTokenSpan[0]), out value); - case JsonBinaryEncoding.TypeMarker.ReferenceString2ByteOffset: + case JsonBinaryEncoding.TypeMarker.StrR2: if (stringTokenSpan.Length < JsonBinaryEncoding.TwoByteOffset) { value = default; @@ -490,7 +490,7 @@ private static bool TryGetBufferedLengthPrefixedString( buffer.Slice(start: MemoryMarshal.Read(stringTokenSpan)), out value); - case JsonBinaryEncoding.TypeMarker.ReferenceString3ByteOffset: + case JsonBinaryEncoding.TypeMarker.StrR3: if (stringTokenSpan.Length < JsonBinaryEncoding.ThreeByteOffset) { value = default; @@ -502,7 +502,7 @@ private static bool TryGetBufferedLengthPrefixedString( buffer.Slice(start: MemoryMarshal.Read(stringTokenSpan)), out value); - case JsonBinaryEncoding.TypeMarker.ReferenceString4ByteOffset: + case JsonBinaryEncoding.TypeMarker.StrR4: if (stringTokenSpan.Length < JsonBinaryEncoding.FourByteOffset) { value = default; diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs index cdb409253e..0558d869e8 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs @@ -157,37 +157,37 @@ public readonly struct TypeMarker /// /// Type marker for a String of 1-byte length /// - public const byte String1ByteLength = 0xC0; + public const byte StrL1 = 0xC0; /// /// Type marker for a String of 2-byte length /// - public const byte String2ByteLength = 0xC1; + public const byte StrL2 = 0xC1; /// /// Type marker for a String of 4-byte length /// - public const byte String4ByteLength = 0xC2; + public const byte StrL4 = 0xC2; /// /// Reference string of 1-byte offset /// - public const byte ReferenceString1ByteOffset = 0xC3; + public const byte StrR1 = 0xC3; /// /// Reference string of 2-byte offset /// - public const byte ReferenceString2ByteOffset = 0xC4; + public const byte StrR2 = 0xC4; /// /// Reference string of 3-byte offset /// - public const byte ReferenceString3ByteOffset = 0xC5; + public const byte StrR3 = 0xC5; /// /// Reference string of 4-byte offset /// - public const byte ReferenceString4ByteOffset = 0xC6; + public const byte StrR4 = 0xC6; /// /// Type marker for a 8-byte unsigned integer @@ -583,7 +583,7 @@ public static bool IsEncodedLengthString(byte typeMarker) /// The input type marker. /// Whether the typeMarker is for a variable length string. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsVariableLengthString(byte typeMarker) => IsEncodedLengthString(typeMarker) || InRange(typeMarker, String1ByteLength, String4ByteLength + 1); + public static bool IsVariableLengthString(byte typeMarker) => IsEncodedLengthString(typeMarker) || InRange(typeMarker, StrL1, StrL4 + 1); /// /// Gets whether a typeMarker is for a reference string. @@ -591,7 +591,7 @@ public static bool IsEncodedLengthString(byte typeMarker) /// The input type marker. /// Whether the typeMarker is for a reference string. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsReferenceString(byte typeMarker) => InRange(typeMarker, ReferenceString1ByteOffset, ReferenceString4ByteOffset + 1); + public static bool IsReferenceString(byte typeMarker) => InRange(typeMarker, StrR1, StrR4 + 1); /// /// Gets whether a typeMarker is for a GUID string. @@ -624,7 +624,7 @@ public static bool IsEncodedLengthString(byte typeMarker) /// Whether the typeMarker is for a string. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsString(byte typeMarker) => InRange(typeMarker, SystemString1ByteLengthMin, UserString2ByteLengthMax) - || InRange(typeMarker, LowercaseGuidString, ReferenceString4ByteOffset + 1); + || InRange(typeMarker, LowercaseGuidString, StrR4 + 1); /// /// Gets the length of a encoded string type marker. diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.JsonBinaryNavigator.cs b/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.JsonBinaryNavigator.cs index e208796bfe..18803cf9b2 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.JsonBinaryNavigator.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.JsonBinaryNavigator.cs @@ -83,9 +83,9 @@ public override JsonNodeType GetNodeType(IJsonNavigatorNode node) } /// - public override Number64 GetNumber64Value(IJsonNavigatorNode numberNode) + public override Number64 GetNumberValue(IJsonNavigatorNode numberNode) { - BinaryNavigatorNode binaryNavigatorNode = this.GetNodeOfType(JsonNodeType.Number64, numberNode); + BinaryNavigatorNode binaryNavigatorNode = this.GetNodeOfType(JsonNodeType.Number, numberNode); return JsonBinaryEncoding.GetNumberValue( this.GetBufferAt(binaryNavigatorNode.Offset), binaryNavigatorNode.ExternalArrayInfo); @@ -432,6 +432,16 @@ public override void WriteNode(IJsonNavigatorNode jsonNavigatorNode, IJsonWriter } #endregion + /// + protected override bool TryGetUInt64Value(IJsonNavigatorNode numberNode, out ulong value) + { + BinaryNavigatorNode binaryNavigatorNode = this.GetNodeOfType(JsonNodeType.Number, numberNode); + return JsonBinaryEncoding.TryGetUInt64Value( + this.GetBufferAt(binaryNavigatorNode.Offset), + binaryNavigatorNode.ExternalArrayInfo, + out value); + } + private IEnumerable GetArrayItemsInternal(BinaryNavigatorNode arrayNode) { return Enumerator @@ -474,13 +484,15 @@ private void WriteToInternal(BinaryNavigatorNode binaryNavigatorNode, IJsonWrite jsonWriter.WriteBoolValue(true); break; - case JsonNodeType.Number64: + case JsonNodeType.Number: + if (JsonBinaryEncoding.TryGetUInt64Value(buffer.Span, binaryNavigatorNode.ExternalArrayInfo, out ulong uint64Value)) { - Number64 value = JsonBinaryEncoding.GetNumberValue( - buffer.Span, - binaryNavigatorNode.ExternalArrayInfo); - - jsonWriter.WriteNumber64Value(value); + jsonWriter.WriteNumberValue(uint64Value); + } + else + { + Number64 value = JsonBinaryEncoding.GetNumberValue(buffer.Span, binaryNavigatorNode.ExternalArrayInfo); + jsonWriter.WriteNumberValue(value); } break; @@ -668,7 +680,7 @@ private JsonNodeType GetNodeType(int offset, UniformArrayInfo externalArrayInfo) case TypeMarker.Float16: case TypeMarker.Float32: case TypeMarker.Float64: - nodeType = JsonNodeType.Number64; + nodeType = JsonNodeType.Number; break; case TypeMarker.ArrNumC1: diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.JsonTextNavigator.cs b/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.JsonTextNavigator.cs index cf87079922..203003b005 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.JsonTextNavigator.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.JsonTextNavigator.cs @@ -7,6 +7,7 @@ namespace Microsoft.Azure.Cosmos.Json using System.Collections.Generic; using System.Runtime.InteropServices; using System.Text; + using System.Xml.Linq; using Microsoft.Azure.Cosmos.Core.Utf8; /// @@ -86,6 +87,7 @@ JsonTextNavigatorNode CreateRootNode() this.rootNode = rootNode; } + #region IJsonNavigator /// public override JsonSerializationFormat SerializationFormat => JsonSerializationFormat.Text; @@ -107,7 +109,7 @@ public override JsonNodeType GetNodeType(IJsonNavigatorNode node) } /// - public override Number64 GetNumber64Value(IJsonNavigatorNode node) + public override Number64 GetNumberValue(IJsonNavigatorNode node) { if (!(node is NumberNode numberNode)) { @@ -404,16 +406,27 @@ public override void WriteNode(IJsonNavigatorNode jsonNavigatorNode, IJsonWriter } } - public override IJsonReader CreateReader(IJsonNavigatorNode jsonNavigatorNode) + public override IJsonReader CreateReader(IJsonNavigatorNode node) { - if (!(jsonNavigatorNode is JsonTextNavigatorNode jsonTextNavigatorNode)) + if (!(node is JsonTextNavigatorNode jsonTextNavigatorNode)) { - throw new ArgumentException($"{nameof(jsonNavigatorNode)} must be a {nameof(JsonTextNavigatorNode)}."); + throw new ArgumentException($"{nameof(node)} must be a {nameof(JsonTextNavigatorNode)}."); } ReadOnlyMemory buffer = JsonTextNavigator.GetNodeBuffer(jsonTextNavigatorNode); return JsonReader.Create(JsonSerializationFormat.Text, buffer); } + #endregion + + protected override bool TryGetUInt64Value(IJsonNavigatorNode node, out ulong value) + { + if (!(node is NumberNode numberNode)) + { + throw new ArgumentException($"{node} was not of type: {nameof(NumberNode)}."); + } + + return JsonTextParser.TryGetUInt64Value(numberNode.BufferedToken.Span, out value); + } private static ReadOnlyMemory GetNodeBuffer(JsonTextNavigatorNode jsonTextNavigatorNode) { @@ -867,7 +880,7 @@ private NumberNode(int value) public ReadOnlyMemory BufferedToken { get; } - public override JsonNodeType Type => JsonNodeType.Number64; + public override JsonNodeType Type => JsonNodeType.Number; public static NumberNode Create(ReadOnlyMemory bufferedToken) { diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.cs b/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.cs index 707c70c2cc..c6e37e7427 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonNavigator.cs @@ -54,6 +54,7 @@ public static IJsonNavigator Create( }; } + #region IJsonNavigator /// public abstract IJsonNavigatorNode GetRootNode(); @@ -61,7 +62,7 @@ public static IJsonNavigator Create( public abstract JsonNodeType GetNodeType(IJsonNavigatorNode node); /// - public abstract Number64 GetNumber64Value(IJsonNavigatorNode numberNode); + public abstract Number64 GetNumberValue(IJsonNavigatorNode numberNode); /// public abstract bool TryGetBufferedStringValue(IJsonNavigatorNode stringNode, out Utf8Memory bufferedStringValue); @@ -118,9 +119,9 @@ public static IJsonNavigator Create( public abstract IEnumerable GetObjectProperties(IJsonNavigatorNode objectNode); /// - public virtual void WriteNode(IJsonNavigatorNode jsonNavigatorNode, IJsonWriter jsonWriter) + public virtual void WriteNode(IJsonNavigatorNode node, IJsonWriter jsonWriter) { - JsonNodeType nodeType = this.GetNodeType(jsonNavigatorNode); + JsonNodeType nodeType = this.GetNodeType(node); switch (nodeType) { case JsonNodeType.Null: @@ -134,17 +135,23 @@ public virtual void WriteNode(IJsonNavigatorNode jsonNavigatorNode, IJsonWriter case JsonNodeType.True: jsonWriter.WriteBoolValue(true); return; - case JsonNodeType.Number64: + + case JsonNodeType.Number: + if (this.TryGetUInt64Value(node, out ulong uint64Value)) + { + jsonWriter.WriteNumberValue(uint64Value); + } + else { - Number64 value = this.GetNumber64Value(jsonNavigatorNode); - jsonWriter.WriteNumber64Value(value); + Number64 value = this.GetNumberValue(node); + jsonWriter.WriteNumberValue(value); } break; case JsonNodeType.String: case JsonNodeType.FieldName: bool fieldName = nodeType == JsonNodeType.FieldName; - if (this.TryGetBufferedStringValue(jsonNavigatorNode, out Utf8Memory bufferedStringValue)) + if (this.TryGetBufferedStringValue(node, out Utf8Memory bufferedStringValue)) { if (fieldName) { @@ -157,7 +164,7 @@ public virtual void WriteNode(IJsonNavigatorNode jsonNavigatorNode, IJsonWriter } else { - string value = this.GetStringValue(jsonNavigatorNode); + string value = this.GetStringValue(node); if (fieldName) { jsonWriter.WriteFieldName(value); @@ -173,7 +180,7 @@ public virtual void WriteNode(IJsonNavigatorNode jsonNavigatorNode, IJsonWriter { jsonWriter.WriteArrayStart(); - foreach (IJsonNavigatorNode arrayItem in this.GetArrayItems(jsonNavigatorNode)) + foreach (IJsonNavigatorNode arrayItem in this.GetArrayItems(node)) { this.WriteNode(arrayItem, jsonWriter); } @@ -186,7 +193,7 @@ public virtual void WriteNode(IJsonNavigatorNode jsonNavigatorNode, IJsonWriter { jsonWriter.WriteObjectStart(); - foreach (ObjectProperty objectProperty in this.GetObjectProperties(jsonNavigatorNode)) + foreach (ObjectProperty objectProperty in this.GetObjectProperties(node)) { this.WriteNode(objectProperty.NameNode, jsonWriter); this.WriteNode(objectProperty.ValueNode, jsonWriter); @@ -198,63 +205,63 @@ public virtual void WriteNode(IJsonNavigatorNode jsonNavigatorNode, IJsonWriter case JsonNodeType.Int8: { - sbyte value = this.GetInt8Value(jsonNavigatorNode); + sbyte value = this.GetInt8Value(node); jsonWriter.WriteInt8Value(value); } break; case JsonNodeType.Int16: { - short value = this.GetInt16Value(jsonNavigatorNode); + short value = this.GetInt16Value(node); jsonWriter.WriteInt16Value(value); } break; case JsonNodeType.Int32: { - int value = this.GetInt32Value(jsonNavigatorNode); + int value = this.GetInt32Value(node); jsonWriter.WriteInt32Value(value); } break; case JsonNodeType.Int64: { - long value = this.GetInt64Value(jsonNavigatorNode); + long value = this.GetInt64Value(node); jsonWriter.WriteInt64Value(value); } break; case JsonNodeType.UInt32: { - uint value = this.GetUInt32Value(jsonNavigatorNode); + uint value = this.GetUInt32Value(node); jsonWriter.WriteUInt32Value(value); } break; case JsonNodeType.Float32: { - float value = this.GetFloat32Value(jsonNavigatorNode); + float value = this.GetFloat32Value(node); jsonWriter.WriteFloat32Value(value); } break; case JsonNodeType.Float64: { - double value = this.GetFloat64Value(jsonNavigatorNode); + double value = this.GetFloat64Value(node); jsonWriter.WriteFloat64Value(value); } break; case JsonNodeType.Binary: { - ReadOnlyMemory value = this.GetBinaryValue(jsonNavigatorNode); + ReadOnlyMemory value = this.GetBinaryValue(node); jsonWriter.WriteBinaryValue(value.Span); } break; case JsonNodeType.Guid: { - Guid value = this.GetGuidValue(jsonNavigatorNode); + Guid value = this.GetGuidValue(node); jsonWriter.WriteGuidValue(value); } break; @@ -265,6 +272,15 @@ public virtual void WriteNode(IJsonNavigatorNode jsonNavigatorNode, IJsonWriter } /// - public abstract IJsonReader CreateReader(IJsonNavigatorNode jsonNavigatorNode); + public abstract IJsonReader CreateReader(IJsonNavigatorNode node); + #endregion + + /// + /// Attempts to read the specified number node as an unsigned 64-bit integer. + /// + /// The number to retrieve its value. + /// When this method returns, contains the value of the specified number node if it was an unsigned 64-bit integer; otherwise, the default value of ulong. + /// true if the number node value is an unsigned 64-bit integer; otherwise, false. + protected abstract bool TryGetUInt64Value(IJsonNavigatorNode numberNode, out ulong value); } } diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonNodeType.cs b/Microsoft.Azure.Cosmos/src/Json/JsonNodeType.cs index 602916d5d2..89eb10254d 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonNodeType.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonNodeType.cs @@ -31,7 +31,7 @@ enum JsonNodeType /// /// Corresponds to the number type in JSON (number = [ minus ] integer [ fraction ] [ exponent ]) /// - Number64, + Number, /// /// Corresponds to the string type in JSON (string = quotation-mark *char quotation-mark) diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonBinaryReader.cs b/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonBinaryReader.cs index ad46764428..04996ebd64 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonBinaryReader.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonBinaryReader.cs @@ -486,8 +486,9 @@ public override ReadOnlyMemory GetBinaryValue() return JsonBinaryEncoding.GetBinaryValue(this.jsonBinaryBuffer.GetBufferedRawJsonToken(this.currentTokenPosition)); } -#endregion + #endregion + #region ITypedJsonReader /// public bool TryReadTypedJsonValueWrapper(out int typeCode) { @@ -540,6 +541,30 @@ public Utf8Span GetUtf8SpanValue() this.rootBuffer, this.jsonBinaryBuffer.GetBufferedRawJsonToken(this.currentTokenPosition)); } + #endregion + + /// + protected override bool TryGetUInt64NumberValue(out ulong value) + { + if (this.JsonObjectState.CurrentTokenType != JsonTokenType.Number) + { + throw new JsonNotNumberTokenException(); + } + + // Check if we are within a uniform array info + if (this.arrayAndObjectEndStack.GetUniformArrayInfo() == null) + { + ReadOnlyMemory currentValue = this.jsonBinaryBuffer.GetBufferedRawJsonToken(this.currentTokenPosition); + if (currentValue.Span[0] == TypeMarker.NumberUInt64) + { + value = JsonBinaryEncoding.GetFixedSizedValue(currentValue.Slice(1).Span); + return true; + } + } + + value = 0; + return false; + } private static JsonTokenType GetJsonTokenType(byte typeMarker, UniformArrayInfo arrayInfo) { diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonTextReader.cs b/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonTextReader.cs index f732c5a702..47a9018588 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonTextReader.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonReader.JsonTextReader.cs @@ -143,6 +143,7 @@ private enum JsonTextTokenType Binary = JsonTokenType.Binary, } + #region IJsonReader /// public override JsonSerializationFormat SerializationFormat => JsonSerializationFormat.Text; @@ -396,6 +397,7 @@ public override ReadOnlyMemory GetBinaryValue() this.token.End).Span; return JsonTextParser.GetBinaryValue(binaryToken); } + #endregion Utf8Memory IJsonTextReaderPrivateImplementation.GetBufferedJsonToken() { @@ -405,6 +407,12 @@ Utf8Memory IJsonTextReaderPrivateImplementation.GetBufferedJsonToken() return Utf8Memory.UnsafeCreateNoValidation(bufferedRawJson); } + protected override bool TryGetUInt64NumberValue(out ulong value) + { + ReadOnlySpan numberToken = this.jsonTextBuffer.GetBufferedRawJsonToken(this.token.Start, this.token.End).Span; + return JsonTextParser.TryGetUInt64Value(numberToken, out value); + } + private static JsonTokenType JsonTextToJsonTokenType(JsonTextTokenType jsonTextTokenType) { return (JsonTokenType)((int)jsonTextTokenType & 0xFFFF); diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonReader.cs b/Microsoft.Azure.Cosmos/src/Json/JsonReader.cs index b773e45dfa..07789ae9de 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonReader.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonReader.cs @@ -21,7 +21,7 @@ abstract partial class JsonReader : IJsonReader /// /// The /// - internal readonly JsonObjectState JsonObjectState; + protected readonly JsonObjectState JsonObjectState; /// /// Initializes a new instance of the JsonReader class. @@ -87,6 +87,7 @@ internal static IJsonReader CreateBinaryFromOffset( ReadOnlyMemory buffer, int offset) => new JsonBinaryReader(buffer, offset); + #region IJsonReader /// public abstract bool Read(); @@ -188,9 +189,14 @@ public virtual void WriteCurrentToken(IJsonWriter writer) break; case JsonTokenType.Number: + if (this.TryGetUInt64NumberValue(out ulong uint64Value)) + { + writer.WriteNumberValue(uint64Value); + } + else { Number64 value = this.GetNumberValue(); - writer.WriteNumber64Value(value); + writer.WriteNumberValue(value); } break; @@ -282,5 +288,13 @@ public virtual void WriteAll(IJsonWriter writer) this.WriteCurrentToken(writer); } } + #endregion + + /// + /// Attempts to read the current number token as an unsigned 64-bit integer. + /// + /// When this method returns, contains the value of the current number token if it was an unsigned 64-bit integer; otherwise, the default value of ulong. + /// true if the number token value is an unsigned 64-bit integer; otherwise, false. + protected abstract bool TryGetUInt64NumberValue(out ulong value); } } diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonSerializer.cs b/Microsoft.Azure.Cosmos/src/Json/JsonSerializer.cs index 0fbf4500b9..d6e4ad2ea2 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonSerializer.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonSerializer.cs @@ -47,7 +47,7 @@ public static void SerializeInternal( break; case Number64 numberValue: - jsonWriter.WriteNumber64Value(numberValue); + jsonWriter.WriteNumberValue(numberValue); break; case sbyte signedByteValue: diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonTextParser.cs b/Microsoft.Azure.Cosmos/src/Json/JsonTextParser.cs index 3a0d3689c5..644d3b71e0 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonTextParser.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonTextParser.cs @@ -51,6 +51,11 @@ public static Number64 GetNumberValue(ReadOnlySpan token) return numberValue; } + public static bool TryGetUInt64Value(ReadOnlySpan token, out ulong value) + { + return Utf8Parser.TryParse(token, out value, out int bytesConsumed1) && (bytesConsumed1 == token.Length); + } + public static Utf8String GetStringValue(Utf8Memory token) { // Offsetting by an additional character and removing 2 from the length since we want to skip the quotes. diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonWriteOptions.cs b/Microsoft.Azure.Cosmos/src/Json/JsonWriteOptions.cs index 82720b8512..c0ec54d963 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonWriteOptions.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonWriteOptions.cs @@ -29,6 +29,6 @@ enum JsonWriteOptions /// /// Enables support for writing 64-bit unsigned integers (UInt64). /// - EnableUInt64 = 1 << 1, + EnableUInt64Values = 1 << 1, } } diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonBinaryWriter.cs b/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonBinaryWriter.cs index a01bc33cf3..ab24c03a49 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonBinaryWriter.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonBinaryWriter.cs @@ -15,6 +15,7 @@ namespace Microsoft.Azure.Cosmos.Json using Microsoft.Azure.Cosmos.Core.Utf8; using Microsoft.Azure.Cosmos.Linq; using Microsoft.Azure.Cosmos.Serialization.HybridRow; + using Newtonsoft.Json.Linq; using static Microsoft.Azure.Cosmos.Json.JsonBinaryEncoding; /// @@ -39,6 +40,7 @@ public static PreblittedBinaryJsonScope CapturePreblittedBinaryJsonScope(Action< JsonBinaryWriter jsonBinaryWriter = new JsonBinaryWriter( initialCapacity: 256, enableNumberArrays: false, + enableUint64Values: false, enableEncodedStrings: false); Contract.Requires(!jsonBinaryWriter.JsonObjectState.InArrayContext); Contract.Requires(!jsonBinaryWriter.JsonObjectState.InObjectContext); @@ -78,7 +80,7 @@ private enum RawValueType : byte Obj, ArrNum, ArrArrNum, - UInt64, + NumUI64, } private const int MaxStackAllocSize = 4 * 1024; @@ -156,7 +158,7 @@ private enum RawValueType : byte RawValueType.StrR2, // RawValueType.StrR2 (Reference string of 2-byte offset) RawValueType.StrR3, // RawValueType.StrR3 (Reference string of 3-byte offset) RawValueType.StrR4, // RawValueType.StrR4 (Reference string of 4-byte offset) - RawValueType.Token, // 0xC7 + RawValueType.NumUI64, // NumUI64 // Number Values RawValueType.Token, // NumUI8 @@ -248,10 +250,15 @@ private enum RawValueType : byte private readonly bool enableEncodedStrings; /// - /// Determines whether to allow writing of uniform number arrays. + /// Determines whether to enable writing of uniform number arrays. /// private readonly bool enableNumberArrays; + /// + /// Determines whether to enable writing of full-precision unsigned 64-bit integer values. + /// + private readonly bool enableUInt64Values; + /// /// Writer used to write fully materialized context to the internal stream. /// @@ -287,13 +294,16 @@ private enum RawValueType : byte /// /// The initial capacity to avoid intermediary allocations. /// Determines whether to enable writing of uniform number arrays. + /// Determines whether to enable writing of full-precision unsigned 64-bit integer values /// Determines whether to enable reference string encoding. public JsonBinaryWriter( int initialCapacity, bool enableNumberArrays, + bool enableUint64Values, bool enableEncodedStrings = true) { this.enableNumberArrays = enableNumberArrays; + this.enableUInt64Values = enableUint64Values; this.enableEncodedStrings = enableEncodedStrings; this.binaryWriter = new JsonBinaryMemoryWriter(initialCapacity); this.bufferedContexts = new Stack(); @@ -335,7 +345,7 @@ public JsonBinaryWriter( public override void WriteStringValue(Utf8Span value) => this.WriteFieldNameOrString(isFieldName: false, value); /// - public override void WriteNumber64Value(Number64 value) + public override void WriteNumberValue(Number64 value) { if (value.IsInteger) { @@ -349,6 +359,14 @@ public override void WriteNumber64Value(Number64 value) this.bufferedContexts.Peek().Count++; } + /// + public override void WriteNumberValue(ulong value) + { + this.WriteIntegerInternal(value); + + this.bufferedContexts.Peek().Count++; + } + /// public override void WriteBoolValue(bool value) { @@ -1308,7 +1326,7 @@ private void FixReferenceStringOffsets(Span binaryWriterRawBuffer) Span offsetBuffer = binaryWriterRawBuffer.Slice(stringReferenceOffset + 1); switch (typeMarker) { - case TypeMarker.ReferenceString1ByteOffset: + case TypeMarker.StrR1: { byte stringIndex = offsetBuffer[0]; SharedStringValue sharedStringValue = this.sharedStrings[stringIndex]; @@ -1316,7 +1334,7 @@ private void FixReferenceStringOffsets(Span binaryWriterRawBuffer) break; } - case TypeMarker.ReferenceString2ByteOffset: + case TypeMarker.StrR2: { ushort stringIndex = JsonBinaryEncoding.GetFixedSizedValue(offsetBuffer); SharedStringValue sharedStringValue = this.sharedStrings[stringIndex]; @@ -1324,7 +1342,7 @@ private void FixReferenceStringOffsets(Span binaryWriterRawBuffer) break; } - case TypeMarker.ReferenceString3ByteOffset: + case TypeMarker.StrR3: { JsonBinaryEncoding.UInt24 stringIndex = JsonBinaryEncoding.GetFixedSizedValue(offsetBuffer); @@ -1335,7 +1353,7 @@ private void FixReferenceStringOffsets(Span binaryWriterRawBuffer) break; } - case TypeMarker.ReferenceString4ByteOffset: + case TypeMarker.StrR4: { int stringIndex = JsonBinaryEncoding.GetFixedSizedValue(offsetBuffer); SharedStringValue sharedStringValue = this.sharedStrings[stringIndex]; @@ -1427,20 +1445,20 @@ private void WriteFieldNameOrString(bool isFieldName, Utf8Span utf8Span) // Just write it out as a regular string with type marker and length prefix else if (utf8Span.Length < byte.MaxValue) { - this.binaryWriter.Write(TypeMarker.String1ByteLength); + this.binaryWriter.Write(TypeMarker.StrL1); this.binaryWriter.Write((byte)utf8Span.Length); this.binaryWriter.Write(utf8Span.Span); } else if (utf8Span.Length < ushort.MaxValue) { - this.binaryWriter.Write(TypeMarker.String2ByteLength); + this.binaryWriter.Write(TypeMarker.StrL2); this.binaryWriter.Write((ushort)utf8Span.Length); this.binaryWriter.Write(utf8Span.Span); } else { // (utf8String.Length < uint.MaxValue) - this.binaryWriter.Write(TypeMarker.String4ByteLength); + this.binaryWriter.Write(TypeMarker.StrL4); this.binaryWriter.Write((uint)utf8Span.Length); this.binaryWriter.Write(utf8Span.Span); } @@ -1482,22 +1500,22 @@ private bool TryRegisterStringValue(Utf8Span utf8Span) if (sharedString.MaxOffset <= byte.MaxValue) { - this.binaryWriter.Write(TypeMarker.ReferenceString1ByteOffset); + this.binaryWriter.Write(TypeMarker.StrR1); this.binaryWriter.Write((byte)hashAndIndex.index); } else if (sharedString.MaxOffset <= ushort.MaxValue) { - this.binaryWriter.Write(TypeMarker.ReferenceString2ByteOffset); + this.binaryWriter.Write(TypeMarker.StrR2); this.binaryWriter.Write((ushort)hashAndIndex.index); } else if (sharedString.MaxOffset <= JsonBinaryEncoding.UInt24.MaxValue) { - this.binaryWriter.Write(TypeMarker.ReferenceString3ByteOffset); + this.binaryWriter.Write(TypeMarker.StrR3); this.binaryWriter.Write((JsonBinaryEncoding.UInt24)(int)hashAndIndex.index); } else if (sharedString.MaxOffset <= int.MaxValue) { - this.binaryWriter.Write(TypeMarker.ReferenceString4ByteOffset); + this.binaryWriter.Write(TypeMarker.StrR4); this.binaryWriter.Write((int)hashAndIndex.index); } else @@ -1563,6 +1581,24 @@ private void WriteIntegerInternal(long value) } } + private void WriteIntegerInternal(ulong value) + { + if (value <= long.MaxValue) + { + this.WriteIntegerInternal((long)value); + } + else if (!this.enableUInt64Values) + { + this.WriteDoubleInternal(value); + } + else + { + this.JsonObjectState.RegisterToken(JsonTokenType.Number); + this.binaryWriter.Write(TypeMarker.NumberUInt64); + this.binaryWriter.Write(value); + } + } + private void WriteDoubleInternal(double value) { this.JsonObjectState.RegisterToken(JsonTokenType.Number); @@ -1596,8 +1632,14 @@ private void ForceRewriteRawJsonValue( { RawValueType rawType = (RawValueType)RawValueTypes[typeMarker]; - // If the writer supports uniform-number arrays then we treat them as a value token - if (this.enableEncodedStrings && ((rawType == RawValueType.ArrNum) || (rawType == RawValueType.ArrArrNum))) + // Check for uniform number array support + if (this.enableNumberArrays && ((rawType == RawValueType.ArrNum) || (rawType == RawValueType.ArrArrNum))) + { + rawType = RawValueType.Token; + } + + // Check for unsigned 64-bit integer support + if (this.enableUInt64Values && (rawType == RawValueType.NumUI64)) { rawType = RawValueType.Token; } @@ -1738,6 +1780,10 @@ private void ForceRewriteRawJsonValue( this.WriteRawNumberArrayArray(rawJsonValue.Span, GetUniformArrayInfo(rawJsonValue.Span)); break; + case RawValueType.NumUI64: + this.WriteNumberValue(GetFixedSizedValue(rawJsonValue.Slice(1).Span)); + break; + default: throw new InvalidOperationException($"Unknown {nameof(RawValueType)} {rawType}."); } @@ -1806,25 +1852,25 @@ private void WriteRawUniformArrayItem( switch (arrayInfo.ItemTypeMarker) { case TypeMarker.Int8: - this.WriteNumber64Value(GetFixedSizedValue(rawValue)); + this.WriteNumberValue(GetFixedSizedValue(rawValue)); break; case TypeMarker.UInt8: - this.WriteNumber64Value(GetFixedSizedValue(rawValue)); + this.WriteNumberValue(GetFixedSizedValue(rawValue)); break; case TypeMarker.Int16: - this.WriteNumber64Value(GetFixedSizedValue(rawValue)); + this.WriteNumberValue(GetFixedSizedValue(rawValue)); break; case TypeMarker.Int32: - this.WriteNumber64Value(GetFixedSizedValue(rawValue)); + this.WriteNumberValue(GetFixedSizedValue(rawValue)); break; case TypeMarker.Int64: - this.WriteNumber64Value(GetFixedSizedValue(rawValue)); + this.WriteNumberValue(GetFixedSizedValue(rawValue)); break; case TypeMarker.Float32: - this.WriteNumber64Value(GetFixedSizedValue(rawValue)); + this.WriteNumberValue(GetFixedSizedValue(rawValue)); break; case TypeMarker.Float64: - this.WriteNumber64Value(GetFixedSizedValue(rawValue)); + this.WriteNumberValue(GetFixedSizedValue(rawValue)); break; case TypeMarker.ArrNumC1: @@ -1845,55 +1891,56 @@ private void WriteRawNumberArray( this.WriteArrayStart(); - int endOffset = arrayInfo.ItemCount * arrayInfo.ItemSize; + int startOffset = arrayInfo.PrefixSize; + int endOffset = startOffset + (arrayInfo.ItemCount * arrayInfo.ItemSize); switch (arrayInfo.ItemTypeMarker) { case TypeMarker.Int8: - for (int offset = 0; offset < endOffset; offset += arrayInfo.ItemSize) + for (int offset = startOffset; offset < endOffset; offset += arrayInfo.ItemSize) { - this.WriteNumber64Value(GetFixedSizedValue(rawValue.Slice(offset))); + this.WriteNumberValue(GetFixedSizedValue(rawValue.Slice(offset))); } break; case TypeMarker.UInt8: - for (int offset = 0; offset < endOffset; offset += arrayInfo.ItemSize) + for (int offset = startOffset; offset < endOffset; offset += arrayInfo.ItemSize) { - this.WriteNumber64Value(GetFixedSizedValue(rawValue.Slice(offset))); + this.WriteNumberValue(GetFixedSizedValue(rawValue.Slice(offset))); } break; case TypeMarker.Int16: - for (int offset = 0; offset < endOffset; offset += arrayInfo.ItemSize) + for (int offset = startOffset; offset < endOffset; offset += arrayInfo.ItemSize) { - this.WriteNumber64Value(GetFixedSizedValue(rawValue.Slice(offset))); + this.WriteNumberValue(GetFixedSizedValue(rawValue.Slice(offset))); } break; case TypeMarker.Int32: - for (int offset = 0; offset < endOffset; offset += arrayInfo.ItemSize) + for (int offset = startOffset; offset < endOffset; offset += arrayInfo.ItemSize) { - this.WriteNumber64Value(GetFixedSizedValue(rawValue.Slice(offset))); + this.WriteNumberValue(GetFixedSizedValue(rawValue.Slice(offset))); } break; case TypeMarker.Int64: - for (int offset = 0; offset < endOffset; offset += arrayInfo.ItemSize) + for (int offset = startOffset; offset < endOffset; offset += arrayInfo.ItemSize) { - this.WriteNumber64Value(GetFixedSizedValue(rawValue.Slice(offset))); + this.WriteNumberValue(GetFixedSizedValue(rawValue.Slice(offset))); } break; case TypeMarker.Float32: - for (int offset = 0; offset < endOffset; offset += arrayInfo.ItemSize) + for (int offset = startOffset; offset < endOffset; offset += arrayInfo.ItemSize) { - this.WriteNumber64Value(GetFixedSizedValue(rawValue.Slice(offset))); + this.WriteNumberValue(GetFixedSizedValue(rawValue.Slice(offset))); } break; case TypeMarker.Float64: - for (int offset = 0; offset < endOffset; offset += arrayInfo.ItemSize) + for (int offset = startOffset; offset < endOffset; offset += arrayInfo.ItemSize) { - this.WriteNumber64Value(GetFixedSizedValue(rawValue.Slice(offset))); + this.WriteNumberValue(GetFixedSizedValue(rawValue.Slice(offset))); } break; @@ -1912,8 +1959,9 @@ private void WriteRawNumberArrayArray( this.WriteArrayStart(); - int endOffset = arrayInfo.ItemCount * arrayInfo.ItemSize; - for (int offset = 0; offset < endOffset; offset += arrayInfo.ItemSize) + int startOffset = arrayInfo.PrefixSize; + int endOffset = startOffset + (arrayInfo.ItemCount * arrayInfo.ItemSize); + for (int offset = startOffset; offset < endOffset; offset += arrayInfo.ItemSize) { this.WriteRawNumberArray(rawValue.Slice(offset), arrayInfo.NestedArrayInfo); } @@ -2002,6 +2050,13 @@ public void Write(long value) this.Position += sizeof(long); } + public void Write(ulong value) + { + this.EnsureRemainingBufferSpace(sizeof(ulong)); + BinaryPrimitives.WriteUInt64LittleEndian(this.Cursor, value); + this.Position += sizeof(long); + } + public void Write(float value) { this.EnsureRemainingBufferSpace(sizeof(float)); diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonTextWriter.cs b/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonTextWriter.cs index 09b19989d0..16c9a5284c 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonTextWriter.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonTextWriter.cs @@ -176,7 +176,7 @@ public override void WriteStringValue(Utf8Span value) } /// - public override void WriteNumber64Value(Number64 value) + public override void WriteNumberValue(Number64 value) { if (value.IsInteger) { @@ -188,6 +188,12 @@ public override void WriteNumber64Value(Number64 value) } } + /// + public override void WriteNumberValue(ulong value) + { + this.WriteIntegerInternal(value); + } + /// public override void WriteBoolValue(bool value) { @@ -329,6 +335,13 @@ private void WriteIntegerInternal(long value) this.jsonTextMemoryWriter.Write(value); } + private void WriteIntegerInternal(ulong value) + { + this.JsonObjectState.RegisterToken(JsonTokenType.Number); + this.PrefixMemberSeparator(); + this.jsonTextMemoryWriter.Write(value); + } + private void WriteDoubleInternal(double value) { this.JsonObjectState.RegisterToken(JsonTokenType.Number); @@ -614,6 +627,18 @@ public void Write(long value) this.Position += bytesWritten; } + public void Write(ulong value) + { + const int MaxUInt64Length = 20; + this.EnsureRemainingBufferSpace(MaxUInt64Length); + if (!Utf8Formatter.TryFormat(value, this.Cursor, out int bytesWritten)) + { + throw new InvalidOperationException($"Failed to {nameof(this.Write)}({typeof(long).FullName}{value})"); + } + + this.Position += bytesWritten; + } + public void Write(float value) { const int MaxNumberLength = 32; diff --git a/Microsoft.Azure.Cosmos/src/Json/JsonWriter.cs b/Microsoft.Azure.Cosmos/src/Json/JsonWriter.cs index 9dd28998de..20535f0c31 100644 --- a/Microsoft.Azure.Cosmos/src/Json/JsonWriter.cs +++ b/Microsoft.Azure.Cosmos/src/Json/JsonWriter.cs @@ -60,6 +60,7 @@ public static IJsonWriter Create( JsonSerializationFormat.Text => new JsonTextWriter(initialCapacity), JsonSerializationFormat.Binary => new JsonBinaryWriter( enableNumberArrays: writeOptions.HasFlag(JsonWriteOptions.EnableNumberArrays), + enableUint64Values: writeOptions.HasFlag(JsonWriteOptions.EnableUInt64Values), initialCapacity: initialCapacity), _ => throw new ArgumentException( string.Format( @@ -110,7 +111,10 @@ public virtual void WriteStringValue(string value) public abstract void WriteStringValue(Utf8Span value); /// - public abstract void WriteNumber64Value(Number64 value); + public abstract void WriteNumberValue(Number64 value); + + /// + public abstract void WriteNumberValue(ulong value); /// public abstract void WriteBoolValue(bool value); @@ -153,7 +157,7 @@ public virtual void WriteNumberArray(IReadOnlyList values) foreach (byte value in values) { Number64 number64 = value; - this.WriteNumber64Value(number64); + this.WriteNumberValue(number64); } this.WriteArrayEnd(); @@ -167,7 +171,7 @@ public virtual void WriteNumberArray(IReadOnlyList values) foreach (sbyte value in values) { Number64 number64 = value; - this.WriteNumber64Value(number64); + this.WriteNumberValue(number64); } this.WriteArrayEnd(); @@ -181,7 +185,7 @@ public virtual void WriteNumberArray(IReadOnlyList values) foreach (short value in values) { Number64 number64 = value; - this.WriteNumber64Value(number64); + this.WriteNumberValue(number64); } this.WriteArrayEnd(); @@ -195,7 +199,7 @@ public virtual void WriteNumberArray(IReadOnlyList values) foreach (int value in values) { Number64 number64 = value; - this.WriteNumber64Value(number64); + this.WriteNumberValue(number64); } this.WriteArrayEnd(); @@ -209,7 +213,7 @@ public virtual void WriteNumberArray(IReadOnlyList values) foreach (long value in values) { Number64 number64 = value; - this.WriteNumber64Value(number64); + this.WriteNumberValue(number64); } this.WriteArrayEnd(); @@ -223,7 +227,7 @@ public virtual void WriteNumberArray(IReadOnlyList values) foreach (float value in values) { Number64 number64 = value; - this.WriteNumber64Value(number64); + this.WriteNumberValue(number64); } this.WriteArrayEnd(); @@ -237,7 +241,7 @@ public virtual void WriteNumberArray(IReadOnlyList values) foreach (double value in values) { Number64 number64 = value; - this.WriteNumber64Value(number64); + this.WriteNumberValue(number64); } this.WriteArrayEnd(); diff --git a/Microsoft.Azure.Cosmos/src/Serializer/CosmosElementSerializer.cs b/Microsoft.Azure.Cosmos/src/Serializer/CosmosElementSerializer.cs index 0564727b0c..8981756db4 100644 --- a/Microsoft.Azure.Cosmos/src/Serializer/CosmosElementSerializer.cs +++ b/Microsoft.Azure.Cosmos/src/Serializer/CosmosElementSerializer.cs @@ -85,7 +85,7 @@ internal static MemoryStream ToStream( // Write the count field and value jsonWriter.WriteFieldName("_count"); - jsonWriter.WriteNumber64Value(count); + jsonWriter.WriteNumberValue(count); jsonWriter.WriteObjectEnd(); diff --git a/Microsoft.Azure.Cosmos/src/Tracing/TraceData/ClientConfigurationTraceDatum.cs b/Microsoft.Azure.Cosmos/src/Tracing/TraceData/ClientConfigurationTraceDatum.cs index afa6aac02c..c9b6904c3e 100644 --- a/Microsoft.Azure.Cosmos/src/Tracing/TraceData/ClientConfigurationTraceDatum.cs +++ b/Microsoft.Azure.Cosmos/src/Tracing/TraceData/ClientConfigurationTraceDatum.cs @@ -105,9 +105,9 @@ private ReadOnlyMemory GetSerializedDatum() jsonTextWriter.WriteStringValue(this.cachedVMRegion); } jsonTextWriter.WriteFieldName("NumberOfClientsCreated"); - jsonTextWriter.WriteNumber64Value(this.cachedNumberOfClientCreated); + jsonTextWriter.WriteNumberValue(this.cachedNumberOfClientCreated); jsonTextWriter.WriteFieldName("NumberOfActiveClients"); - jsonTextWriter.WriteNumber64Value(this.cachedNumberOfActiveClient); + jsonTextWriter.WriteNumberValue(this.cachedNumberOfActiveClient); jsonTextWriter.WriteFieldName("ConnectionMode"); jsonTextWriter.WriteStringValue(this.ConnectionMode.ToString()); jsonTextWriter.WriteFieldName("User Agent"); @@ -128,7 +128,7 @@ private ReadOnlyMemory GetSerializedDatum() jsonTextWriter.WriteFieldName("ConsistencyConfig"); jsonTextWriter.WriteStringValue(this.ConsistencyConfig.ToString()); jsonTextWriter.WriteFieldName("ProcessorCount"); - jsonTextWriter.WriteNumber64Value(this.ProcessorCount); + jsonTextWriter.WriteNumberValue(this.ProcessorCount); jsonTextWriter.WriteObjectEnd(); diff --git a/Microsoft.Azure.Cosmos/src/Tracing/TraceData/SummaryDiagnostics.cs b/Microsoft.Azure.Cosmos/src/Tracing/TraceData/SummaryDiagnostics.cs index 3a50b96841..64f3414529 100644 --- a/Microsoft.Azure.Cosmos/src/Tracing/TraceData/SummaryDiagnostics.cs +++ b/Microsoft.Azure.Cosmos/src/Tracing/TraceData/SummaryDiagnostics.cs @@ -125,7 +125,7 @@ public void WriteSummaryDiagnostics(IJsonWriter jsonWriter) foreach (KeyValuePair<(int, int), int> kvp in this.DirectRequestsSummary.Value) { jsonWriter.WriteFieldName(kvp.Key.ToString()); - jsonWriter.WriteNumber64Value(kvp.Value); + jsonWriter.WriteNumberValue(kvp.Value); } jsonWriter.WriteObjectEnd(); } @@ -133,7 +133,7 @@ public void WriteSummaryDiagnostics(IJsonWriter jsonWriter) if (this.AllRegionsContacted.IsValueCreated) { jsonWriter.WriteFieldName("RegionsContacted"); - jsonWriter.WriteNumber64Value(this.AllRegionsContacted.Value.Count); + jsonWriter.WriteNumberValue(this.AllRegionsContacted.Value.Count); } if (this.GatewayRequestsSummary.IsValueCreated) @@ -143,7 +143,7 @@ public void WriteSummaryDiagnostics(IJsonWriter jsonWriter) foreach (KeyValuePair<(int, int), int> kvp in this.GatewayRequestsSummary.Value) { jsonWriter.WriteFieldName(kvp.Key.ToString()); - jsonWriter.WriteNumber64Value(kvp.Value); + jsonWriter.WriteNumberValue(kvp.Value); } jsonWriter.WriteObjectEnd(); } diff --git a/Microsoft.Azure.Cosmos/src/Tracing/TraceWriter.TraceJsonWriter.cs b/Microsoft.Azure.Cosmos/src/Tracing/TraceWriter.TraceJsonWriter.cs index db7512e82b..b13a1e545b 100644 --- a/Microsoft.Azure.Cosmos/src/Tracing/TraceWriter.TraceJsonWriter.cs +++ b/Microsoft.Azure.Cosmos/src/Tracing/TraceWriter.TraceJsonWriter.cs @@ -50,7 +50,7 @@ public static void WriteTrace( writer.WriteStringValue(trace.StartTime.ToString(TraceWriter.DateTimeFormatString)); } writer.WriteFieldName("duration in milliseconds"); - writer.WriteNumber64Value(trace.Duration.TotalMilliseconds); + writer.WriteNumberValue(trace.Duration.TotalMilliseconds); if (trace.Data.Any()) { @@ -96,11 +96,11 @@ private static void WriteTraceDatum(IJsonWriter writer, object value) } else if (value is double doubleValue) { - writer.WriteNumber64Value(doubleValue); + writer.WriteNumberValue(doubleValue); } else if (value is long longValue) { - writer.WriteNumber64Value(longValue); + writer.WriteNumberValue(longValue); } else if (value is IEnumerable enumerable) { @@ -162,13 +162,13 @@ public void Visit(PointOperationStatisticsTraceDatum pointOperationStatisticsTra this.WriteDateTimeStringValue(pointOperationStatisticsTraceDatum.ResponseTimeUtc); this.jsonWriter.WriteFieldName("StatusCode"); - this.jsonWriter.WriteNumber64Value((int)pointOperationStatisticsTraceDatum.StatusCode); + this.jsonWriter.WriteNumberValue((int)pointOperationStatisticsTraceDatum.StatusCode); this.jsonWriter.WriteFieldName("SubStatusCode"); - this.jsonWriter.WriteNumber64Value((int)pointOperationStatisticsTraceDatum.SubStatusCode); + this.jsonWriter.WriteNumberValue((int)pointOperationStatisticsTraceDatum.SubStatusCode); this.jsonWriter.WriteFieldName("RequestCharge"); - this.jsonWriter.WriteNumber64Value(pointOperationStatisticsTraceDatum.RequestCharge); + this.jsonWriter.WriteNumberValue(pointOperationStatisticsTraceDatum.RequestCharge); this.jsonWriter.WriteFieldName("RequestUri"); this.WriteStringValueOrNull(pointOperationStatisticsTraceDatum.RequestUri); @@ -245,7 +245,7 @@ private void VisitHttpResponseStatistics(ClientSideRequestStatisticsTraceDatum.H this.WriteDateTimeStringValue(stat.RequestStartTime); jsonWriter.WriteFieldName("DurationInMs"); - jsonWriter.WriteNumber64Value(stat.Duration.TotalMilliseconds); + jsonWriter.WriteNumberValue(stat.Duration.TotalMilliseconds); jsonWriter.WriteFieldName("RequestUri"); jsonWriter.WriteStringValue(stat.RequestUri.ToString()); @@ -326,7 +326,7 @@ private void VisitStoreResponseStatistics( if (storeResponseStatistics.RequestStartTime.HasValue) { TimeSpan latency = storeResponseStatistics.RequestResponseTime - storeResponseStatistics.RequestStartTime.Value; - this.jsonWriter.WriteNumber64Value(latency.TotalMilliseconds); + this.jsonWriter.WriteNumberValue(latency.TotalMilliseconds); } else { @@ -397,34 +397,34 @@ public void Visit(StoreResult storeResult) this.jsonWriter.WriteStringValue(storeResult.SubStatusCode.ToString()); this.jsonWriter.WriteFieldName(nameof(storeResult.LSN)); - this.jsonWriter.WriteNumber64Value(storeResult.LSN); + this.jsonWriter.WriteNumberValue(storeResult.LSN); this.jsonWriter.WriteFieldName(nameof(storeResult.PartitionKeyRangeId)); this.WriteStringValueOrNull(storeResult.PartitionKeyRangeId); this.jsonWriter.WriteFieldName(nameof(storeResult.GlobalCommittedLSN)); - this.jsonWriter.WriteNumber64Value(storeResult.GlobalCommittedLSN); + this.jsonWriter.WriteNumberValue(storeResult.GlobalCommittedLSN); this.jsonWriter.WriteFieldName(nameof(storeResult.ItemLSN)); - this.jsonWriter.WriteNumber64Value(storeResult.ItemLSN); + this.jsonWriter.WriteNumberValue(storeResult.ItemLSN); this.jsonWriter.WriteFieldName(nameof(storeResult.UsingLocalLSN)); this.jsonWriter.WriteBoolValue(storeResult.UsingLocalLSN); this.jsonWriter.WriteFieldName(nameof(storeResult.QuorumAckedLSN)); - this.jsonWriter.WriteNumber64Value(storeResult.QuorumAckedLSN); + this.jsonWriter.WriteNumberValue(storeResult.QuorumAckedLSN); this.jsonWriter.WriteFieldName(nameof(storeResult.SessionToken)); this.WriteStringValueOrNull(storeResult.SessionToken?.ConvertToString()); this.jsonWriter.WriteFieldName(nameof(storeResult.CurrentWriteQuorum)); - this.jsonWriter.WriteNumber64Value(storeResult.CurrentWriteQuorum); + this.jsonWriter.WriteNumberValue(storeResult.CurrentWriteQuorum); this.jsonWriter.WriteFieldName(nameof(storeResult.CurrentReplicaSetSize)); - this.jsonWriter.WriteNumber64Value(storeResult.CurrentReplicaSetSize); + this.jsonWriter.WriteNumberValue(storeResult.CurrentReplicaSetSize); this.jsonWriter.WriteFieldName(nameof(storeResult.NumberOfReadRegions)); - this.jsonWriter.WriteNumber64Value(storeResult.NumberOfReadRegions); + this.jsonWriter.WriteNumberValue(storeResult.NumberOfReadRegions); this.jsonWriter.WriteFieldName(nameof(storeResult.IsValid)); this.jsonWriter.WriteBoolValue(storeResult.IsValid); @@ -433,7 +433,7 @@ public void Visit(StoreResult storeResult) this.WriteStringValueOrNull(storeResult.StorePhysicalAddress?.ToString()); this.jsonWriter.WriteFieldName(nameof(storeResult.RequestCharge)); - this.jsonWriter.WriteNumber64Value(storeResult.RequestCharge); + this.jsonWriter.WriteNumberValue(storeResult.RequestCharge); this.jsonWriter.WriteFieldName(nameof(storeResult.RetryAfterInMs)); this.WriteStringValueOrNull(storeResult.RetryAfterInMs); @@ -567,7 +567,7 @@ private void WriteJsonUriArrayWithDuplicatesCounted(string propertyName, IReadOn { this.jsonWriter.WriteObjectStart(); this.jsonWriter.WriteFieldName("Count"); - this.jsonWriter.WriteNumber64Value(contactedCount.Value); + this.jsonWriter.WriteNumberValue(contactedCount.Value); this.jsonWriter.WriteFieldName("Uri"); this.WriteStringValueOrNull(contactedCount.Key.ToString()); this.jsonWriter.WriteObjectEnd(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs index eea629aa33..5fe7f28c87 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs @@ -104,7 +104,7 @@ private static async Task UntypedTests(Container container) IJsonNavigatorNode rootNode = navigator.GetRootNode(); Assert.IsTrue(navigator.TryGetObjectProperty(rootNode, "_count", out ObjectProperty countProperty)); - long count = Number64.ToLong(navigator.GetNumber64Value(countProperty.ValueNode)); + long count = Number64.ToLong(navigator.GetNumberValue(countProperty.ValueNode)); actualCount += count; Assert.IsTrue(navigator.TryGetObjectProperty(rootNode, "Documents", out ObjectProperty documentsProperty)); @@ -700,7 +700,7 @@ public override string ToString() writer.WriteObjectStart(); writer.WriteFieldName(nameof(this.Index)); - writer.WriteNumber64Value(this.Index); + writer.WriteNumberValue(this.Index); if (this.MixedTypeField is not CosmosUndefined) { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/DistributedQueryClientTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/DistributedQueryClientTests.cs index 3901f4287e..d25808911a 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/DistributedQueryClientTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/DistributedQueryClientTests.cs @@ -328,9 +328,9 @@ private static async Task RunStreamIteratorTestsAsync(Container container, IEnum IEnumerable arrayItems = navigator.GetArrayItems(documentsProperty.ValueNode); foreach (IJsonNavigatorNode node in arrayItems) { - Assert.AreEqual(JsonNodeType.Number64, navigator.GetNodeType(node)); + Assert.AreEqual(JsonNodeType.Number, navigator.GetNodeType(node)); - extractedResults.Add((int)Number64.ToLong(navigator.GetNumber64Value(node))); + extractedResults.Add((int)Number64.ToLong(navigator.GetNumberValue(node))); } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Json/JsonMicroBenchmarksBase.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Json/JsonMicroBenchmarksBase.cs index 66c85486f3..56461dbe27 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Json/JsonMicroBenchmarksBase.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Json/JsonMicroBenchmarksBase.cs @@ -15,8 +15,8 @@ protected static class WriteDelegates internal static readonly Action Null = (jsonWriter) => { jsonWriter.WriteNullValue(); }; internal static readonly Action True = (jsonWriter) => { jsonWriter.WriteBoolValue(true); }; internal static readonly Action False = (jsonWriter) => { jsonWriter.WriteBoolValue(false); }; - internal static readonly Action Number64Integer = (jsonWriter) => { jsonWriter.WriteNumber64Value(123); }; - internal static readonly Action Number64Double = (jsonWriter) => { jsonWriter.WriteNumber64Value(6.0221409e+23); }; + internal static readonly Action Number64Integer = (jsonWriter) => { jsonWriter.WriteNumberValue(123); }; + internal static readonly Action Number64Double = (jsonWriter) => { jsonWriter.WriteNumberValue(6.0221409e+23); }; internal static readonly Action String = (jsonWriter) => { jsonWriter.WriteStringValue("Hello World"); }; internal static readonly Action Array = (jsonWriter) => { jsonWriter.WriteArrayStart(); jsonWriter.WriteArrayEnd(); }; internal static readonly Action Object = (jsonWriter) => { jsonWriter.WriteObjectStart(); jsonWriter.WriteObjectEnd(); }; diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Json/Utils.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Json/Utils.cs index 68ecf03b3b..57e2f961cf 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Json/Utils.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Performance.Tests/Json/Utils.cs @@ -141,7 +141,7 @@ public static void FlushToWriter(IJsonWriter jsonWriter, IReadOnlyList buffer = jsonWriter.GetResult(); @@ -52,7 +52,7 @@ public void ArrayTest() jsonWriter.WriteFieldName("name"); jsonWriter.WriteStringValue("John"); jsonWriter.WriteFieldName("age"); - jsonWriter.WriteNumber64Value(24); + jsonWriter.WriteNumberValue(24); jsonWriter.WriteObjectEnd(); jsonWriter.WriteArrayEnd(); ReadOnlyMemory buffer = jsonWriter.GetResult(); @@ -160,7 +160,7 @@ public void NumberTest() { int integerValue = 42; IJsonWriter integerWriter = JsonWriter.Create(JsonSerializationFormat.Binary); - integerWriter.WriteNumber64Value(integerValue); + integerWriter.WriteNumberValue(integerValue); ReadOnlyMemory integerBuffer = integerWriter.GetResult(); { @@ -229,7 +229,7 @@ public void NumberTest() float doubleValue = 42.1337f; IJsonWriter doubleWriter = JsonWriter.Create(JsonSerializationFormat.Binary); - doubleWriter.WriteNumber64Value(doubleValue); + doubleWriter.WriteNumberValue(doubleValue); ReadOnlyMemory doubleBuffer = doubleWriter.GetResult(); { @@ -301,7 +301,7 @@ public void ObjectTest() jsonWriter.WriteStringValue("John"); jsonWriter.WriteFieldName("age"); - jsonWriter.WriteNumber64Value(24); + jsonWriter.WriteNumberValue(24); jsonWriter.WriteObjectEnd(); ReadOnlyMemory buffer = jsonWriter.GetResult(); diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonNavigatorTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonNavigatorTests.cs index 68cca1d38f..f1fbd1b270 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonNavigatorTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonNavigatorTests.cs @@ -572,7 +572,7 @@ private static JsonToken[] GetTokensFromNode(IJsonNavigatorNode node, IJsonNavig JsonNodeType.Null => new JsonToken[] { JsonToken.Null() }, JsonNodeType.False => new JsonToken[] { JsonToken.Boolean(false) }, JsonNodeType.True => new JsonToken[] { JsonToken.Boolean(true) }, - JsonNodeType.Number64 => new JsonToken[] { JsonToken.Number(navigator.GetNumber64Value(node)) }, + JsonNodeType.Number => new JsonToken[] { JsonToken.Number(navigator.GetNumberValue(node)) }, JsonNodeType.String => new JsonToken[] { JsonToken.String(navigator.GetStringValue(node)) }, JsonNodeType.Array => JsonNavigatorTests.GetTokensFromArrayNode(node, navigator, performCorrectnessCheck), JsonNodeType.Object => JsonNavigatorTests.GetTokensFromObjectNode(node, navigator, performCorrectnessCheck), diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonNewtonsoftNavigator.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonNewtonsoftNavigator.cs index 6ee09b8317..370fa36148 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonNewtonsoftNavigator.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonNewtonsoftNavigator.cs @@ -28,6 +28,7 @@ public JsonNewtonsoftNavigator(string input) this.root = new NewtonsoftNode(rootJToken, this.JTokenToJsonNodeType(rootJToken)); } + #region IJsonNavigator public override JsonSerializationFormat SerializationFormat => JsonSerializationFormat.Text; public override IJsonNavigatorNode GetArrayItemAt(IJsonNavigatorNode arrayNode, int index) @@ -56,27 +57,12 @@ public override IEnumerable GetArrayItems(IJsonNavigatorNode } } - private JsonNodeType JTokenToJsonNodeType(JToken jToken) - { - return jToken.Type switch - { - JTokenType.Object => JsonNodeType.Object, - JTokenType.Array => JsonNodeType.Array, - JTokenType.Integer or JTokenType.Float => JsonNodeType.Number64, - JTokenType.String => JsonNodeType.String, - JTokenType.Boolean => ((bool)jToken) ? JsonNodeType.True : JsonNodeType.False, - JTokenType.Null or JTokenType.Undefined => JsonNodeType.Null, - JTokenType.Constructor or JTokenType.Property or JTokenType.Comment or JTokenType.Date or JTokenType.Raw or JTokenType.Bytes or JTokenType.Guid or JTokenType.Uri or JTokenType.TimeSpan => JsonNodeType.String, - _ => throw new InvalidOperationException(), - }; - } - public override JsonNodeType GetNodeType(IJsonNavigatorNode node) { return ((NewtonsoftNode)node).JsonNodeType; } - public override Number64 GetNumber64Value(IJsonNavigatorNode numberNode) + public override Number64 GetNumberValue(IJsonNavigatorNode numberNode) { return (double)((NewtonsoftNode)numberNode).JToken; } @@ -184,6 +170,29 @@ public override IJsonReader CreateReader(IJsonNavigatorNode jsonNavigatorNode) { throw new NotImplementedException(); } + #endregion + + /// + protected override bool TryGetUInt64Value(IJsonNavigatorNode numberNode, out ulong value) + { + value = 0; + return false; + } + + private JsonNodeType JTokenToJsonNodeType(JToken jToken) + { + return jToken.Type switch + { + JTokenType.Object => JsonNodeType.Object, + JTokenType.Array => JsonNodeType.Array, + JTokenType.Integer or JTokenType.Float => JsonNodeType.Number, + JTokenType.String => JsonNodeType.String, + JTokenType.Boolean => ((bool)jToken) ? JsonNodeType.True : JsonNodeType.False, + JTokenType.Null or JTokenType.Undefined => JsonNodeType.Null, + JTokenType.Constructor or JTokenType.Property or JTokenType.Comment or JTokenType.Date or JTokenType.Raw or JTokenType.Bytes or JTokenType.Guid or JTokenType.Uri or JTokenType.TimeSpan => JsonNodeType.String, + _ => throw new InvalidOperationException(), + }; + } private readonly struct NewtonsoftNode : IJsonNavigatorNode { diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonReaderTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonReaderTests.cs index 6d3d38a795..e0d33c2513 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonReaderTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonReaderTests.cs @@ -560,7 +560,7 @@ public void StringTest() byte[] binary1ByteLengthInput = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)"Hello World".Length, // Hello World as a utf8 string 72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100 @@ -569,7 +569,7 @@ public void StringTest() byte[] binary2ByteLengthInput = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String2ByteLength, + JsonBinaryEncoding.TypeMarker.StrL2, // (ushort)"Hello World".Length, 0x0B, 0x00, // Hello World as a utf8 string @@ -579,7 +579,7 @@ public void StringTest() byte[] binary4ByteLengthInput = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String4ByteLength, + JsonBinaryEncoding.TypeMarker.StrL4, // (uint)"Hello World".Length, 0x0B, 0x00, 0x00, 0x00, // Hello World as a utf8 string @@ -702,7 +702,7 @@ public void GuidStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)guidString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(guidString)).ToArray(); @@ -732,7 +732,7 @@ public void GuidStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)guidString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(guidString)).ToArray(); @@ -762,7 +762,7 @@ public void GuidStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)guidString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(guidString)).ToArray(); @@ -792,7 +792,7 @@ public void GuidStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)guidString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(guidString)).ToArray(); @@ -822,7 +822,7 @@ public void GuidStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)guidString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(guidString)).ToArray(); @@ -852,7 +852,7 @@ public void GuidStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)guidString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(guidString)).ToArray(); @@ -882,7 +882,7 @@ public void GuidStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)guidString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(guidString)).ToArray(); @@ -912,7 +912,7 @@ public void GuidStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)guidString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(guidString)).ToArray(); @@ -933,7 +933,7 @@ public void GuidStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)guidString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(guidString)).ToArray(); @@ -963,7 +963,7 @@ public void GuidStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)guidString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(guidString)).ToArray(); @@ -997,7 +997,7 @@ public void DateTimeStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)dateTimeString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(dateTimeString)).ToArray(); @@ -1025,7 +1025,7 @@ public void DateTimeStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)dateTimeString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(dateTimeString)).ToArray(); @@ -1053,7 +1053,7 @@ public void DateTimeStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)dateTimeString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(dateTimeString)).ToArray(); @@ -1081,7 +1081,7 @@ public void DateTimeStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)dateTimeString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(dateTimeString)).ToArray(); @@ -1109,7 +1109,7 @@ public void DateTimeStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)dateTimeString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(dateTimeString)).ToArray(); @@ -1137,7 +1137,7 @@ public void DateTimeStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)dateTimeString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(dateTimeString)).ToArray(); @@ -1165,7 +1165,7 @@ public void DateTimeStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)dateTimeString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(dateTimeString)).ToArray(); @@ -1198,7 +1198,7 @@ public void HexStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)hexString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(hexString)).ToArray(); @@ -1226,7 +1226,7 @@ public void HexStringsTest() byte[] binaryPayload = { BinaryFormat, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)hexString.Length, }; binaryPayload = binaryPayload.Concat(Encoding.UTF8.GetBytes(hexString)).ToArray(); @@ -1291,11 +1291,11 @@ public void ReferenceStringTests() BinaryFormat, JsonBinaryEncoding.TypeMarker.ArrL1, 9, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)"hello".Length, (byte)'h', (byte)'e', (byte)'l', (byte)'l', (byte)'o', - JsonBinaryEncoding.TypeMarker.ReferenceString1ByteOffset, + JsonBinaryEncoding.TypeMarker.StrR1, 3, }; @@ -1309,11 +1309,11 @@ public void ReferenceStringTests() BinaryFormat, JsonBinaryEncoding.TypeMarker.ArrL1, 10, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)"hello".Length, (byte)'h', (byte)'e', (byte)'l', (byte)'l', (byte)'o', - JsonBinaryEncoding.TypeMarker.ReferenceString2ByteOffset, + JsonBinaryEncoding.TypeMarker.StrR2, 3, 0, }; @@ -1327,11 +1327,11 @@ public void ReferenceStringTests() BinaryFormat, JsonBinaryEncoding.TypeMarker.ArrL1, 11, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)"hello".Length, (byte)'h', (byte)'e', (byte)'l', (byte)'l', (byte)'o', - JsonBinaryEncoding.TypeMarker.ReferenceString3ByteOffset, + JsonBinaryEncoding.TypeMarker.StrR3, 3, 0, 0 }; @@ -1345,11 +1345,11 @@ public void ReferenceStringTests() BinaryFormat, JsonBinaryEncoding.TypeMarker.ArrL1, 12, - JsonBinaryEncoding.TypeMarker.String1ByteLength, + JsonBinaryEncoding.TypeMarker.StrL1, (byte)"hello".Length, (byte)'h', (byte)'e', (byte)'l', (byte)'l', (byte)'o', - JsonBinaryEncoding.TypeMarker.ReferenceString4ByteOffset, + JsonBinaryEncoding.TypeMarker.StrR4, 3, 0, 0, 0 }; @@ -1739,7 +1739,7 @@ public void StringArrayTest() { new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.EncodedStringLengthMin + "Hello".Length) }, Encoding.UTF8.GetBytes("Hello"), - new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength }, + new byte[] { JsonBinaryEncoding.TypeMarker.StrL1 }, new byte[] { (byte)"World".Length }, Encoding.UTF8.GetBytes("World"), new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.EncodedStringLengthMin + "Bye".Length) }, @@ -1867,7 +1867,7 @@ public void AllPrimitiveArrayTest() new byte[] { JsonBinaryEncoding.TypeMarker.NumberDouble, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xBF }, new byte[] { JsonBinaryEncoding.TypeMarker.LiteralIntMin + 1 }, new byte[] { JsonBinaryEncoding.TypeMarker.LiteralIntMin + 2 }, - new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"hello".Length, 104, 101, 108, 108, 111 }, + new byte[] { JsonBinaryEncoding.TypeMarker.StrL1, (byte)"hello".Length, 104, 101, 108, 108, 111 }, new byte[] { JsonBinaryEncoding.TypeMarker.Null }, new byte[] { JsonBinaryEncoding.TypeMarker.True }, new byte[] { JsonBinaryEncoding.TypeMarker.False } @@ -2384,7 +2384,7 @@ public void AllPrimitivesObjectTest() elements.Add(innerObjectElementsBytes); elements.Add(new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.EncodedStringLengthMin + "text".Length), 116, 101, 120, 116 }); - elements.Add(new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"tiger diamond newbrunswick snowleopard chocolate dog snowleopard turtle cat sapphire peach sapphire vancouver white chocolate horse diamond lion superlongcolourname ruby".Length, 116, 105, 103, 101, 114, 32, 100, 105, 97, 109, 111, 110, 100, 32, 110, 101, 119, 98, 114, 117, 110, 115, 119, 105, 99, 107, 32, 115, 110, 111, 119, 108, 101, 111, 112, 97, 114, 100, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 32, 100, 111, 103, 32, 115, 110, 111, 119, 108, 101, 111, 112, 97, 114, 100, 32, 116, 117, 114, 116, 108, 101, 32, 99, 97, 116, 32, 115, 97, 112, 112, 104, 105, 114, 101, 32, 112, 101, 97, 99, 104, 32, 115, 97, 112, 112, 104, 105, 114, 101, 32, 118, 97, 110, 99, 111, 117, 118, 101, 114, 32, 119, 104, 105, 116, 101, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 32, 104, 111, 114, 115, 101, 32, 100, 105, 97, 109, 111, 110, 100, 32, 108, 105, 111, 110, 32, 115, 117, 112, 101, 114, 108, 111, 110, 103, 99, 111, 108, 111, 117, 114, 110, 97, 109, 101, 32, 114, 117, 98, 121 }); + elements.Add(new byte[] { JsonBinaryEncoding.TypeMarker.StrL1, (byte)"tiger diamond newbrunswick snowleopard chocolate dog snowleopard turtle cat sapphire peach sapphire vancouver white chocolate horse diamond lion superlongcolourname ruby".Length, 116, 105, 103, 101, 114, 32, 100, 105, 97, 109, 111, 110, 100, 32, 110, 101, 119, 98, 114, 117, 110, 115, 119, 105, 99, 107, 32, 115, 110, 111, 119, 108, 101, 111, 112, 97, 114, 100, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 32, 100, 111, 103, 32, 115, 110, 111, 119, 108, 101, 111, 112, 97, 114, 100, 32, 116, 117, 114, 116, 108, 101, 32, 99, 97, 116, 32, 115, 97, 112, 112, 104, 105, 114, 101, 32, 112, 101, 97, 99, 104, 32, 115, 97, 112, 112, 104, 105, 114, 101, 32, 118, 97, 110, 99, 111, 117, 118, 101, 114, 32, 119, 104, 105, 116, 101, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 32, 104, 111, 114, 115, 101, 32, 100, 105, 97, 109, 111, 110, 100, 32, 108, 105, 111, 110, 32, 115, 117, 112, 101, 114, 108, 111, 110, 103, 99, 111, 108, 111, 117, 114, 110, 97, 109, 101, 32, 114, 117, 98, 121 }); byte[] elementsBytes = elements.SelectMany(x => x).ToArray(); @@ -2448,7 +2448,7 @@ public void AllPrimitivesObjectTest() elements.Add(innerObjectElementsBytes); elements.Add(new byte[] { (byte)(JsonBinaryEncoding.TypeMarker.UserString1ByteLengthMin + 7) }); - elements.Add(new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"tiger diamond newbrunswick snowleopard chocolate dog snowleopard turtle cat sapphire peach sapphire vancouver white chocolate horse diamond lion superlongcolourname ruby".Length, 116, 105, 103, 101, 114, 32, 100, 105, 97, 109, 111, 110, 100, 32, 110, 101, 119, 98, 114, 117, 110, 115, 119, 105, 99, 107, 32, 115, 110, 111, 119, 108, 101, 111, 112, 97, 114, 100, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 32, 100, 111, 103, 32, 115, 110, 111, 119, 108, 101, 111, 112, 97, 114, 100, 32, 116, 117, 114, 116, 108, 101, 32, 99, 97, 116, 32, 115, 97, 112, 112, 104, 105, 114, 101, 32, 112, 101, 97, 99, 104, 32, 115, 97, 112, 112, 104, 105, 114, 101, 32, 118, 97, 110, 99, 111, 117, 118, 101, 114, 32, 119, 104, 105, 116, 101, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 32, 104, 111, 114, 115, 101, 32, 100, 105, 97, 109, 111, 110, 100, 32, 108, 105, 111, 110, 32, 115, 117, 112, 101, 114, 108, 111, 110, 103, 99, 111, 108, 111, 117, 114, 110, 97, 109, 101, 32, 114, 117, 98, 121 }); + elements.Add(new byte[] { JsonBinaryEncoding.TypeMarker.StrL1, (byte)"tiger diamond newbrunswick snowleopard chocolate dog snowleopard turtle cat sapphire peach sapphire vancouver white chocolate horse diamond lion superlongcolourname ruby".Length, 116, 105, 103, 101, 114, 32, 100, 105, 97, 109, 111, 110, 100, 32, 110, 101, 119, 98, 114, 117, 110, 115, 119, 105, 99, 107, 32, 115, 110, 111, 119, 108, 101, 111, 112, 97, 114, 100, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 32, 100, 111, 103, 32, 115, 110, 111, 119, 108, 101, 111, 112, 97, 114, 100, 32, 116, 117, 114, 116, 108, 101, 32, 99, 97, 116, 32, 115, 97, 112, 112, 104, 105, 114, 101, 32, 112, 101, 97, 99, 104, 32, 115, 97, 112, 112, 104, 105, 114, 101, 32, 118, 97, 110, 99, 111, 117, 118, 101, 114, 32, 119, 104, 105, 116, 101, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 32, 104, 111, 114, 115, 101, 32, 100, 105, 97, 109, 111, 110, 100, 32, 108, 105, 111, 110, 32, 115, 117, 112, 101, 114, 108, 111, 110, 103, 99, 111, 108, 111, 117, 114, 110, 97, 109, 101, 32, 114, 117, 98, 121 }); byte[] elementsBytes = elements.SelectMany(x => x).ToArray(); @@ -2516,20 +2516,20 @@ public void TrailingGarbageTest() List elements = new List { - new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"name".Length, 110, 97, 109, 101 }, - new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"477cecf7-5547-4f87-81c2-72ee2c7d6179".Length, 52, 55, 55, 99, 101, 99, 102, 55, 45, 53, 53, 52, 55, 45, 52, 102, 56, 55, 45, 56, 49, 99, 50, 45, 55, 50, 101, 101, 50, 99, 55, 100, 54, 49, 55, 57 }, + new byte[] { JsonBinaryEncoding.TypeMarker.StrL1, (byte)"name".Length, 110, 97, 109, 101 }, + new byte[] { JsonBinaryEncoding.TypeMarker.StrL1, (byte)"477cecf7-5547-4f87-81c2-72ee2c7d6179".Length, 52, 55, 55, 99, 101, 99, 102, 55, 45, 53, 53, 52, 55, 45, 52, 102, 56, 55, 45, 56, 49, 99, 50, 45, 55, 50, 101, 101, 50, 99, 55, 100, 54, 49, 55, 57 }, - new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"permissionMode".Length, 112, 101, 114, 109, 105, 115, 115, 105, 111, 110, 77, 111, 100, 101 }, - new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"Read".Length, 82, 101, 97, 100 }, + new byte[] { JsonBinaryEncoding.TypeMarker.StrL1, (byte)"permissionMode".Length, 112, 101, 114, 109, 105, 115, 115, 105, 111, 110, 77, 111, 100, 101 }, + new byte[] { JsonBinaryEncoding.TypeMarker.StrL1, (byte)"Read".Length, 82, 101, 97, 100 }, - new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"resource".Length, 114, 101, 115, 111, 117, 114, 99, 101 }, - new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"-iQET8M3A0c=".Length, 45, 105, 81, 69, 84, 56, 77, 51, 65, 48, 99, 61 } + new byte[] { JsonBinaryEncoding.TypeMarker.StrL1, (byte)"resource".Length, 114, 101, 115, 111, 117, 114, 99, 101 }, + new byte[] { JsonBinaryEncoding.TypeMarker.StrL1, (byte)"-iQET8M3A0c=".Length, 45, 105, 81, 69, 84, 56, 77, 51, 65, 48, 99, 61 } }; byte[] elementsBytes = elements.SelectMany(x => x).ToArray(); binaryInputBuilder.Add(BitConverter.GetBytes(elementsBytes.Length)); binaryInputBuilder.Add(elementsBytes); - binaryInputBuilder.Add(new byte[] { JsonBinaryEncoding.TypeMarker.String1ByteLength, (byte)"..garbage..".Length, 46, 46, 103, 97, 114, 98, 97, 103, 101, 46, 46 }); + binaryInputBuilder.Add(new byte[] { JsonBinaryEncoding.TypeMarker.StrL1, (byte)"..garbage..".Length, 46, 46, 103, 97, 114, 98, 97, 103, 101, 46, 46 }); byte[] binaryInput = binaryInputBuilder.SelectMany(x => x).ToArray(); JsonToken[] expectedTokens = diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonRoundtripTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonRoundtripTests.cs index 403d8c43d6..df38deaebe 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonRoundtripTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonRoundtripTests.cs @@ -497,7 +497,7 @@ public void MillionSong1KDocumentsTest() [Owner("mayapainter")] public void MsnCollectionTest() { - this.VerifyCuratedJsonRoundTripTest("MsnCollection.json"); + this.VerifyCuratedJsonRoundTripTest("MsnCollection.json", expectRefStringDiffs: true); } [TestMethod] @@ -595,19 +595,19 @@ private static void VerifyRoundTripTest(string inputJson) { // Do the actual roundtrips JsonToken[] inputTokens = JsonTestUtils.ReadJsonDocument(inputJson); - JsonRoundTripsTests.MultiSerializationRoundTrip(inputTokens, inputJson); + JsonRoundTripsTests.MultiSerializationRoundTrip(inputTokens, inputJson, expectRefStringDiffs: false); } // Checks to see if we can go from a JsonReader to a NewtonsoftWriter and get back the original document and visa versa - private void VerifyCuratedJsonRoundTripTest(string filename, int maxNumberOfItems = 100) + private void VerifyCuratedJsonRoundTripTest(string filename, int maxNumberOfItems = 100, bool expectRefStringDiffs = false) { string inputJson = JsonTestUtils.LoadJsonCuratedDocument(filename); inputJson = JsonTestUtils.RandomSampleJson(inputJson, maxNumberOfItems, seed: 42); JsonToken[] inputTokens = JsonTestUtils.ReadJsonDocument(inputJson); - MultiSerializationRoundTrip(inputTokens, inputJson); + MultiSerializationRoundTrip(inputTokens, inputJson, expectRefStringDiffs); } - private static void MultiSerializationRoundTrip(JsonToken[] inputTokens, string inputJson) + private static void MultiSerializationRoundTrip(JsonToken[] inputTokens, string inputJson, bool expectRefStringDiffs) { { // Verify native Cosmos formats and write options round-trips @@ -615,7 +615,7 @@ private static void MultiSerializationRoundTrip(JsonToken[] inputTokens, string { SerializationSpec.Text(JsonWriteOptions.None), SerializationSpec.Binary(JsonWriteOptions.None), - SerializationSpec.Binary(JsonWriteOptions.EnableNumberArrays), + SerializationSpec.Binary(JsonWriteOptions.EnableNumberArrays | JsonWriteOptions.EnableUInt64Values) }; RewriteScenario[] rewriteScenarios = @@ -626,7 +626,7 @@ private static void MultiSerializationRoundTrip(JsonToken[] inputTokens, string RewriteScenario.ReaderToken, }; - MultiSerializationRoundTrip(inputTokens, inputJson, serializationSpecs, rewriteScenarios); + MultiSerializationRoundTrip(inputTokens, inputJson, serializationSpecs, rewriteScenarios, expectRefStringDiffs); } { @@ -643,7 +643,7 @@ private static void MultiSerializationRoundTrip(JsonToken[] inputTokens, string RewriteScenario.ReaderToken, }; - MultiSerializationRoundTrip(inputTokens, inputJson, serializationSpecs, rewriteScenarios); + MultiSerializationRoundTrip(inputTokens, inputJson, serializationSpecs, rewriteScenarios, expectRefStringDiffs); } } @@ -651,7 +651,8 @@ private static void MultiSerializationRoundTrip( JsonToken[] inputTokens, string inputJson, SerializationSpec[] serializationSpecs, - RewriteScenario[] rewriteScenarios) + RewriteScenario[] rewriteScenarios, + bool expectRefStringDiffs) { Console.WriteLine(); Console.WriteLine($"Input JSON Length: {inputJson.Length}"); @@ -683,6 +684,7 @@ private static void MultiSerializationRoundTrip( Console.WriteLine($" -- Output Format '{outputSpec.SerializationFormatToString()}'"); + bool strictComparison = false; RoundTripResult roundTripResult = null; foreach (RewriteScenario scenario in rewriteScenarios) { @@ -692,7 +694,7 @@ private static void MultiSerializationRoundTrip( inputSpec, outputSpec, scenario, - expectedOutputResults[i], + new RoundTripBaseline(expectedOutputResults[i], strictComparison), (string _) => new JsonNewtonsoftNavigator(_)); expectedOutputResults[i] = roundTripResult.OutputResult; @@ -700,6 +702,8 @@ private static void MultiSerializationRoundTrip( Console.WriteLine($" Scenario '{scenario}'"); Console.WriteLine($" Execution Time (ms): {roundTripResult.ExecutionTime,5}"); Console.WriteLine($" Verification Time (ms): {roundTripResult.VerificationTime,5}"); + + strictComparison = !expectRefStringDiffs || (outputSpec.SerializationFormat != JsonSerializationFormat.Binary); } } } diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonTestUtils.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonTestUtils.cs index 8eafcda026..1e7f4f6616 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonTestUtils.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonTestUtils.cs @@ -214,8 +214,15 @@ public static void WriteTokens( break; case JsonTokenType.Number: - Number64 numberValue = (token as JsonNumberToken).Value; - jsonWriter.WriteNumber64Value(numberValue); + if(token is JsonUInt64NumberToken uint64Token) + { + jsonWriter.WriteNumberValue(uint64Token.Value); + } + else + { + Number64 numberValue = (token as JsonNumberToken).Value; + jsonWriter.WriteNumberValue(numberValue); + } break; case JsonTokenType.True: @@ -302,7 +309,7 @@ public static RoundTripResult VerifyJsonRoundTrip( SerializationSpec inputSpec, SerializationSpec outputSpec, RewriteScenario rewriteScenario, - ReadOnlyMemory expectedOutputResult = default, + RoundTripBaseline roundTripBaseline = null, Func newtonsoftNavigatorCreate = default) { Func createReader = (SerializationSpec spec) => spec.IsNewtonsoft ? @@ -394,11 +401,15 @@ public static RoundTripResult VerifyJsonRoundTrip( byte[] inputBytes = inputResult.ToArray(); byte[] outputBytes = outputResult.ToArray(); - byte[] expectedOutputBytes = expectedOutputResult.ToArray(); + byte[] expectedBytes = roundTripBaseline?.ExpectedResult.ToArray(); + + bool strictComparison = (roundTripBaseline != null) && roundTripBaseline.StrictComparison; + + bool identical = (expectedBytes != null) && outputBytes.SequenceEqual(expectedBytes); StringBuilder verboseOutput = new StringBuilder(); - if (!outputBytes.SequenceEqual(expectedOutputBytes) && - !CompareResults(inputBytes, outputBytes, verboseWriter: new StringWriter(verboseOutput))) + if (!identical && + (strictComparison || !CompareResults(inputBytes, outputBytes, verboseWriter: new StringWriter(verboseOutput)))) { string[] inputTextLines = SerializeResultBuffer(inputBytes, inputSpec.SerializationFormat); string[] outputTextLines = SerializeResultBuffer(outputBytes, outputSpec.SerializationFormat); @@ -421,6 +432,18 @@ public static RoundTripResult VerifyJsonRoundTrip( Console.WriteLine("Output Result:"); foreach (string line in outputTextLines) Console.WriteLine(line); + Console.WriteLine(); + Console.WriteLine("Expected Result:"); + if(expectedBytes != null) + { + string[] expectedTextLines = SerializeResultBuffer(expectedBytes, outputSpec.SerializationFormat); + foreach (string line in expectedTextLines) Console.WriteLine(line); + } + else + { + Console.WriteLine(""); + } + Assert.Fail(); } @@ -806,6 +829,18 @@ public string SerializationFormatToString() public bool IsNewtonsoft { get; } } + public class RoundTripBaseline + { + public RoundTripBaseline(ReadOnlyMemory expectedResult, bool strictComparison) + { + this.ExpectedResult = expectedResult; + this.StrictComparison = strictComparison; + } + + public ReadOnlyMemory ExpectedResult { get; } + public bool StrictComparison { get; } + } + public class RoundTripResult { public RoundTripResult(ReadOnlyMemory outputResult, long executionTime, long verificationTime) diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonTokenInfo.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonTokenInfo.cs index 76092c6736..273817c3f1 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonTokenInfo.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonTokenInfo.cs @@ -46,6 +46,11 @@ public static JsonToken Number(Number64 number) return new JsonNumberToken(number); } + public static JsonToken Number(ulong value) + { + return new JsonUInt64NumberToken(value); + } + public static JsonToken Boolean(bool value) { return value ? (JsonToken)new JsonTrueToken() : (JsonToken)new JsonFalseToken(); @@ -248,7 +253,7 @@ public override int GetHashCode() } } - internal sealed class JsonNumberToken : JsonToken + internal class JsonNumberToken : JsonToken { public JsonNumberToken(Number64 value) : base(JsonTokenType.Number) @@ -277,6 +282,35 @@ public override int GetHashCode() } } + internal sealed class JsonUInt64NumberToken : JsonNumberToken + { + public JsonUInt64NumberToken(ulong value) + : base(value <= long.MaxValue ? (Number64)(long)value : (Number64)(double)value) + { + this.Value = value; + } + + public new ulong Value + { + get; + } + + public override bool Equals(object obj) + { + if (obj is JsonUInt64NumberToken other) + { + return this.Value == other.Value; + } + + return base.Equals(obj); + } + + public override int GetHashCode() + { + return 0; + } + } + internal sealed class JsonTrueToken : JsonToken { public JsonTrueToken() diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonWriterTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonWriterTests.cs index fe660cf10e..32c108cafc 100644 --- a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonWriterTests.cs +++ b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Json/JsonWriterTests.cs @@ -8,6 +8,7 @@ using System.Text; using Microsoft.Azure.Cosmos.Json; using Microsoft.VisualStudio.TestTools.UnitTesting; + using Newtonsoft.Json.Linq; using static Microsoft.Azure.Cosmos.Tests.Json.JsonTestUtils; [TestClass] @@ -330,6 +331,212 @@ public void LargeNumbersTest() this.VerifyWriter(tokensToWrite, expectedString); } + + [TestMethod] + [Owner("sboshra")] + public void UInt64Test() + { + // ------------------------- + // Max UINT64 value + // ------------------------- + { + JsonToken[] tokensToWrite = + { + JsonToken.Number(ulong.MaxValue), + }; + + string[] expectedText = + { + @"18446744073709551615" + }; + + string[] expectedBinary1 = + { + "00000000 80 CC 00 00 00 00 00 00 F0 43" + }; + + string[] expectedBinary2 = + { + "00000000 80 C7 FF FF FF FF FF FF FF FF" + }; + + ExecuteAndValidate(tokensToWrite, expectedText, expectedBinary1, JsonWriteOptions.None); + ExecuteAndValidate(tokensToWrite, expectedText, expectedBinary2, JsonWriteOptions.EnableUInt64Values); + } + + // ------------------------- + // Signed integer max values + // ------------------------- + { + JsonToken[] tokensToWrite = + { + JsonToken.ArrayStart(), + JsonToken.Number(0), + JsonToken.Number(sbyte.MaxValue), + JsonToken.Number(short.MaxValue), + JsonToken.Number(int.MaxValue), + JsonToken.Number(long.MaxValue), + JsonToken.ArrayEnd(), + }; + + string[] expectedText = + { + @"[0,127,32767,2147483647,9223372036854775807]" + }; + + string[] expectedBinary = + { + "00000000 80 E2 14 00 C8 7F C9 FF 7F CA FF FF FF 7F CB FF", + "00000010 FF FF FF FF FF FF 7F" + }; + + ExecuteAndValidate(tokensToWrite, expectedText, expectedBinary, JsonWriteOptions.None); + ExecuteAndValidate(tokensToWrite, expectedText, expectedBinary, JsonWriteOptions.EnableUInt64Values); + } + + // ------------------------- + // Unsigned integer max values + // ------------------------- + { + JsonToken[] tokensToWrite = + { + JsonToken.ArrayStart(), + JsonToken.Number(0), + JsonToken.Number(byte.MaxValue), + JsonToken.Number(ushort.MaxValue), + JsonToken.Number(uint.MaxValue), + JsonToken.Number(ulong.MaxValue), + JsonToken.ArrayEnd(), + }; + + string[] expectedText = + { + @"[0,255,65535,4294967295,18446744073709551615]" + }; + + string[] expectedBinary1 = + { + "00000000 80 E2 1A 00 C8 FF CA FF FF 00 00 CB FF FF FF FF", + "00000010 00 00 00 00 CC 00 00 00 00 00 00 F0 43" + }; + + string[] expectedBinary2 = + { + "00000000 80 E2 1A 00 C8 FF CA FF FF 00 00 CB FF FF FF FF", + "00000010 00 00 00 00 C7 FF FF FF FF FF FF FF FF" + }; + + ExecuteAndValidate(tokensToWrite, expectedText, expectedBinary1, JsonWriteOptions.None); + ExecuteAndValidate(tokensToWrite, expectedText, expectedBinary2, JsonWriteOptions.EnableUInt64Values); + } + + // ------------------------- + // Unsigned Integer values > INT_MAX + // ------------------------- + { + const ulong Int64Max = (ulong)long.MaxValue; + const ulong UInt64Max = ulong.MaxValue; + + JsonToken[] tokensToWrite = + { + JsonToken.ArrayStart(), + JsonToken.Number(Int64Max + 1UL), + JsonToken.Number(Int64Max + 100UL), + JsonToken.Number(Int64Max + (Int64Max / 2)), + JsonToken.Number(UInt64Max - 100UL), + JsonToken.Number(UInt64Max - 1UL), + JsonToken.ArrayEnd(), + }; + + string[] expectedText = + { + @"[9223372036854775808,9223372036854775907,13835058055282163710,18446744073709551515,18446744073709551", + @"614]" + }; + + string[] expectedBinary1 = + { + "00000000 80 E2 2D CC 00 00 00 00 00 00 E0 43 CC 00 00 00", + "00000010 00 00 00 E0 43 CC 00 00 00 00 00 00 E8 43 CC 00", + "00000020 00 00 00 00 00 F0 43 CC 00 00 00 00 00 00 F0 43" + }; + + string[] expectedBinary2 = + { + "00000000 80 E2 2D C7 00 00 00 00 00 00 00 80 C7 63 00 00", + "00000010 00 00 00 00 80 C7 FE FF FF FF FF FF FF BF C7 9B", + "00000020 FF FF FF FF FF FF FF C7 FE FF FF FF FF FF FF FF" + }; + + ExecuteAndValidate(tokensToWrite, expectedText, expectedBinary1, JsonWriteOptions.None); + ExecuteAndValidate(tokensToWrite, expectedText, expectedBinary2, JsonWriteOptions.EnableUInt64Values); + } + + // ------------------------- + // Within an object + // ------------------------- + { + const ulong Int64Max = long.MaxValue; + + JsonToken[] tokensToWrite = + { + JsonToken.ObjectStart(), + JsonToken.FieldName("value1"), + JsonToken.Number(Int64Max + 1UL), + JsonToken.FieldName("value2"), + JsonToken.Number(0UL), + JsonToken.FieldName("value3"), + JsonToken.ObjectStart(), + JsonToken.FieldName("value3.1"), + JsonToken.Number(Int64Max + 1UL), + JsonToken.FieldName("value3.2"), + JsonToken.Number(Int64Max + 1UL), + JsonToken.FieldName("value3.3"), + JsonToken.Number(0UL), + JsonToken.FieldName("value3.4"), + JsonToken.Number(Int64Max + 1UL), + JsonToken.ObjectEnd(), + JsonToken.FieldName("value4"), + JsonToken.Number(0UL), + JsonToken.FieldName("value5"), + JsonToken.Number(Int64Max + 1UL), + JsonToken.ObjectEnd(), + }; + + string[] expectedText = + { + @"{""value1"":9223372036854775808,""value2"":0,""value3"":{""value3.1"":9223372036854775808,""value3.2"":9223372", + @"036854775808,""value3.3"":0,""value3.4"":9223372036854775808},""value4"":0,""value5"":9223372036854775808}" + }; + + string[] expectedBinary1 = + { + "00000000 80 EA 79 86 76 61 6C 75 65 31 CC 00 00 00 00 00", + "00000010 00 E0 43 86 76 61 6C 75 65 32 00 86 76 61 6C 75", + "00000020 65 33 EA 40 88 76 61 6C 75 65 33 2E 31 CC 00 00", + "00000030 00 00 00 00 E0 43 88 76 61 6C 75 65 33 2E 32 CC", + "00000040 00 00 00 00 00 00 E0 43 88 76 61 6C 75 65 33 2E", + "00000050 33 00 88 76 61 6C 75 65 33 2E 34 CC 00 00 00 00", + "00000060 00 00 E0 43 86 76 61 6C 75 65 34 00 86 76 61 6C", + "00000070 75 65 35 CC 00 00 00 00 00 00 E0 43" + }; + + string[] expectedBinary2 = + { + "00000000 80 EA 79 86 76 61 6C 75 65 31 C7 00 00 00 00 00", + "00000010 00 00 80 86 76 61 6C 75 65 32 00 86 76 61 6C 75", + "00000020 65 33 EA 40 88 76 61 6C 75 65 33 2E 31 C7 00 00", + "00000030 00 00 00 00 00 80 88 76 61 6C 75 65 33 2E 32 C7", + "00000040 00 00 00 00 00 00 00 80 88 76 61 6C 75 65 33 2E", + "00000050 33 00 88 76 61 6C 75 65 33 2E 34 C7 00 00 00 00", + "00000060 00 00 00 80 86 76 61 6C 75 65 34 00 86 76 61 6C", + "00000070 75 65 35 C7 00 00 00 00 00 00 00 80" + }; + + ExecuteAndValidate(tokensToWrite, expectedText, expectedBinary1, JsonWriteOptions.None); + ExecuteAndValidate(tokensToWrite, expectedText, expectedBinary2, JsonWriteOptions.EnableUInt64Values); + } + } #endregion #region String @@ -1047,7 +1254,7 @@ public void ReferenceStringsTest() 8, (byte)(JsonBinaryEncoding.TypeMarker.EncodedStringLengthMin + "hello".Length), (byte)'h', (byte)'e', (byte)'l', (byte)'l', (byte)'o', - JsonBinaryEncoding.TypeMarker.ReferenceString1ByteOffset, + JsonBinaryEncoding.TypeMarker.StrR1, 3, }; @@ -8582,7 +8789,7 @@ private static void ExecuteAndValidate( SerializationSpec.Text(JsonWriteOptions.None), SerializationSpec.Binary(JsonWriteOptions.None), SerializationSpec.Binary(JsonWriteOptions.EnableNumberArrays), - SerializationSpec.Binary(JsonWriteOptions.EnableUInt64), + SerializationSpec.Binary(JsonWriteOptions.EnableUInt64Values), }; RewriteScenario[] rewriteScenarios = new RewriteScenario[] @@ -8601,14 +8808,25 @@ private static void ExecuteAndValidate( foreach (SerializationSpec outputSpec in serializationSpecs) { + RoundTripResult roundTripResult = null; foreach (RewriteScenario rewriteScenario in rewriteScenarios) { - VerifyJsonRoundTrip( + // For Binary(EnableNumberArrays) to Binary(EnableNumberArrays) we need + // to relax the strict comparison since the logic of converting a regular + // array into a uniform number array might be different from the input. + bool strictComparison = !object.ReferenceEquals(inputSpec, outputSpec) || + (inputSpec.SerializationFormat != JsonSerializationFormat.Binary) || + (inputSpec.WriteOptions != JsonWriteOptions.EnableNumberArrays); + + RoundTripBaseline roundTripBaseline = roundTripResult != null ? new RoundTripBaseline(roundTripResult.OutputResult, strictComparison) : null; + + roundTripResult = VerifyJsonRoundTrip( inputResult, inputJson: null, inputSpec, outputSpec, - rewriteScenario); + rewriteScenario, + roundTripBaseline); } } }