Skip to content

Commit

Permalink
Add useDeterministicOrdering to JSONEncodingOptions (#1478)
Browse files Browse the repository at this point in the history 
* Add `useDeterministicOrdering` to `JSONEncodingOptions`

Adds an option to ensure that JSON serialization is deterministic when serializing Protobuf `map` fields. This should be the only type that needs to be sorted, since individual fields are serialized in order by the generated code that invokes `try visitor.visit(...)` for each field.

Fixes #1477
  • Loading branch information
@rebello95
rebello95 authored Oct 26, 2023
1 parent 514b0d8 commit 5c44631
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 39 deletions.
14 changes: 14 additions & 0 deletions Sources/SwiftProtobuf/JSONEncodingOptions.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,19 @@ public struct JSONEncodingOptions: Sendable {
/// By default they are converted to JSON(lowerCamelCase) names.
public var preserveProtoFieldNames: Bool = false

/// Whether to use deterministic ordering when serializing.
///
/// Note that the deterministic serialization is NOT canonical across languages.
/// It is NOT guaranteed to remain stable over time. It is unstable across
/// different builds with schema changes due to unknown fields. Users who need
/// canonical serialization (e.g., persistent storage in a canonical form,
/// fingerprinting, etc.) should define their own canonicalization specification
/// and implement their own serializer rather than relying on this API.
///
/// If deterministic serialization is requested, map entries will be sorted
/// by keys in lexographical order. This is an implementation detail
/// and subject to change.
public var useDeterministicOrdering: Bool = false

public init() {}
}
52 changes: 31 additions & 21 deletions Sources/SwiftProtobuf/JSONEncodingVisitor.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,39 +358,49 @@ internal struct JSONEncodingVisitor: Visitor {
// Packed fields are handled the same as non-packed fields, so JSON just
// relies on the default implementations in Visitor.swift



mutating func visitMapField<KeyType, ValueType: MapValueType>(fieldType: _ProtobufMap<KeyType, ValueType>.Type, value: _ProtobufMap<KeyType, ValueType>.BaseType, fieldNumber: Int) throws {
try startField(for: fieldNumber)
encoder.append(text: "{")
var mapVisitor = JSONMapEncodingVisitor(encoder: JSONEncoder(), options: options)
for (k,v) in value {
try KeyType.visitSingular(value: k, fieldNumber: 1, with: &mapVisitor)
try ValueType.visitSingular(value: v, fieldNumber: 2, with: &mapVisitor)
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
(visitor: inout JSONMapEncodingVisitor, key, value) throws -> () in
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
try ValueType.visitSingular(value: value, fieldNumber: 2, with: &visitor)
}
encoder.append(utf8Bytes: mapVisitor.bytesResult)
encoder.append(text: "}")
}

mutating func visitMapField<KeyType, ValueType>(fieldType: _ProtobufEnumMap<KeyType, ValueType>.Type, value: _ProtobufEnumMap<KeyType, ValueType>.BaseType, fieldNumber: Int) throws where ValueType.RawValue == Int {
try startField(for: fieldNumber)
encoder.append(text: "{")
var mapVisitor = JSONMapEncodingVisitor(encoder: JSONEncoder(), options: options)
for (k, v) in value {
try KeyType.visitSingular(value: k, fieldNumber: 1, with: &mapVisitor)
try mapVisitor.visitSingularEnumField(value: v, fieldNumber: 2)
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
(visitor: inout JSONMapEncodingVisitor, key, value) throws -> () in
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
try visitor.visitSingularEnumField(value: value, fieldNumber: 2)
}
encoder.append(utf8Bytes: mapVisitor.bytesResult)
encoder.append(text: "}")
}

mutating func visitMapField<KeyType, ValueType>(fieldType: _ProtobufMessageMap<KeyType, ValueType>.Type, value: _ProtobufMessageMap<KeyType, ValueType>.BaseType, fieldNumber: Int) throws {
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
(visitor: inout JSONMapEncodingVisitor, key, value) throws -> () in
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
try visitor.visitSingularMessageField(value: value, fieldNumber: 2)
}
}

/// Helper to encapsulate the common structure of iterating over a map
/// and encoding the keys and values.
private mutating func iterateAndEncode<K, V>(
map: Dictionary<K, V>,
fieldNumber: Int,
isOrderedBefore: (K, K) -> Bool,
encode: (inout JSONMapEncodingVisitor, K, V) throws -> ()
) throws {
try startField(for: fieldNumber)
encoder.append(text: "{")
var mapVisitor = JSONMapEncodingVisitor(encoder: JSONEncoder(), options: options)
for (k,v) in value {
try KeyType.visitSingular(value: k, fieldNumber: 1, with: &mapVisitor)
try mapVisitor.visitSingularMessageField(value: v, fieldNumber: 2)
if options.useDeterministicOrdering {
for (k,v) in map.sorted(by: { isOrderedBefore( $0.0, $1.0) }) {
try encode(&mapVisitor, k, v)
}
} else {
for (k,v) in map {
try encode(&mapVisitor, k, v)
}
}
encoder.append(utf8Bytes: mapVisitor.bytesResult)
encoder.append(text: "}")
Expand Down
36 changes: 18 additions & 18 deletions Sources/SwiftProtobuf/TextFormatEncodingVisitor.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -503,16 +503,16 @@ internal struct TextFormatEncodingVisitor: Visitor {
// fields (including proto3's default use of packed) without
// introducing the baggage of a separate option.

private mutating func _visitPacked<T>(
value: [T], fieldNumber: Int,
private mutating func iterateAndEncode<T>(
packedValue: [T], fieldNumber: Int,
encode: (T, inout TextFormatEncoder) -> ()
) throws {
assert(!value.isEmpty)
assert(!packedValue.isEmpty)
emitFieldName(lookingUp: fieldNumber)
encoder.startRegularField()
var firstItem = true
encoder.startArray()
for v in value {
for v in packedValue {
if !firstItem {
encoder.arraySeparator()
}
Expand All @@ -524,42 +524,42 @@ internal struct TextFormatEncodingVisitor: Visitor {
}

mutating func visitPackedFloatField(value: [Float], fieldNumber: Int) throws {
try _visitPacked(value: value, fieldNumber: fieldNumber) {
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
(v: Float, encoder: inout TextFormatEncoder) in
encoder.putFloatValue(value: v)
}
}

mutating func visitPackedDoubleField(value: [Double], fieldNumber: Int) throws {
try _visitPacked(value: value, fieldNumber: fieldNumber) {
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
(v: Double, encoder: inout TextFormatEncoder) in
encoder.putDoubleValue(value: v)
}
}

mutating func visitPackedInt32Field(value: [Int32], fieldNumber: Int) throws {
try _visitPacked(value: value, fieldNumber: fieldNumber) {
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
(v: Int32, encoder: inout TextFormatEncoder) in
encoder.putInt64(value: Int64(v))
}
}

mutating func visitPackedInt64Field(value: [Int64], fieldNumber: Int) throws {
try _visitPacked(value: value, fieldNumber: fieldNumber) {
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
(v: Int64, encoder: inout TextFormatEncoder) in
encoder.putInt64(value: v)
}
}

mutating func visitPackedUInt32Field(value: [UInt32], fieldNumber: Int) throws {
try _visitPacked(value: value, fieldNumber: fieldNumber) {
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
(v: UInt32, encoder: inout TextFormatEncoder) in
encoder.putUInt64(value: UInt64(v))
}
}

mutating func visitPackedUInt64Field(value: [UInt64], fieldNumber: Int) throws {
try _visitPacked(value: value, fieldNumber: fieldNumber) {
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
(v: UInt64, encoder: inout TextFormatEncoder) in
encoder.putUInt64(value: v)
}
Expand Down Expand Up @@ -590,26 +590,26 @@ internal struct TextFormatEncodingVisitor: Visitor {
}

mutating func visitPackedBoolField(value: [Bool], fieldNumber: Int) throws {
try _visitPacked(value: value, fieldNumber: fieldNumber) {
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
(v: Bool, encoder: inout TextFormatEncoder) in
encoder.putBoolValue(value: v)
}
}

mutating func visitPackedEnumField<E: Enum>(value: [E], fieldNumber: Int) throws {
try _visitPacked(value: value, fieldNumber: fieldNumber) {
try iterateAndEncode(packedValue: value, fieldNumber: fieldNumber) {
(v: E, encoder: inout TextFormatEncoder) in
encoder.putEnumValue(value: v)
}
}

/// Helper to encapsulate the common structure of iterating over a map
/// and encoding the keys and values.
private mutating func _visitMap<K, V>(
private mutating func iterateAndEncode<K, V>(
map: Dictionary<K, V>,
fieldNumber: Int,
isOrderedBefore: (K, K) -> Bool,
coder: (inout TextFormatEncodingVisitor, K, V) throws -> ()
encode: (inout TextFormatEncodingVisitor, K, V) throws -> ()
) throws {
// Cache old visitor configuration
let oldNameMap = self.nameMap
Expand All @@ -625,7 +625,7 @@ internal struct TextFormatEncodingVisitor: Visitor {
self.nameResolver = mapNameResolver
self.extensions = nil

try coder(&self, k, v)
try encode(&self, k, v)

// Restore configuration before resuming containing message
self.extensions = oldExtensions
Expand All @@ -641,7 +641,7 @@ internal struct TextFormatEncodingVisitor: Visitor {
value: _ProtobufMap<KeyType, ValueType>.BaseType,
fieldNumber: Int
) throws {
try _visitMap(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
(visitor: inout TextFormatEncodingVisitor, key, value) throws -> () in
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
try ValueType.visitSingular(value: value, fieldNumber: 2, with: &visitor)
Expand All @@ -653,7 +653,7 @@ internal struct TextFormatEncodingVisitor: Visitor {
value: _ProtobufEnumMap<KeyType, ValueType>.BaseType,
fieldNumber: Int
) throws where ValueType.RawValue == Int {
try _visitMap(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
(visitor: inout TextFormatEncodingVisitor, key, value) throws -> () in
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
try visitor.visitSingularEnumField(value: value, fieldNumber: 2)
Expand All @@ -665,7 +665,7 @@ internal struct TextFormatEncodingVisitor: Visitor {
value: _ProtobufMessageMap<KeyType, ValueType>.BaseType,
fieldNumber: Int
) throws {
try _visitMap(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
try iterateAndEncode(map: value, fieldNumber: fieldNumber, isOrderedBefore: KeyType._lessThan) {
(visitor: inout TextFormatEncodingVisitor, key, value) throws -> () in
try KeyType.visitSingular(value: key, fieldNumber: 1, with: &visitor)
try visitor.visitSingularMessageField(value: value, fieldNumber: 2)
Expand Down
44 changes: 44 additions & 0 deletions Tests/SwiftProtobufTests/Test_JSONEncodingOptions.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -272,4 +272,48 @@ class Test_JSONEncodingOptions: XCTestCase {
XCTAssertEqual(try msg7.jsonString(options: protoNames),
"{\"@type\":\"type.googleapis.com/swift_proto_testing.TestAllTypes\",\"optional_nested_enum\":\"NEG\"}")
}

func testUseDeterministicOrdering() {
var options = JSONEncodingOptions()
options.useDeterministicOrdering = true

let stringMap = SwiftProtoTesting_Message3.with {
$0.mapStringString = [
"b": "B",
"a": "A",
"0": "0",
"UPPER": "v",
"x": "X",
]
}
XCTAssertEqual(
try stringMap.jsonString(options: options),
"{\"mapStringString\":{\"0\":\"0\",\"UPPER\":\"v\",\"a\":\"A\",\"b\":\"B\",\"x\":\"X\"}}"
)

let messageMap = SwiftProtoTesting_Message3.with {
$0.mapInt32Message = [
5: .with { $0.optionalSint32 = 5 },
1: .with { $0.optionalSint32 = 1 },
3: .with { $0.optionalSint32 = 3 },
]
}
XCTAssertEqual(
try messageMap.jsonString(options: options),
"{\"mapInt32Message\":{\"1\":{\"optionalSint32\":1},\"3\":{\"optionalSint32\":3},\"5\":{\"optionalSint32\":5}}}"
)

let enumMap = SwiftProtoTesting_Message3.with {
$0.mapInt32Enum = [
5: .foo,
3: .bar,
0: .baz,
1: .extra3,
]
}
XCTAssertEqual(
try enumMap.jsonString(options: options),
"{\"mapInt32Enum\":{\"0\":\"BAZ\",\"1\":\"EXTRA_3\",\"3\":\"BAR\",\"5\":\"FOO\"}}"
)
}
}

0 comments on commit 5c44631

Please sign in to comment.