From 0a1e1b3d9cd8731fe47ec65e14d170f3790d558f Mon Sep 17 00:00:00 2001 From: ported Date: Mon, 30 May 2022 13:57:13 +0200 Subject: [PATCH 1/6] Update well-known types --- .../lib/google/protobuf/__init__.py | 1210 +++++++++++------ 1 file changed, 762 insertions(+), 448 deletions(-) diff --git a/src/betterproto/lib/google/protobuf/__init__.py b/src/betterproto/lib/google/protobuf/__init__.py index 822b870ec..9744ee0f8 100644 --- a/src/betterproto/lib/google/protobuf/__init__.py +++ b/src/betterproto/lib/google/protobuf/__init__.py @@ -15,54 +15,123 @@ class Syntax(betterproto.Enum): """The syntax in which a protocol buffer element is defined.""" - # Syntax `proto2`. SYNTAX_PROTO2 = 0 - # Syntax `proto3`. + """Syntax `proto2`.""" + SYNTAX_PROTO3 = 1 + """Syntax `proto3`.""" class FieldKind(betterproto.Enum): + """Basic field types.""" + TYPE_UNKNOWN = 0 + """Field type unknown.""" + TYPE_DOUBLE = 1 + """Field type double.""" + TYPE_FLOAT = 2 + """Field type float.""" + TYPE_INT64 = 3 + """Field type int64.""" + TYPE_UINT64 = 4 + """Field type uint64.""" + TYPE_INT32 = 5 + """Field type int32.""" + TYPE_FIXED64 = 6 + """Field type fixed64.""" + TYPE_FIXED32 = 7 + """Field type fixed32.""" + TYPE_BOOL = 8 + """Field type bool.""" + TYPE_STRING = 9 + """Field type string.""" + TYPE_GROUP = 10 + """Field type group. Proto2 syntax only, and deprecated.""" + TYPE_MESSAGE = 11 + """Field type message.""" + TYPE_BYTES = 12 + """Field type bytes.""" + TYPE_UINT32 = 13 + """Field type uint32.""" + TYPE_ENUM = 14 + """Field type enum.""" + TYPE_SFIXED32 = 15 + """Field type sfixed32.""" + TYPE_SFIXED64 = 16 + """Field type sfixed64.""" + TYPE_SINT32 = 17 + """Field type sint32.""" + TYPE_SINT64 = 18 + """Field type sint64.""" class FieldCardinality(betterproto.Enum): + """Whether a field is optional, required, or repeated.""" + CARDINALITY_UNKNOWN = 0 + """For fields with unknown cardinality.""" + CARDINALITY_OPTIONAL = 1 + """For optional fields.""" + CARDINALITY_REQUIRED = 2 + """For required fields. Proto2 syntax only.""" + CARDINALITY_REPEATED = 3 + """For repeated fields.""" class FieldDescriptorProtoType(betterproto.Enum): TYPE_DOUBLE = 1 + """0 is reserved for errors. Order is weird for historical reasons.""" + TYPE_FLOAT = 2 TYPE_INT64 = 3 + """ + Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + negative values are likely. + """ + TYPE_UINT64 = 4 TYPE_INT32 = 5 + """ + Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + negative values are likely. + """ + TYPE_FIXED64 = 6 TYPE_FIXED32 = 7 TYPE_BOOL = 8 TYPE_STRING = 9 TYPE_GROUP = 10 + """ + Tag-delimited aggregate. Group type is deprecated and not supported in + proto3. However, Proto3 implementations should still be able to parse the + group wire format and treat group fields as unknown fields. + """ + TYPE_MESSAGE = 11 TYPE_BYTES = 12 + """New in version 2.""" + TYPE_UINT32 = 13 TYPE_ENUM = 14 TYPE_SFIXED32 = 15 @@ -73,29 +142,48 @@ class FieldDescriptorProtoType(betterproto.Enum): class FieldDescriptorProtoLabel(betterproto.Enum): LABEL_OPTIONAL = 1 + """0 is reserved for errors""" + LABEL_REQUIRED = 2 LABEL_REPEATED = 3 class FileOptionsOptimizeMode(betterproto.Enum): + """Generated classes can be optimized for speed or code size.""" + SPEED = 1 CODE_SIZE = 2 + """etc.""" + LITE_RUNTIME = 3 class FieldOptionsCType(betterproto.Enum): STRING = 0 + """Default mode.""" + CORD = 1 STRING_PIECE = 2 class FieldOptionsJsType(betterproto.Enum): JS_NORMAL = 0 + """Use the default type.""" + JS_STRING = 1 + """Use JavaScript strings.""" + JS_NUMBER = 2 + """Use JavaScript numbers.""" class MethodOptionsIdempotencyLevel(betterproto.Enum): + """ + Is this method side-effect-free (or safe in HTTP parlance), or idempotent, + or neither? HTTP based RPC implementation may choose GET verb for safe + methods, and PUT verb for idempotent methods instead of the default POST. + """ + IDEMPOTENCY_UNKNOWN = 0 NO_SIDE_EFFECTS = 1 IDEMPOTENT = 2 @@ -108,8 +196,8 @@ class NullValue(betterproto.Enum): `null`. """ - # Null value. NULL_VALUE = 0 + """Null value.""" @dataclass(eq=False, repr=False) @@ -126,49 +214,53 @@ class Any(betterproto.Message): Example 3: Pack and unpack a message in Python. foo = Foo(...) any = Any() any.Pack(foo) ... if any.Is(Foo.DESCRIPTOR): any.Unpack(foo) ... Example 4: Pack and unpack a message in Go - foo := &pb.Foo{...} any, err := anypb.New(foo) if err != nil { - ... } ... foo := &pb.Foo{} if err := - any.UnmarshalTo(foo); err != nil { ... } The pack methods - provided by protobuf library will by default use - 'type.googleapis.com/full.type.name' as the type URL and the unpack methods - only use the fully qualified type name after the last '/' in the type URL, - for example "foo.bar.com/x/y.z" will yield type name "y.z". JSON ==== The - JSON representation of an `Any` value uses the regular representation of - the deserialized, embedded message, with an additional field `@type` which - contains the type URL. Example: package google.profile; message - Person { string first_name = 1; string last_name = 2; } - { "@type": "type.googleapis.com/google.profile.Person", - "firstName": , "lastName": } If the embedded - message type is well-known and has a custom JSON representation, that - representation will be embedded adding a field `value` which holds the - custom JSON in addition to the `@type` field. Example (for message - [google.protobuf.Duration][]): { "@type": - "type.googleapis.com/google.protobuf.Duration", "value": "1.212s" - } - """ - - # A URL/resource name that uniquely identifies the type of the serialized - # protocol buffer message. This string must contain at least one "/" - # character. The last segment of the URL's path must represent the fully - # qualified name of the type (as in `path/google.protobuf.Duration`). The - # name should be in a canonical form (e.g., leading "." is not accepted). In - # practice, teams usually precompile into the binary all types that they - # expect it to use in the context of Any. However, for URLs which use the - # scheme `http`, `https`, or no scheme, one can optionally set up a type - # server that maps type URLs to message definitions as follows: * If no - # scheme is provided, `https` is assumed. * An HTTP GET on the URL must yield - # a [google.protobuf.Type][] value in binary format, or produce an error. * - # Applications are allowed to cache lookup results based on the URL, or - # have them precompiled into a binary to avoid any lookup. Therefore, - # binary compatibility needs to be preserved on changes to types. (Use - # versioned type names to manage breaking changes.) Note: this - # functionality is not currently available in the official protobuf release, - # and it is not used for type URLs beginning with type.googleapis.com. - # Schemes other than `http`, `https` (or the empty scheme) might be used with - # implementation specific semantics. + foo := &pb.Foo{...} any, err := ptypes.MarshalAny(foo) ... + foo := &pb.Foo{} if err := ptypes.UnmarshalAny(any, foo); err != nil { + ... } The pack methods provided by protobuf library will by default + use 'type.googleapis.com/full.type.name' as the type URL and the unpack + methods only use the fully qualified type name after the last '/' in the + type URL, for example "foo.bar.com/x/y.z" will yield type name "y.z". JSON + ==== The JSON representation of an `Any` value uses the regular + representation of the deserialized, embedded message, with an additional + field `@type` which contains the type URL. Example: package + google.profile; message Person { string first_name = 1; + string last_name = 2; } { "@type": + "type.googleapis.com/google.profile.Person", "firstName": , + "lastName": } If the embedded message type is well-known and + has a custom JSON representation, that representation will be embedded + adding a field `value` which holds the custom JSON in addition to the + `@type` field. Example (for message [google.protobuf.Duration][]): { + "@type": "type.googleapis.com/google.protobuf.Duration", "value": + "1.212s" } + """ + type_url: str = betterproto.string_field(1) - # Must be a valid serialized protocol buffer of the above specified type. + """ + A URL/resource name that uniquely identifies the type of the serialized + protocol buffer message. This string must contain at least one "/" + character. The last segment of the URL's path must represent the fully + qualified name of the type (as in `path/google.protobuf.Duration`). The + name should be in a canonical form (e.g., leading "." is not accepted). In + practice, teams usually precompile into the binary all types that they + expect it to use in the context of Any. However, for URLs which use the + scheme `http`, `https`, or no scheme, one can optionally set up a type + server that maps type URLs to message definitions as follows: * If no + scheme is provided, `https` is assumed. * An HTTP GET on the URL must yield + a [google.protobuf.Type][] value in binary format, or produce an error. * + Applications are allowed to cache lookup results based on the URL, or + have them precompiled into a binary to avoid any lookup. Therefore, + binary compatibility needs to be preserved on changes to types. (Use + versioned type names to manage breaking changes.) Note: this + functionality is not currently available in the official protobuf release, + and it is not used for type URLs beginning with type.googleapis.com. + Schemes other than `http`, `https` (or the empty scheme) might be used with + implementation specific semantics. + """ + value: bytes = betterproto.bytes_field(2) + """ + Must be a valid serialized protocol buffer of the above specified type. + """ @dataclass(eq=False, repr=False) @@ -178,85 +270,113 @@ class SourceContext(betterproto.Message): element, like the file in which it is defined. """ - # The path-qualified name of the .proto file that contained the associated - # protobuf element. For example: `"google/protobuf/source_context.proto"`. file_name: str = betterproto.string_field(1) + """ + The path-qualified name of the .proto file that contained the associated + protobuf element. For example: `"google/protobuf/source_context.proto"`. + """ @dataclass(eq=False, repr=False) class Type(betterproto.Message): """A protocol buffer message type.""" - # The fully qualified message name. name: str = betterproto.string_field(1) - # The list of fields. + """The fully qualified message name.""" + fields: List["Field"] = betterproto.message_field(2) - # The list of types appearing in `oneof` definitions in this type. + """The list of fields.""" + oneofs: List[str] = betterproto.string_field(3) - # The protocol buffer options. + """The list of types appearing in `oneof` definitions in this type.""" + options: List["Option"] = betterproto.message_field(4) - # The source context. + """The protocol buffer options.""" + source_context: "SourceContext" = betterproto.message_field(5) - # The source syntax. + """The source context.""" + syntax: "Syntax" = betterproto.enum_field(6) + """The source syntax.""" @dataclass(eq=False, repr=False) class Field(betterproto.Message): """A single field of a message type.""" - # The field type. kind: "FieldKind" = betterproto.enum_field(1) - # The field cardinality. + """The field type.""" + cardinality: "FieldCardinality" = betterproto.enum_field(2) - # The field number. + """The field cardinality.""" + number: int = betterproto.int32_field(3) - # The field name. + """The field number.""" + name: str = betterproto.string_field(4) - # The field type URL, without the scheme, for message or enumeration types. - # Example: `"type.googleapis.com/google.protobuf.Timestamp"`. + """The field name.""" + type_url: str = betterproto.string_field(6) - # The index of the field type in `Type.oneofs`, for message or enumeration - # types. The first type has index 1; zero means the type is not in the list. + """ + The field type URL, without the scheme, for message or enumeration types. + Example: `"type.googleapis.com/google.protobuf.Timestamp"`. + """ + oneof_index: int = betterproto.int32_field(7) - # Whether to use alternative packed wire representation. + """ + The index of the field type in `Type.oneofs`, for message or enumeration + types. The first type has index 1; zero means the type is not in the list. + """ + packed: bool = betterproto.bool_field(8) - # The protocol buffer options. + """Whether to use alternative packed wire representation.""" + options: List["Option"] = betterproto.message_field(9) - # The field JSON name. + """The protocol buffer options.""" + json_name: str = betterproto.string_field(10) - # The string value of the default value of this field. Proto2 syntax only. + """The field JSON name.""" + default_value: str = betterproto.string_field(11) + """ + The string value of the default value of this field. Proto2 syntax only. + """ @dataclass(eq=False, repr=False) class Enum(betterproto.Message): """Enum type definition.""" - # Enum type name. name: str = betterproto.string_field(1) - # Enum value definitions. + """Enum type name.""" + enumvalue: List["EnumValue"] = betterproto.message_field( 2, wraps=betterproto.TYPE_ENUM ) - # Protocol buffer options. + """Enum value definitions.""" + options: List["Option"] = betterproto.message_field(3) - # The source context. + """Protocol buffer options.""" + source_context: "SourceContext" = betterproto.message_field(4) - # The source syntax. + """The source context.""" + syntax: "Syntax" = betterproto.enum_field(5) + """The source syntax.""" @dataclass(eq=False, repr=False) class EnumValue(betterproto.Message): """Enum value definition.""" - # Enum value name. name: str = betterproto.string_field(1) - # Enum value number. + """Enum value name.""" + number: int = betterproto.int32_field(2) - # Protocol buffer options. + """Enum value number.""" + options: List["Option"] = betterproto.message_field(3) + """Protocol buffer options.""" @dataclass(eq=False, repr=False) @@ -266,16 +386,21 @@ class Option(betterproto.Message): enumeration, etc. """ - # The option's name. For protobuf built-in options (options defined in - # descriptor.proto), this is the short name. For example, `"map_entry"`. For - # custom options, it should be the fully-qualified name. For example, - # `"google.api.http"`. name: str = betterproto.string_field(1) - # The option's value packed in an Any message. If the value is a primitive, - # the corresponding wrapper type defined in google/protobuf/wrappers.proto - # should be used. If the value is an enum, it should be stored as an int32 - # value using the google.protobuf.Int32Value type. + """ + The option's name. For protobuf built-in options (options defined in + descriptor.proto), this is the short name. For example, `"map_entry"`. For + custom options, it should be the fully-qualified name. For example, + `"google.api.http"`. + """ + value: "Any" = betterproto.message_field(2) + """ + The option's value packed in an Any message. If the value is a primitive, + the corresponding wrapper type defined in google/protobuf/wrappers.proto + should be used. If the value is an enum, it should be stored as an int32 + value using the google.protobuf.Int32Value type. + """ @dataclass(eq=False, repr=False) @@ -291,54 +416,72 @@ class Api(betterproto.Message): for detailed terminology. """ - # The fully qualified name of this interface, including package name followed - # by the interface's simple name. name: str = betterproto.string_field(1) - # The methods of this interface, in unspecified order. + """ + The fully qualified name of this interface, including package name followed + by the interface's simple name. + """ + methods: List["Method"] = betterproto.message_field(2) - # Any metadata attached to the interface. + """The methods of this interface, in unspecified order.""" + options: List["Option"] = betterproto.message_field(3) - # A version string for this interface. If specified, must have the form - # `major-version.minor-version`, as in `1.10`. If the minor version is - # omitted, it defaults to zero. If the entire version field is empty, the - # major version is derived from the package name, as outlined below. If the - # field is not empty, the version in the package name will be verified to be - # consistent with what is provided here. The versioning schema uses [semantic - # versioning](http://semver.org) where the major version number indicates a - # breaking change and the minor version an additive, non-breaking change. - # Both version numbers are signals to users what to expect from different - # versions, and should be carefully chosen based on the product plan. The - # major version is also reflected in the package name of the interface, which - # must end in `v`, as in `google.feature.v1`. For major - # versions 0 and 1, the suffix can be omitted. Zero major versions must only - # be used for experimental, non-GA interfaces. + """Any metadata attached to the interface.""" + version: str = betterproto.string_field(4) - # Source context for the protocol buffer service represented by this message. + """ + A version string for this interface. If specified, must have the form + `major-version.minor-version`, as in `1.10`. If the minor version is + omitted, it defaults to zero. If the entire version field is empty, the + major version is derived from the package name, as outlined below. If the + field is not empty, the version in the package name will be verified to be + consistent with what is provided here. The versioning schema uses [semantic + versioning](http://semver.org) where the major version number indicates a + breaking change and the minor version an additive, non-breaking change. + Both version numbers are signals to users what to expect from different + versions, and should be carefully chosen based on the product plan. The + major version is also reflected in the package name of the interface, which + must end in `v`, as in `google.feature.v1`. For major + versions 0 and 1, the suffix can be omitted. Zero major versions must only + be used for experimental, non-GA interfaces. + """ + source_context: "SourceContext" = betterproto.message_field(5) - # Included interfaces. See [Mixin][]. + """ + Source context for the protocol buffer service represented by this message. + """ + mixins: List["Mixin"] = betterproto.message_field(6) - # The source syntax of the service. + """Included interfaces. See [Mixin][].""" + syntax: "Syntax" = betterproto.enum_field(7) + """The source syntax of the service.""" @dataclass(eq=False, repr=False) class Method(betterproto.Message): """Method represents a method of an API interface.""" - # The simple name of this method. name: str = betterproto.string_field(1) - # A URL of the input message type. + """The simple name of this method.""" + request_type_url: str = betterproto.string_field(2) - # If true, the request is streamed. + """A URL of the input message type.""" + request_streaming: bool = betterproto.bool_field(3) - # The URL of the output message type. + """If true, the request is streamed.""" + response_type_url: str = betterproto.string_field(4) - # If true, the response is streamed. + """The URL of the output message type.""" + response_streaming: bool = betterproto.bool_field(5) - # Any metadata attached to the method. + """If true, the response is streamed.""" + options: List["Option"] = betterproto.message_field(6) - # The source syntax of this method. + """Any metadata attached to the method.""" + syntax: "Syntax" = betterproto.enum_field(7) + """The source syntax of this method.""" @dataclass(eq=False, repr=False) @@ -366,7 +509,7 @@ class Mixin(betterproto.Message): implies that all methods in `AccessControl` are also declared with same name and request/response types in `Storage`. A documentation generator or annotation processor will see the effective `Storage.GetAcl` method after - inheriting documentation and annotations as follows: service Storage { + inherting documentation and annotations as follows: service Storage { // Get the underlying ACL object. rpc GetAcl(GetAclRequest) returns (Acl) { option (google.api.http).get = "/v2/{resource=**}:getAcl"; } ... } Note how the version in the path pattern changed from @@ -380,10 +523,13 @@ class Mixin(betterproto.Message): ... } """ - # The fully qualified name of the interface which is included. name: str = betterproto.string_field(1) - # If non-empty specifies a path under which inherited HTTP paths are rooted. + """The fully qualified name of the interface which is included.""" + root: str = betterproto.string_field(2) + """ + If non-empty specifies a path under which inherited HTTP paths are rooted. + """ @dataclass(eq=False, repr=False) @@ -402,27 +548,38 @@ class FileDescriptorProto(betterproto.Message): name: str = betterproto.string_field(1) package: str = betterproto.string_field(2) - # Names of files imported by this file. dependency: List[str] = betterproto.string_field(3) - # Indexes of the public imported files in the dependency list above. + """Names of files imported by this file.""" + public_dependency: List[int] = betterproto.int32_field(10) - # Indexes of the weak imported files in the dependency list. For Google- - # internal migration only. Do not use. + """Indexes of the public imported files in the dependency list above.""" + weak_dependency: List[int] = betterproto.int32_field(11) - # All top-level definitions in this file. + """ + Indexes of the weak imported files in the dependency list. For Google- + internal migration only. Do not use. + """ + message_type: List["DescriptorProto"] = betterproto.message_field(4) + """All top-level definitions in this file.""" + enum_type: List["EnumDescriptorProto"] = betterproto.message_field(5) service: List["ServiceDescriptorProto"] = betterproto.message_field(6) extension: List["FieldDescriptorProto"] = betterproto.message_field(7) options: "FileOptions" = betterproto.message_field(8) - # This field contains optional information about the original source code. - # You may safely remove this entire field without harming runtime - # functionality of the descriptors -- the information is needed only by - # development tools. source_code_info: "SourceCodeInfo" = betterproto.message_field(9) - # The syntax of the proto file. The supported values are "proto2" and - # "proto3". + """ + This field contains optional information about the original source code. + You may safely remove this entire field without harming runtime + functionality of the descriptors -- the information is needed only by + development tools. + """ + syntax: str = betterproto.string_field(12) + """ + The syntax of the proto file. The supported values are "proto2" and + "proto3". + """ @dataclass(eq=False, repr=False) @@ -440,9 +597,11 @@ class DescriptorProto(betterproto.Message): oneof_decl: List["OneofDescriptorProto"] = betterproto.message_field(8) options: "MessageOptions" = betterproto.message_field(7) reserved_range: List["DescriptorProtoReservedRange"] = betterproto.message_field(9) - # Reserved field names, which may not be used by fields in the same message. - # A given name may only be reserved once. reserved_name: List[str] = betterproto.string_field(10) + """ + Reserved field names, which may not be used by fields in the same message. + A given name may only be reserved once. + """ @dataclass(eq=False, repr=False) @@ -466,8 +625,8 @@ class DescriptorProtoReservedRange(betterproto.Message): @dataclass(eq=False, repr=False) class ExtensionRangeOptions(betterproto.Message): - # The parser stores options it doesn't recognize here. See above. uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + """The parser stores options it doesn't recognize here. See above.""" @dataclass(eq=False, repr=False) @@ -477,48 +636,68 @@ class FieldDescriptorProto(betterproto.Message): name: str = betterproto.string_field(1) number: int = betterproto.int32_field(3) label: "FieldDescriptorProtoLabel" = betterproto.enum_field(4) - # If type_name is set, this need not be set. If both this and type_name are - # set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. type: "FieldDescriptorProtoType" = betterproto.enum_field(5) - # For message and enum types, this is the name of the type. If the name - # starts with a '.', it is fully-qualified. Otherwise, C++-like scoping - # rules are used to find the type (i.e. first the nested types within this - # message are searched, then within the parent, on up to the root namespace). + """ + If type_name is set, this need not be set. If both this and type_name are + set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. + """ + type_name: str = betterproto.string_field(6) - # For extensions, this is the name of the type being extended. It is - # resolved in the same manner as type_name. + """ + For message and enum types, this is the name of the type. If the name + starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + rules are used to find the type (i.e. first the nested types within this + message are searched, then within the parent, on up to the root namespace). + """ + extendee: str = betterproto.string_field(2) - # For numeric types, contains the original text representation of the value. - # For booleans, "true" or "false". For strings, contains the default text - # contents (not escaped in any way). For bytes, contains the C escaped value. - # All bytes >= 128 are escaped. TODO(kenton): Base-64 encode? + """ + For extensions, this is the name of the type being extended. It is + resolved in the same manner as type_name. + """ + default_value: str = betterproto.string_field(7) - # If set, gives the index of a oneof in the containing type's oneof_decl - # list. This field is a member of that oneof. + """ + For numeric types, contains the original text representation of the value. + For booleans, "true" or "false". For strings, contains the default text + contents (not escaped in any way). For bytes, contains the C escaped value. + All bytes >= 128 are escaped. TODO(kenton): Base-64 encode? + """ + oneof_index: int = betterproto.int32_field(9) - # JSON name of this field. The value is set by protocol compiler. If the user - # has set a "json_name" option on this field, that option's value will be - # used. Otherwise, it's deduced from the field's name by converting it to - # camelCase. + """ + If set, gives the index of a oneof in the containing type's oneof_decl + list. This field is a member of that oneof. + """ + json_name: str = betterproto.string_field(10) + """ + JSON name of this field. The value is set by protocol compiler. If the user + has set a "json_name" option on this field, that option's value will be + used. Otherwise, it's deduced from the field's name by converting it to + camelCase. + """ + options: "FieldOptions" = betterproto.message_field(8) - # If true, this is a proto3 "optional". When a proto3 field is optional, it - # tracks presence regardless of field type. When proto3_optional is true, - # this field must be belong to a oneof to signal to old proto3 clients that - # presence is tracked for this field. This oneof is known as a "synthetic" - # oneof, and this field must be its sole member (each proto3 optional field - # gets its own synthetic oneof). Synthetic oneofs exist in the descriptor - # only, and do not generate any API. Synthetic oneofs must be ordered after - # all "real" oneofs. For message fields, proto3_optional doesn't create any - # semantic change, since non-repeated message fields always track presence. - # However it still indicates the semantic detail of whether the user wrote - # "optional" or not. This can be useful for round-tripping the .proto file. - # For consistency we give message fields a synthetic oneof also, even though - # it is not required to track presence. This is especially important because - # the parser can't tell if a field is a message or an enum, so it must always - # create a synthetic oneof. Proto2 optional fields do not set this flag, - # because they already indicate optional with `LABEL_OPTIONAL`. proto3_optional: bool = betterproto.bool_field(17) + """ + If true, this is a proto3 "optional". When a proto3 field is optional, it + tracks presence regardless of field type. When proto3_optional is true, + this field must be belong to a oneof to signal to old proto3 clients that + presence is tracked for this field. This oneof is known as a "synthetic" + oneof, and this field must be its sole member (each proto3 optional field + gets its own synthetic oneof). Synthetic oneofs exist in the descriptor + only, and do not generate any API. Synthetic oneofs must be ordered after + all "real" oneofs. For message fields, proto3_optional doesn't create any + semantic change, since non-repeated message fields always track presence. + However it still indicates the semantic detail of whether the user wrote + "optional" or not. This can be useful for round-tripping the .proto file. + For consistency we give message fields a synthetic oneof also, even though + it is not required to track presence. This is especially important because + the parser can't tell if a field is a message or an enum, so it must always + create a synthetic oneof. Proto2 optional fields do not set this flag, + because they already indicate optional with `LABEL_OPTIONAL`. + """ @dataclass(eq=False, repr=False) @@ -536,15 +715,20 @@ class EnumDescriptorProto(betterproto.Message): name: str = betterproto.string_field(1) value: List["EnumValueDescriptorProto"] = betterproto.message_field(2) options: "EnumOptions" = betterproto.message_field(3) - # Range of reserved numeric values. Reserved numeric values may not be used - # by enum values in the same enum declaration. Reserved ranges may not - # overlap. reserved_range: List[ "EnumDescriptorProtoEnumReservedRange" ] = betterproto.message_field(4) - # Reserved enum value names, which may not be reused. A given name may only - # be reserved once. + """ + Range of reserved numeric values. Reserved numeric values may not be used + by enum values in the same enum declaration. Reserved ranges may not + overlap. + """ + reserved_name: List[str] = betterproto.string_field(5) + """ + Reserved enum value names, which may not be reused. A given name may only + be reserved once. + """ @dataclass(eq=False, repr=False) @@ -583,103 +767,152 @@ class MethodDescriptorProto(betterproto.Message): """Describes a method of a service.""" name: str = betterproto.string_field(1) - # Input and output type names. These are resolved in the same way as - # FieldDescriptorProto.type_name, but must refer to a message type. input_type: str = betterproto.string_field(2) + """ + Input and output type names. These are resolved in the same way as + FieldDescriptorProto.type_name, but must refer to a message type. + """ + output_type: str = betterproto.string_field(3) options: "MethodOptions" = betterproto.message_field(4) - # Identifies if client streams multiple client messages client_streaming: bool = betterproto.bool_field(5) - # Identifies if server streams multiple server messages + """Identifies if client streams multiple client messages""" + server_streaming: bool = betterproto.bool_field(6) + """Identifies if server streams multiple server messages""" @dataclass(eq=False, repr=False) class FileOptions(betterproto.Message): - # Sets the Java package where classes generated from this .proto will be - # placed. By default, the proto package is used, but this is often - # inappropriate because proto packages do not normally start with backwards - # domain names. java_package: str = betterproto.string_field(1) - # Controls the name of the wrapper Java class generated for the .proto file. - # That class will always contain the .proto file's getDescriptor() method as - # well as any top-level extensions defined in the .proto file. If - # java_multiple_files is disabled, then all the other classes from the .proto - # file will be nested inside the single wrapper outer class. + """ + Sets the Java package where classes generated from this .proto will be + placed. By default, the proto package is used, but this is often + inappropriate because proto packages do not normally start with backwards + domain names. + """ + java_outer_classname: str = betterproto.string_field(8) - # If enabled, then the Java code generator will generate a separate .java - # file for each top-level message, enum, and service defined in the .proto - # file. Thus, these types will *not* be nested inside the wrapper class - # named by java_outer_classname. However, the wrapper class will still be - # generated to contain the file's getDescriptor() method as well as any top- - # level extensions defined in the file. + """ + If set, all the classes from the .proto file are wrapped in a single outer + class with the given name. This applies to both Proto1 (equivalent to the + old "--one_java_file" option) and Proto2 (where a .proto always translates + to a single class, but you may want to explicitly choose the class name). + """ + java_multiple_files: bool = betterproto.bool_field(10) - # This option does nothing. + """ + If set true, then the Java code generator will generate a separate .java + file for each top-level message, enum, and service defined in the .proto + file. Thus, these types will *not* be nested inside the outer class named + by java_outer_classname. However, the outer class will still be generated + to contain the file's getDescriptor() method as well as any top-level + extensions defined in the file. + """ + java_generate_equals_and_hash: bool = betterproto.bool_field(20) - # If set true, then the Java2 code generator will generate code that throws - # an exception whenever an attempt is made to assign a non-UTF-8 byte - # sequence to a string field. Message reflection will do the same. However, - # an extension field still accepts non-UTF-8 byte sequences. This option has - # no effect on when used with the lite runtime. + """This option does nothing.""" + java_string_check_utf8: bool = betterproto.bool_field(27) + """ + If set true, then the Java2 code generator will generate code that throws + an exception whenever an attempt is made to assign a non-UTF-8 byte + sequence to a string field. Message reflection will do the same. However, + an extension field still accepts non-UTF-8 byte sequences. This option has + no effect on when used with the lite runtime. + """ + optimize_for: "FileOptionsOptimizeMode" = betterproto.enum_field(9) - # Sets the Go package where structs generated from this .proto will be - # placed. If omitted, the Go package will be derived from the following: - - # The basename of the package import path, if provided. - Otherwise, the - # package statement in the .proto file, if present. - Otherwise, the - # basename of the .proto file, without extension. go_package: str = betterproto.string_field(11) - # Should generic services be generated in each language? "Generic" services - # are not specific to any particular RPC system. They are generated by the - # main code generators in each language (without additional plugins). Generic - # services were the only kind of service generation supported by early - # versions of google.protobuf. Generic services are now considered deprecated - # in favor of using plugins that generate code specific to your particular - # RPC system. Therefore, these default to false. Old code which depends on - # generic services should explicitly set them to true. + """ + Sets the Go package where structs generated from this .proto will be + placed. If omitted, the Go package will be derived from the following: - + The basename of the package import path, if provided. - Otherwise, the + package statement in the .proto file, if present. - Otherwise, the + basename of the .proto file, without extension. + """ + cc_generic_services: bool = betterproto.bool_field(16) + """ + Should generic services be generated in each language? "Generic" services + are not specific to any particular RPC system. They are generated by the + main code generators in each language (without additional plugins). Generic + services were the only kind of service generation supported by early + versions of google.protobuf. Generic services are now considered deprecated + in favor of using plugins that generate code specific to your particular + RPC system. Therefore, these default to false. Old code which depends on + generic services should explicitly set them to true. + """ + java_generic_services: bool = betterproto.bool_field(17) py_generic_services: bool = betterproto.bool_field(18) php_generic_services: bool = betterproto.bool_field(42) - # Is this file deprecated? Depending on the target platform, this can emit - # Deprecated annotations for everything in the file, or it will be completely - # ignored; in the very least, this is a formalization for deprecating files. deprecated: bool = betterproto.bool_field(23) - # Enables the use of arenas for the proto messages in this file. This applies - # only to generated classes for C++. + """ + Is this file deprecated? Depending on the target platform, this can emit + Deprecated annotations for everything in the file, or it will be completely + ignored; in the very least, this is a formalization for deprecating files. + """ + cc_enable_arenas: bool = betterproto.bool_field(31) - # Sets the objective c class prefix which is prepended to all objective c - # generated classes from this .proto. There is no default. + """ + Enables the use of arenas for the proto messages in this file. This applies + only to generated classes for C++. + """ + objc_class_prefix: str = betterproto.string_field(36) - # Namespace for generated classes; defaults to the package. + """ + Sets the objective c class prefix which is prepended to all objective c + generated classes from this .proto. There is no default. + """ + csharp_namespace: str = betterproto.string_field(37) - # By default Swift generators will take the proto package and CamelCase it - # replacing '.' with underscore and use that to prefix the types/symbols - # defined. When this options is provided, they will use this value instead to - # prefix the types/symbols defined. + """Namespace for generated classes; defaults to the package.""" + swift_prefix: str = betterproto.string_field(39) - # Sets the php class prefix which is prepended to all php generated classes - # from this .proto. Default is empty. + """ + By default Swift generators will take the proto package and CamelCase it + replacing '.' with underscore and use that to prefix the types/symbols + defined. When this options is provided, they will use this value instead to + prefix the types/symbols defined. + """ + php_class_prefix: str = betterproto.string_field(40) - # Use this option to change the namespace of php generated classes. Default - # is empty. When this option is empty, the package name will be used for - # determining the namespace. + """ + Sets the php class prefix which is prepended to all php generated classes + from this .proto. Default is empty. + """ + php_namespace: str = betterproto.string_field(41) - # Use this option to change the namespace of php generated metadata classes. - # Default is empty. When this option is empty, the proto file name will be - # used for determining the namespace. + """ + Use this option to change the namespace of php generated classes. Default + is empty. When this option is empty, the package name will be used for + determining the namespace. + """ + php_metadata_namespace: str = betterproto.string_field(44) - # Use this option to change the package of ruby generated classes. Default is - # empty. When this option is not set, the package name will be used for - # determining the ruby package. + """ + Use this option to change the namespace of php generated metadata classes. + Default is empty. When this option is empty, the proto file name will be + used for determining the namespace. + """ + ruby_package: str = betterproto.string_field(45) - # The parser stores options it doesn't recognize here. See the documentation - # for the "Options" section above. + """ + Use this option to change the package of ruby generated classes. Default is + empty. When this option is not set, the package name will be used for + determining the ruby package. + """ + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + """ + The parser stores options it doesn't recognize here. See the documentation + for the "Options" section above. + """ def __post_init__(self) -> None: super().__post_init__() - if self.java_generate_equals_and_hash: + if self.is_set("java_generate_equals_and_hash"): warnings.warn( "FileOptions.java_generate_equals_and_hash is deprecated", DeprecationWarning, @@ -688,147 +921,190 @@ def __post_init__(self) -> None: @dataclass(eq=False, repr=False) class MessageOptions(betterproto.Message): - # Set true to use the old proto1 MessageSet wire format for extensions. This - # is provided for backwards-compatibility with the MessageSet wire format. - # You should not use this for any other reason: It's less efficient, has - # fewer features, and is more complicated. The message must be defined - # exactly as follows: message Foo { option message_set_wire_format = - # true; extensions 4 to max; } Note that the message cannot have any - # defined fields; MessageSets only have extensions. All extensions of your - # type must be singular messages; e.g. they cannot be int32s, enums, or - # repeated messages. Because this is an option, the above two restrictions - # are not enforced by the protocol compiler. message_set_wire_format: bool = betterproto.bool_field(1) - # Disables the generation of the standard "descriptor()" accessor, which can - # conflict with a field of the same name. This is meant to make migration - # from proto1 easier; new code should avoid fields named "descriptor". + """ + Set true to use the old proto1 MessageSet wire format for extensions. This + is provided for backwards-compatibility with the MessageSet wire format. + You should not use this for any other reason: It's less efficient, has + fewer features, and is more complicated. The message must be defined + exactly as follows: message Foo { option message_set_wire_format = + true; extensions 4 to max; } Note that the message cannot have any + defined fields; MessageSets only have extensions. All extensions of your + type must be singular messages; e.g. they cannot be int32s, enums, or + repeated messages. Because this is an option, the above two restrictions + are not enforced by the protocol compiler. + """ + no_standard_descriptor_accessor: bool = betterproto.bool_field(2) - # Is this message deprecated? Depending on the target platform, this can emit - # Deprecated annotations for the message, or it will be completely ignored; - # in the very least, this is a formalization for deprecating messages. + """ + Disables the generation of the standard "descriptor()" accessor, which can + conflict with a field of the same name. This is meant to make migration + from proto1 easier; new code should avoid fields named "descriptor". + """ + deprecated: bool = betterproto.bool_field(3) - # Whether the message is an automatically generated map entry type for the - # maps field. For maps fields: map map_field = 1; The - # parsed descriptor looks like: message MapFieldEntry { option - # map_entry = true; optional KeyType key = 1; optional - # ValueType value = 2; } repeated MapFieldEntry map_field = 1; - # Implementations may choose not to generate the map_entry=true message, but - # use a native map in the target language to hold the keys and values. The - # reflection APIs in such implementations still need to work as if the field - # is a repeated message field. NOTE: Do not set the option in .proto files. - # Always use the maps syntax instead. The option should only be implicitly - # set by the proto compiler parser. + """ + Is this message deprecated? Depending on the target platform, this can emit + Deprecated annotations for the message, or it will be completely ignored; + in the very least, this is a formalization for deprecating messages. + """ + map_entry: bool = betterproto.bool_field(7) - # The parser stores options it doesn't recognize here. See above. + """ + Whether the message is an automatically generated map entry type for the + maps field. For maps fields: map map_field = 1; The + parsed descriptor looks like: message MapFieldEntry { option + map_entry = true; optional KeyType key = 1; optional + ValueType value = 2; } repeated MapFieldEntry map_field = 1; + Implementations may choose not to generate the map_entry=true message, but + use a native map in the target language to hold the keys and values. The + reflection APIs in such implementations still need to work as if the field + is a repeated message field. NOTE: Do not set the option in .proto files. + Always use the maps syntax instead. The option should only be implicitly + set by the proto compiler parser. + """ + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + """The parser stores options it doesn't recognize here. See above.""" @dataclass(eq=False, repr=False) class FieldOptions(betterproto.Message): - # The ctype option instructs the C++ code generator to use a different - # representation of the field than it normally would. See the specific - # options below. This option is not yet implemented in the open source - # release -- sorry, we'll try to include it in a future version! ctype: "FieldOptionsCType" = betterproto.enum_field(1) - # The packed option can be enabled for repeated primitive fields to enable a - # more efficient representation on the wire. Rather than repeatedly writing - # the tag and type for each element, the entire array is encoded as a single - # length-delimited blob. In proto3, only explicit setting it to false will - # avoid using packed encoding. + """ + The ctype option instructs the C++ code generator to use a different + representation of the field than it normally would. See the specific + options below. This option is not yet implemented in the open source + release -- sorry, we'll try to include it in a future version! + """ + packed: bool = betterproto.bool_field(2) - # The jstype option determines the JavaScript type used for values of the - # field. The option is permitted only for 64 bit integral and fixed types - # (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING - # is represented as JavaScript string, which avoids loss of precision that - # can happen when a large value is converted to a floating point JavaScript. - # Specifying JS_NUMBER for the jstype causes the generated JavaScript code to - # use the JavaScript "number" type. The behavior of the default option - # JS_NORMAL is implementation dependent. This option is an enum to permit - # additional types to be added, e.g. goog.math.Integer. + """ + The packed option can be enabled for repeated primitive fields to enable a + more efficient representation on the wire. Rather than repeatedly writing + the tag and type for each element, the entire array is encoded as a single + length-delimited blob. In proto3, only explicit setting it to false will + avoid using packed encoding. + """ + jstype: "FieldOptionsJsType" = betterproto.enum_field(6) - # Should this field be parsed lazily? Lazy applies only to message-type - # fields. It means that when the outer message is initially parsed, the - # inner message's contents will not be parsed but instead stored in encoded - # form. The inner message will actually be parsed when it is first accessed. - # This is only a hint. Implementations are free to choose whether to use - # eager or lazy parsing regardless of the value of this option. However, - # setting this option true suggests that the protocol author believes that - # using lazy parsing on this field is worth the additional bookkeeping - # overhead typically needed to implement it. This option does not affect the - # public interface of any generated code; all method signatures remain the - # same. Furthermore, thread-safety of the interface is not affected by this - # option; const methods remain safe to call from multiple threads - # concurrently, while non-const methods continue to require exclusive access. - # Note that implementations may choose not to check required fields within a - # lazy sub-message. That is, calling IsInitialized() on the outer message - # may return true even if the inner message has missing required fields. This - # is necessary because otherwise the inner message would have to be parsed in - # order to perform the check, defeating the purpose of lazy parsing. An - # implementation which chooses not to check required fields must be - # consistent about it. That is, for any particular sub-message, the - # implementation must either *always* check its required fields, or *never* - # check its required fields, regardless of whether or not the message has - # been parsed. + """ + The jstype option determines the JavaScript type used for values of the + field. The option is permitted only for 64 bit integral and fixed types + (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + is represented as JavaScript string, which avoids loss of precision that + can happen when a large value is converted to a floating point JavaScript. + Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + use the JavaScript "number" type. The behavior of the default option + JS_NORMAL is implementation dependent. This option is an enum to permit + additional types to be added, e.g. goog.math.Integer. + """ + lazy: bool = betterproto.bool_field(5) - # Is this field deprecated? Depending on the target platform, this can emit - # Deprecated annotations for accessors, or it will be completely ignored; in - # the very least, this is a formalization for deprecating fields. + """ + Should this field be parsed lazily? Lazy applies only to message-type + fields. It means that when the outer message is initially parsed, the + inner message's contents will not be parsed but instead stored in encoded + form. The inner message will actually be parsed when it is first accessed. + This is only a hint. Implementations are free to choose whether to use + eager or lazy parsing regardless of the value of this option. However, + setting this option true suggests that the protocol author believes that + using lazy parsing on this field is worth the additional bookkeeping + overhead typically needed to implement it. This option does not affect the + public interface of any generated code; all method signatures remain the + same. Furthermore, thread-safety of the interface is not affected by this + option; const methods remain safe to call from multiple threads + concurrently, while non-const methods continue to require exclusive access. + Note that implementations may choose not to check required fields within a + lazy sub-message. That is, calling IsInitialized() on the outer message + may return true even if the inner message has missing required fields. This + is necessary because otherwise the inner message would have to be parsed in + order to perform the check, defeating the purpose of lazy parsing. An + implementation which chooses not to check required fields must be + consistent about it. That is, for any particular sub-message, the + implementation must either *always* check its required fields, or *never* + check its required fields, regardless of whether or not the message has + been parsed. + """ + deprecated: bool = betterproto.bool_field(3) - # For Google-internal migration only. Do not use. + """ + Is this field deprecated? Depending on the target platform, this can emit + Deprecated annotations for accessors, or it will be completely ignored; in + the very least, this is a formalization for deprecating fields. + """ + weak: bool = betterproto.bool_field(10) - # The parser stores options it doesn't recognize here. See above. + """For Google-internal migration only. Do not use.""" + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + """The parser stores options it doesn't recognize here. See above.""" @dataclass(eq=False, repr=False) class OneofOptions(betterproto.Message): - # The parser stores options it doesn't recognize here. See above. uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + """The parser stores options it doesn't recognize here. See above.""" @dataclass(eq=False, repr=False) class EnumOptions(betterproto.Message): - # Set this option to true to allow mapping different tag names to the same - # value. allow_alias: bool = betterproto.bool_field(2) - # Is this enum deprecated? Depending on the target platform, this can emit - # Deprecated annotations for the enum, or it will be completely ignored; in - # the very least, this is a formalization for deprecating enums. + """ + Set this option to true to allow mapping different tag names to the same + value. + """ + deprecated: bool = betterproto.bool_field(3) - # The parser stores options it doesn't recognize here. See above. + """ + Is this enum deprecated? Depending on the target platform, this can emit + Deprecated annotations for the enum, or it will be completely ignored; in + the very least, this is a formalization for deprecating enums. + """ + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + """The parser stores options it doesn't recognize here. See above.""" @dataclass(eq=False, repr=False) class EnumValueOptions(betterproto.Message): - # Is this enum value deprecated? Depending on the target platform, this can - # emit Deprecated annotations for the enum value, or it will be completely - # ignored; in the very least, this is a formalization for deprecating enum - # values. deprecated: bool = betterproto.bool_field(1) - # The parser stores options it doesn't recognize here. See above. + """ + Is this enum value deprecated? Depending on the target platform, this can + emit Deprecated annotations for the enum value, or it will be completely + ignored; in the very least, this is a formalization for deprecating enum + values. + """ + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + """The parser stores options it doesn't recognize here. See above.""" @dataclass(eq=False, repr=False) class ServiceOptions(betterproto.Message): - # Is this service deprecated? Depending on the target platform, this can emit - # Deprecated annotations for the service, or it will be completely ignored; - # in the very least, this is a formalization for deprecating services. deprecated: bool = betterproto.bool_field(33) - # The parser stores options it doesn't recognize here. See above. + """ + Is this service deprecated? Depending on the target platform, this can emit + Deprecated annotations for the service, or it will be completely ignored; + in the very least, this is a formalization for deprecating services. + """ + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + """The parser stores options it doesn't recognize here. See above.""" @dataclass(eq=False, repr=False) class MethodOptions(betterproto.Message): - # Is this method deprecated? Depending on the target platform, this can emit - # Deprecated annotations for the method, or it will be completely ignored; in - # the very least, this is a formalization for deprecating methods. deprecated: bool = betterproto.bool_field(33) + """ + Is this method deprecated? Depending on the target platform, this can emit + Deprecated annotations for the method, or it will be completely ignored; in + the very least, this is a formalization for deprecating methods. + """ + idempotency_level: "MethodOptionsIdempotencyLevel" = betterproto.enum_field(34) - # The parser stores options it doesn't recognize here. See above. uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + """The parser stores options it doesn't recognize here. See above.""" @dataclass(eq=False, repr=False) @@ -843,9 +1119,12 @@ class UninterpretedOption(betterproto.Message): """ name: List["UninterpretedOptionNamePart"] = betterproto.message_field(2) - # The value of the uninterpreted option, in whatever type the tokenizer - # identified it as during parsing. Exactly one of these should be set. identifier_value: str = betterproto.string_field(3) + """ + The value of the uninterpreted option, in whatever type the tokenizer + identified it as during parsing. Exactly one of these should be set. + """ + positive_int_value: int = betterproto.uint64_field(4) negative_int_value: int = betterproto.int64_field(5) double_value: float = betterproto.double_field(6) @@ -874,81 +1153,92 @@ class SourceCodeInfo(betterproto.Message): FileDescriptorProto was generated. """ - # A Location identifies a piece of source code in a .proto file which - # corresponds to a particular definition. This information is intended to be - # useful to IDEs, code indexers, documentation generators, and similar tools. - # For example, say we have a file like: message Foo { optional string - # foo = 1; } Let's look at just the field definition: optional string foo - # = 1; ^ ^^ ^^ ^ ^^^ a bc de f ghi We have the - # following locations: span path represents [a,i) [ 4, - # 0, 2, 0 ] The whole field definition. [a,b) [ 4, 0, 2, 0, 4 ] The - # label (optional). [c,d) [ 4, 0, 2, 0, 5 ] The type (string). [e,f) [ - # 4, 0, 2, 0, 1 ] The name (foo). [g,h) [ 4, 0, 2, 0, 3 ] The number - # (1). Notes: - A location may refer to a repeated field itself (i.e. not to - # any particular index within it). This is used whenever a set of elements - # are logically enclosed in a single code segment. For example, an entire - # extend block (possibly containing multiple extension definitions) will - # have an outer location whose path refers to the "extensions" repeated - # field without an index. - Multiple locations may have the same path. This - # happens when a single logical declaration is spread out across multiple - # places. The most obvious example is the "extend" block again -- there - # may be multiple extend blocks in the same scope, each of which will have - # the same path. - A location's span is not always a subset of its parent's - # span. For example, the "extendee" of an extension declaration appears at - # the beginning of the "extend" block and is shared by all extensions - # within the block. - Just because a location's span is a subset of some - # other location's span does not mean that it is a descendant. For - # example, a "group" defines both a type and a field in a single - # declaration. Thus, the locations corresponding to the type and field and - # their components will overlap. - Code which tries to interpret locations - # should probably be designed to ignore those that it doesn't understand, - # as more types of locations could be recorded in the future. location: List["SourceCodeInfoLocation"] = betterproto.message_field(1) + """ + A Location identifies a piece of source code in a .proto file which + corresponds to a particular definition. This information is intended to be + useful to IDEs, code indexers, documentation generators, and similar tools. + For example, say we have a file like: message Foo { optional string + foo = 1; } Let's look at just the field definition: optional string foo + = 1; ^ ^^ ^^ ^ ^^^ a bc de f ghi We have the + following locations: span path represents [a,i) [ 4, + 0, 2, 0 ] The whole field definition. [a,b) [ 4, 0, 2, 0, 4 ] The + label (optional). [c,d) [ 4, 0, 2, 0, 5 ] The type (string). [e,f) [ + 4, 0, 2, 0, 1 ] The name (foo). [g,h) [ 4, 0, 2, 0, 3 ] The number + (1). Notes: - A location may refer to a repeated field itself (i.e. not to + any particular index within it). This is used whenever a set of elements + are logically enclosed in a single code segment. For example, an entire + extend block (possibly containing multiple extension definitions) will + have an outer location whose path refers to the "extensions" repeated + field without an index. - Multiple locations may have the same path. This + happens when a single logical declaration is spread out across multiple + places. The most obvious example is the "extend" block again -- there + may be multiple extend blocks in the same scope, each of which will have + the same path. - A location's span is not always a subset of its parent's + span. For example, the "extendee" of an extension declaration appears at + the beginning of the "extend" block and is shared by all extensions + within the block. - Just because a location's span is a subset of some + other location's span does not mean that it is a descendant. For + example, a "group" defines both a type and a field in a single + declaration. Thus, the locations corresponding to the type and field and + their components will overlap. - Code which tries to interpret locations + should probably be designed to ignore those that it doesn't understand, + as more types of locations could be recorded in the future. + """ @dataclass(eq=False, repr=False) class SourceCodeInfoLocation(betterproto.Message): - # Identifies which part of the FileDescriptorProto was defined at this - # location. Each element is a field number or an index. They form a path - # from the root FileDescriptorProto to the place where the definition. For - # example, this path: [ 4, 3, 2, 7, 1 ] refers to: file.message_type(3) - # // 4, 3 .field(7) // 2, 7 .name() // 1 This - # is because FileDescriptorProto.message_type has field number 4: repeated - # DescriptorProto message_type = 4; and DescriptorProto.field has field - # number 2: repeated FieldDescriptorProto field = 2; and - # FieldDescriptorProto.name has field number 1: optional string name = 1; - # Thus, the above path gives the location of a field name. If we removed the - # last element: [ 4, 3, 2, 7 ] this path refers to the whole field - # declaration (from the beginning of the label to the terminating semicolon). path: List[int] = betterproto.int32_field(1) - # Always has exactly three or four elements: start line, start column, end - # line (optional, otherwise assumed same as start line), end column. These - # are packed into a single field for efficiency. Note that line and column - # numbers are zero-based -- typically you will want to add 1 to each before - # displaying to a user. + """ + Identifies which part of the FileDescriptorProto was defined at this + location. Each element is a field number or an index. They form a path + from the root FileDescriptorProto to the place where the definition. For + example, this path: [ 4, 3, 2, 7, 1 ] refers to: file.message_type(3) + // 4, 3 .field(7) // 2, 7 .name() // 1 This + is because FileDescriptorProto.message_type has field number 4: repeated + DescriptorProto message_type = 4; and DescriptorProto.field has field + number 2: repeated FieldDescriptorProto field = 2; and + FieldDescriptorProto.name has field number 1: optional string name = 1; + Thus, the above path gives the location of a field name. If we removed the + last element: [ 4, 3, 2, 7 ] this path refers to the whole field + declaration (from the beginning of the label to the terminating semicolon). + """ + span: List[int] = betterproto.int32_field(2) - # If this SourceCodeInfo represents a complete declaration, these are any - # comments appearing before and after the declaration which appear to be - # attached to the declaration. A series of line comments appearing on - # consecutive lines, with no other tokens appearing on those lines, will be - # treated as a single comment. leading_detached_comments will keep paragraphs - # of comments that appear before (but not connected to) the current element. - # Each paragraph, separated by empty lines, will be one comment element in - # the repeated field. Only the comment content is provided; comment markers - # (e.g. //) are stripped out. For block comments, leading whitespace and an - # asterisk will be stripped from the beginning of each line other than the - # first. Newlines are included in the output. Examples: optional int32 foo - # = 1; // Comment attached to foo. // Comment attached to bar. optional - # int32 bar = 2; optional string baz = 3; // Comment attached to baz. - # // Another line attached to baz. // Comment attached to qux. // // - # Another line attached to qux. optional double qux = 4; // Detached - # comment for corge. This is not leading or trailing comments // to qux or - # corge because there are blank lines separating it from // both. // - # Detached comment for corge paragraph 2. optional string corge = 5; /* - # Block comment attached * to corge. Leading asterisks * will be - # removed. */ /* Block comment attached to * grault. */ optional int32 - # grault = 6; // ignored detached comments. + """ + Always has exactly three or four elements: start line, start column, end + line (optional, otherwise assumed same as start line), end column. These + are packed into a single field for efficiency. Note that line and column + numbers are zero-based -- typically you will want to add 1 to each before + displaying to a user. + """ + leading_comments: str = betterproto.string_field(3) + """ + If this SourceCodeInfo represents a complete declaration, these are any + comments appearing before and after the declaration which appear to be + attached to the declaration. A series of line comments appearing on + consecutive lines, with no other tokens appearing on those lines, will be + treated as a single comment. leading_detached_comments will keep paragraphs + of comments that appear before (but not connected to) the current element. + Each paragraph, separated by empty lines, will be one comment element in + the repeated field. Only the comment content is provided; comment markers + (e.g. //) are stripped out. For block comments, leading whitespace and an + asterisk will be stripped from the beginning of each line other than the + first. Newlines are included in the output. Examples: optional int32 foo + = 1; // Comment attached to foo. // Comment attached to bar. optional + int32 bar = 2; optional string baz = 3; // Comment attached to baz. + // Another line attached to baz. // Comment attached to qux. // // + Another line attached to qux. optional double qux = 4; // Detached + comment for corge. This is not leading or trailing comments // to qux or + corge because there are blank lines separating it from // both. // + Detached comment for corge paragraph 2. optional string corge = 5; /* + Block comment attached * to corge. Leading asterisks * will be + removed. */ /* Block comment attached to * grault. */ optional int32 + grault = 6; // ignored detached comments. + """ + trailing_comments: str = betterproto.string_field(4) leading_detached_comments: List[str] = betterproto.string_field(6) @@ -961,25 +1251,36 @@ class GeneratedCodeInfo(betterproto.Message): source file, but may contain references to different source .proto files. """ - # An Annotation connects some span of text in generated code to an element of - # its generating .proto file. annotation: List["GeneratedCodeInfoAnnotation"] = betterproto.message_field(1) + """ + An Annotation connects some span of text in generated code to an element of + its generating .proto file. + """ @dataclass(eq=False, repr=False) class GeneratedCodeInfoAnnotation(betterproto.Message): - # Identifies the element in the original source .proto file. This field is - # formatted the same as SourceCodeInfo.Location.path. path: List[int] = betterproto.int32_field(1) - # Identifies the filesystem path to the original source .proto. + """ + Identifies the element in the original source .proto file. This field is + formatted the same as SourceCodeInfo.Location.path. + """ + source_file: str = betterproto.string_field(2) - # Identifies the starting offset in bytes in the generated code that relates - # to the identified object. + """Identifies the filesystem path to the original source .proto.""" + begin: int = betterproto.int32_field(3) - # Identifies the ending offset in bytes in the generated code that relates to - # the identified offset. The end offset should be one past the last relevant - # byte (so the length of the text = end - begin). + """ + Identifies the starting offset in bytes in the generated code that relates + to the identified object. + """ + end: int = betterproto.int32_field(4) + """ + Identifies the ending offset in bytes in the generated code that relates to + the identified offset. The end offset should be one past the last relevant + byte (so the length of the text = end - begin). + """ @dataclass(eq=False, repr=False) @@ -1015,16 +1316,21 @@ class Duration(betterproto.Message): format as "3.000001s". """ - # Signed seconds of the span of time. Must be from -315,576,000,000 to - # +315,576,000,000 inclusive. Note: these bounds are computed from: 60 - # sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years seconds: int = betterproto.int64_field(1) - # Signed fractions of a second at nanosecond resolution of the span of time. - # Durations less than one second are represented with a 0 `seconds` field and - # a positive or negative `nanos` field. For durations of one second or more, - # a non-zero value for the `nanos` field must be of the same sign as the - # `seconds` field. Must be from -999,999,999 to +999,999,999 inclusive. + """ + Signed seconds of the span of time. Must be from -315,576,000,000 to + +315,576,000,000 inclusive. Note: these bounds are computed from: 60 + sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years + """ + nanos: int = betterproto.int32_field(2) + """ + Signed fractions of a second at nanosecond resolution of the span of time. + Durations less than one second are represented with a 0 `seconds` field and + a positive or negative `nanos` field. For durations of one second or more, + a non-zero value for the `nanos` field must be of the same sign as the + `seconds` field. Must be from -999,999,999 to +999,999,999 inclusive. + """ @dataclass(eq=False, repr=False) @@ -1120,8 +1426,8 @@ class FieldMask(betterproto.Message): `INVALID_ARGUMENT` error if any path is unmappable. """ - # The set of field mask paths. paths: List[str] = betterproto.string_field(1) + """The set of field mask paths.""" @dataclass(eq=False, repr=False) @@ -1135,10 +1441,10 @@ class Struct(betterproto.Message): representation for `Struct` is JSON object. """ - # Unordered map of dynamically typed values. fields: Dict[str, "Value"] = betterproto.map_field( 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE ) + """Unordered map of dynamically typed values.""" @dataclass(eq=False, repr=False) @@ -1151,18 +1457,23 @@ class Value(betterproto.Message): value. """ - # Represents a null value. null_value: "NullValue" = betterproto.enum_field(1, group="kind") - # Represents a double value. + """Represents a null value.""" + number_value: float = betterproto.double_field(2, group="kind") - # Represents a string value. + """Represents a double value.""" + string_value: str = betterproto.string_field(3, group="kind") - # Represents a boolean value. + """Represents a string value.""" + bool_value: bool = betterproto.bool_field(4, group="kind") - # Represents a structured value. + """Represents a boolean value.""" + struct_value: "Struct" = betterproto.message_field(5, group="kind") - # Represents a repeated `Value`. + """Represents a structured value.""" + list_value: "ListValue" = betterproto.message_field(6, group="kind") + """Represents a repeated `Value`.""" @dataclass(eq=False, repr=False) @@ -1172,8 +1483,8 @@ class ListValue(betterproto.Message): representation for `ListValue` is JSON array. """ - # Repeated field of dynamically typed values. values: List["Value"] = betterproto.message_field(1) + """Repeated field of dynamically typed values.""" @dataclass(eq=False, repr=False) @@ -1206,22 +1517,20 @@ class Timestamp(betterproto.Message): long millis = System.currentTimeMillis(); Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) .setNanos((int) ((millis % 1000) * 1000000)).build(); Example 5: Compute Timestamp from - Java `Instant.now()`. Instant now = Instant.now(); Timestamp - timestamp = Timestamp.newBuilder().setSeconds(now.getEpochSecond()) - .setNanos(now.getNano()).build(); Example 6: Compute Timestamp from current - time in Python. timestamp = Timestamp() timestamp.GetCurrentTime() - # JSON Mapping In JSON format, the Timestamp type is encoded as a string in - the [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the - format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" where - {year} is always expressed using four digits while {month}, {day}, {hour}, - {min}, and {sec} are zero-padded to two digits each. The fractional - seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), - are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone - is required. A proto3 JSON serializer should always use UTC (as indicated - by "Z") when printing the Timestamp type and a proto3 JSON parser should be - able to accept both UTC and other timezones (as indicated by an offset). - For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past 01:30 UTC - on January 15, 2017. In JavaScript, one can convert a Date object to this + current time in Python. timestamp = Timestamp() + timestamp.GetCurrentTime() # JSON Mapping In JSON format, the Timestamp + type is encoded as a string in the [RFC + 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the format is + "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" where {year} is + always expressed using four digits while {month}, {day}, {hour}, {min}, and + {sec} are zero-padded to two digits each. The fractional seconds, which can + go up to 9 digits (i.e. up to 1 nanosecond resolution), are optional. The + "Z" suffix indicates the timezone ("UTC"); the timezone is required. A + proto3 JSON serializer should always use UTC (as indicated by "Z") when + printing the Timestamp type and a proto3 JSON parser should be able to + accept both UTC and other timezones (as indicated by an offset). For + example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past 01:30 UTC on + January 15, 2017. In JavaScript, one can convert a Date object to this format using the standard [toISOString()](https://developer.mozilla.org/en- US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) method. In Python, a standard `datetime.datetime` object can be converted to this @@ -1234,13 +1543,18 @@ class Timestamp(betterproto.Message): to obtain a formatter capable of generating timestamps in this format. """ - # Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z. Must - # be from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z inclusive. seconds: int = betterproto.int64_field(1) - # Non-negative fractions of a second at nanosecond resolution. Negative - # second values with fractions must still have non-negative nanos values that - # count forward in time. Must be from 0 to 999,999,999 inclusive. + """ + Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z. Must + be from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z inclusive. + """ + nanos: int = betterproto.int32_field(2) + """ + Non-negative fractions of a second at nanosecond resolution. Negative + second values with fractions must still have non-negative nanos values that + count forward in time. Must be from 0 to 999,999,999 inclusive. + """ @dataclass(eq=False, repr=False) @@ -1250,8 +1564,8 @@ class DoubleValue(betterproto.Message): JSON number. """ - # The double value. value: float = betterproto.double_field(1) + """The double value.""" @dataclass(eq=False, repr=False) @@ -1261,8 +1575,8 @@ class FloatValue(betterproto.Message): JSON number. """ - # The float value. value: float = betterproto.float_field(1) + """The float value.""" @dataclass(eq=False, repr=False) @@ -1272,8 +1586,8 @@ class Int64Value(betterproto.Message): JSON string. """ - # The int64 value. value: int = betterproto.int64_field(1) + """The int64 value.""" @dataclass(eq=False, repr=False) @@ -1283,8 +1597,8 @@ class UInt64Value(betterproto.Message): JSON string. """ - # The uint64 value. value: int = betterproto.uint64_field(1) + """The uint64 value.""" @dataclass(eq=False, repr=False) @@ -1294,8 +1608,8 @@ class Int32Value(betterproto.Message): JSON number. """ - # The int32 value. value: int = betterproto.int32_field(1) + """The int32 value.""" @dataclass(eq=False, repr=False) @@ -1305,8 +1619,8 @@ class UInt32Value(betterproto.Message): JSON number. """ - # The uint32 value. value: int = betterproto.uint32_field(1) + """The uint32 value.""" @dataclass(eq=False, repr=False) @@ -1316,8 +1630,8 @@ class BoolValue(betterproto.Message): `true` and `false`. """ - # The bool value. value: bool = betterproto.bool_field(1) + """The bool value.""" @dataclass(eq=False, repr=False) @@ -1327,8 +1641,8 @@ class StringValue(betterproto.Message): JSON string. """ - # The string value. value: str = betterproto.string_field(1) + """The string value.""" @dataclass(eq=False, repr=False) @@ -1338,5 +1652,5 @@ class BytesValue(betterproto.Message): JSON string. """ - # The bytes value. value: bytes = betterproto.bytes_field(1) + """The bytes value.""" From 5ead759e58edd454ef3ada49551705c67f68ecb5 Mon Sep 17 00:00:00 2001 From: ported Date: Thu, 2 Jun 2022 12:47:44 +0200 Subject: [PATCH 2/6] mostly working serialisation --- src/betterproto/__init__.py | 232 ++++++++++++++---- src/betterproto/compile/importing.py | 15 ++ .../lib/google/protobuf/__init__.py | 163 ++++++++---- src/betterproto/plugin/models.py | 27 +- tests/inputs/config.py | 2 - .../googletypes_value/googletypes_value.json | 10 +- .../googletypes_value/googletypes_value.proto | 4 - .../test_googletypes_value.py | 9 + tests/test_get_ref_type.py | 19 +- tests/test_inputs.py | 8 +- 10 files changed, 354 insertions(+), 135 deletions(-) create mode 100644 tests/inputs/googletypes_value/test_googletypes_value.py diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index 384c260ba..5d94eb694 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -35,6 +35,7 @@ from dateutil.parser import isoparse +import betterproto from ._types import T from ._version import __version__ from .casing import ( @@ -66,6 +67,13 @@ TYPE_MAP = "map" +class SpecialTypes(enum.Enum): + GOOGLE_VALUE = ".google.protobuf.Value" + GOOGLE_STRUCT = ".google.protobuf.Struct" + GOOGLE_LIST_VALUE = ".google.protobuf.ListValue" + GOOGLE_NULL_VALUE = ".google.protobuf.NullValue" + + # Fields that use a fixed amount of space (4 or 8 bytes) FIXED_TYPES = [ TYPE_FLOAT, @@ -155,10 +163,14 @@ class FieldMetadata: proto_type: str # Map information if the proto_type is a map map_types: Optional[Tuple[str, str]] = None + # If this field is repeated (a list) + repeated: Optional[bool] = False # Groups several "one-of" fields together group: Optional[str] = None # Describes the wrapped type (e.g. when using google.protobuf.BoolValue) wraps: Optional[str] = None + # Describes the wrapped type with special conversion handling (e.g. google.protobuf.Struct, google.protobuf.Value) + special: Optional[SpecialTypes] = None # Is the field optional optional: Optional[bool] = False @@ -173,8 +185,10 @@ def dataclass_field( proto_type: str, *, map_types: Optional[Tuple[str, str]] = None, + repeated: Optional[bool] = False, group: Optional[str] = None, wraps: Optional[str] = None, + special: Optional[SpecialTypes] = None, optional: bool = False, ) -> dataclasses.Field: """Creates a dataclass field with attached protobuf metadata.""" @@ -182,7 +196,7 @@ def dataclass_field( default=None if optional else PLACEHOLDER, metadata={ "betterproto": FieldMetadata( - number, proto_type, map_types, group, wraps, optional + number, proto_type, map_types, repeated, group, wraps, special, optional ) }, ) @@ -193,114 +207,116 @@ def dataclass_field( # out at runtime. The generated dataclass variables are still typed correctly. -def enum_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any: - return dataclass_field(number, TYPE_ENUM, group=group, optional=optional) +def enum_field(number: int, repeated: Optional[bool] = False, group: Optional[str] = None, special: Optional[SpecialTypes] = None, optional: bool = False) -> Any: + return dataclass_field(number, TYPE_ENUM, repeated=repeated, group=group, special=special, optional=optional) -def bool_field(number: int, group: Optional[str] = None, optional: bool = False) -> Any: - return dataclass_field(number, TYPE_BOOL, group=group, optional=optional) +def bool_field(number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False) -> Any: + return dataclass_field(number, TYPE_BOOL, repeated=repeated, group=group, optional=optional) def int32_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_INT32, group=group, optional=optional) + return dataclass_field(number, TYPE_INT32, repeated=repeated, group=group, optional=optional) def int64_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_INT64, group=group, optional=optional) + return dataclass_field(number, TYPE_INT64, repeated=repeated, group=group, optional=optional) def uint32_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_UINT32, group=group, optional=optional) + return dataclass_field(number, TYPE_UINT32, repeated=repeated, group=group, optional=optional) def uint64_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_UINT64, group=group, optional=optional) + return dataclass_field(number, TYPE_UINT64, repeated=repeated, group=group, optional=optional) def sint32_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_SINT32, group=group, optional=optional) + return dataclass_field(number, TYPE_SINT32, repeated=repeated, group=group, optional=optional) def sint64_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_SINT64, group=group, optional=optional) + return dataclass_field(number, TYPE_SINT64, repeated=repeated, group=group, optional=optional) def float_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_FLOAT, group=group, optional=optional) + return dataclass_field(number, TYPE_FLOAT, repeated=repeated, group=group, optional=optional) def double_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_DOUBLE, group=group, optional=optional) + return dataclass_field(number, TYPE_DOUBLE, repeated=repeated, group=group, optional=optional) def fixed32_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_FIXED32, group=group, optional=optional) + return dataclass_field(number, TYPE_FIXED32, repeated=repeated, group=group, optional=optional) def fixed64_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_FIXED64, group=group, optional=optional) + return dataclass_field(number, TYPE_FIXED64, repeated=repeated, group=group, optional=optional) def sfixed32_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_SFIXED32, group=group, optional=optional) + return dataclass_field(number, TYPE_SFIXED32, repeated=repeated, group=group, optional=optional) def sfixed64_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_SFIXED64, group=group, optional=optional) + return dataclass_field(number, TYPE_SFIXED64, repeated=repeated, group=group, optional=optional) def string_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_STRING, group=group, optional=optional) + return dataclass_field(number, TYPE_STRING, repeated=repeated, group=group, optional=optional) def bytes_field( - number: int, group: Optional[str] = None, optional: bool = False + number: int, repeated: Optional[bool] = False, group: Optional[str] = None, optional: bool = False ) -> Any: - return dataclass_field(number, TYPE_BYTES, group=group, optional=optional) + return dataclass_field(number, TYPE_BYTES, repeated=repeated, group=group, optional=optional) def message_field( number: int, + repeated: Optional[bool] = False, group: Optional[str] = None, wraps: Optional[str] = None, + special: Optional[SpecialTypes] = None, optional: bool = False, ) -> Any: return dataclass_field( - number, TYPE_MESSAGE, group=group, wraps=wraps, optional=optional + number, TYPE_MESSAGE, repeated=repeated, group=group, wraps=wraps, special=special, optional=optional ) def map_field( - number: int, key_type: str, value_type: str, group: Optional[str] = None + number: int, key_type: str, value_type: str, group: Optional[str] = None, value_special: Optional[SpecialTypes] = None, ) -> Any: return dataclass_field( - number, TYPE_MAP, map_types=(key_type, value_type), group=group + number, TYPE_MAP, map_types=(key_type, value_type), group=group, special=value_special ) @@ -358,7 +374,8 @@ def encode_varint(value: int) -> bytes: return bytes(b + [bits]) -def _preprocess_single(proto_type: str, wraps: str, value: Any) -> bytes: +def _preprocess_single(proto_type: str, wraps: str, special: Optional[SpecialTypes], value: Any) -> bytes: + # print("_preprocess_single: proto_type:", proto_type, ", wraps:", wraps, ", special:", special, ", value:", value) """Adjusts values before serialization.""" if proto_type in ( TYPE_ENUM, @@ -392,7 +409,8 @@ def _preprocess_single(proto_type: str, wraps: str, value: Any) -> bytes: if value is None: return b"" value = _get_wrapper(wraps)(value=value) - + # elif special: + # value = get_special_type(special)(value) return bytes(value) return value @@ -405,9 +423,10 @@ def _serialize_single( *, serialize_empty: bool = False, wraps: str = "", + special: Optional[SpecialTypes] = None, ) -> bytes: """Serializes a single field and value.""" - value = _preprocess_single(proto_type, wraps, value) + value = _preprocess_single(proto_type, wraps, special, value) output = bytearray() if proto_type in WIRE_VARINT_TYPES: @@ -759,14 +778,23 @@ def __bytes__(self) -> bytes: """ output = bytearray() for field_name, meta in self._betterproto.meta_by_field_name.items(): - value = getattr(self, field_name) + _value = getattr(self, field_name) + # print("1. field_name:", field_name, ", meta:", meta, ", value: ", value) + + # If this field is to be converted from/to a message type with special handling, convert it here + # We skip this step if the value is repeated or a map to not infinitely recurse wrapping {} and [] + if meta.special and not meta.map_types and not meta.repeated: + value = get_special_type(meta.special)(_value) + else: + value = _value if value is None: # Optional items should be skipped. This is used for the Google # wrapper types and proto3 field presence/optional fields. + # print("2. field_name:", field_name, ", value is none, skipped") continue - # Being selected in a a group means this field is the one that is + # Being selected in a group means this field is the one that is # currently set in a `oneof` group, so it must be serialized even # if the value is the default zero value. # @@ -792,6 +820,9 @@ def __bytes__(self) -> bytes: # if this is the selected oneof item or if we know we have to # serialize an empty message (i.e. zero value was explicitly # set by the user). + + # TODO this skips google.protobuf.NullValue + # print("2. field_name:", field_name, ", value is default (", value, "), skipped") continue if isinstance(value, list): @@ -801,7 +832,7 @@ def __bytes__(self) -> bytes: # treat it like a field of raw bytes. buf = bytearray() for item in value: - buf += _preprocess_single(meta.proto_type, "", item) + buf += _preprocess_single(meta.proto_type, "", None, item) output += _serialize_single(meta.number, TYPE_BYTES, buf) else: for item in value: @@ -811,6 +842,7 @@ def __bytes__(self) -> bytes: meta.proto_type, item, wraps=meta.wraps or "", + special=meta.special, ) # if it's an empty message it still needs to be represented # as an item in the repeated list @@ -819,6 +851,9 @@ def __bytes__(self) -> bytes: elif isinstance(value, dict): for k, v in value.items(): + if meta.special: + v = get_special_type(meta.special)(v) + # TODO get map_types from type referenced by meta.special if applicable? assert meta.map_types sk = _serialize_single(1, meta.map_types[0], k) sv = _serialize_single(2, meta.map_types[1], v) @@ -841,6 +876,7 @@ def __bytes__(self) -> bytes: value, serialize_empty=serialize_empty or bool(selected_in_group), wraps=meta.wraps or "", + special=meta.special, ) output += self._unknown_fields @@ -897,7 +933,7 @@ def _get_field_default_gen(cls, field: dataclasses.Field) -> Any: elif t.__origin__ in (list, List): # This is some kind of list (repeated) field. return list - elif t.__origin__ is Union and t.__args__[1] is type(None): + elif t.__origin__ is Union and type(None) in t.__args__: # This is an optional field (either wrapped, or using proto3 # field presence). For setting the default we really don't care # what kind of field it is. @@ -918,6 +954,7 @@ def _get_field_default_gen(cls, field: dataclasses.Field) -> Any: def _postprocess_single( self, wire_type: int, meta: FieldMetadata, field_name: str, value: Any ) -> Any: + # print("wire_type: ", wire_type, ", meta: ", meta, ", field_name: ", field_name, ", value: ", value) """Adjusts values after parsing.""" if wire_type == WIRE_VARINT: if meta.proto_type in (TYPE_INT32, TYPE_INT64): @@ -940,6 +977,8 @@ def _postprocess_single( elif meta.proto_type == TYPE_MESSAGE: cls = self._betterproto.cls_by_field[field_name] + # print("cls: ", cls, "meta: ", meta, ", value: ", value) + if cls == datetime: value = _Timestamp().parse(value).to_datetime() elif cls == timedelta: @@ -948,6 +987,9 @@ def _postprocess_single( # This is a Google wrapper value message around a single # scalar type. value = _get_wrapper(meta.wraps)().parse(value).value + elif meta.special: + # This is a Google well-known type that has special handling + value = get_special_type(meta.special)().parse(value) else: value = cls().parse(value) value._serialized_on_wire = True @@ -1095,8 +1137,8 @@ def to_dict( ) ): output[cased_name] = _Duration.delta_to_json(value) - elif meta.wraps: - if value is not None or include_default_values: + elif meta.wraps or meta.special: + if meta.special or value is not None or include_default_values: output[cased_name] = value elif field_is_repeated: # Convert each item. @@ -1204,6 +1246,8 @@ def from_dict(self: T, value: Dict[str, Any]) -> T: if not meta: continue + # TODO none has to be an accepted value for google NullValue + # maybe it doesn't matter because the field will always be defaulted to None anyway? a bit confusing if value[key] is not None: if meta.proto_type == TYPE_MESSAGE: v = getattr(self, field_name) @@ -1220,14 +1264,12 @@ def from_dict(self: T, value: Dict[str, Any]) -> T: v = [cls().from_dict(item) for item in value[key]] elif cls == datetime: v = isoparse(value[key]) - setattr(self, field_name, v) elif cls == timedelta: v = timedelta(seconds=float(value[key][:-1])) - setattr(self, field_name, v) - elif meta.wraps: - setattr(self, field_name, value[key]) + elif meta.wraps or meta.special: + v = value[key] elif v is None: - setattr(self, field_name, cls().from_dict(value[key])) + v = cls().from_dict(value[key]) else: # NOTE: `from_dict` mutates the underlying message, so no # assignment here is necessary. @@ -1501,6 +1543,10 @@ def which_one_of(message: Message, group_name: str) -> Tuple[str, Optional[Any]] Timestamp, UInt32Value, UInt64Value, + Value, + ListValue, + NullValue, + Struct, ) @@ -1543,8 +1589,6 @@ def timestamp_to_json(dt: datetime) -> str: def _get_wrapper(proto_type: str) -> Type: """Get the wrapper message class for a wrapped type.""" - - # TODO: include ListValue and NullValue? return { TYPE_BOOL: BoolValue, TYPE_BYTES: BytesValue, @@ -1557,3 +1601,87 @@ def _get_wrapper(proto_type: str) -> Type: TYPE_UINT32: UInt32Value, TYPE_UINT64: UInt64Value, }[proto_type] + + +class BetterprotoValue(Value): + # TODO replace this with type alias for Value + def __init__(self, value: Optional[Any] = None): + super().__init__() + # print("BetterprotoValue.__init__, value:", value) + if value: + if isinstance(value, str): + self.string_value = value + elif isinstance(value, bool): + self.bool_value = value + elif isinstance(value, int) or isinstance(value, float): + self.number_value = value + elif isinstance(value, dict) and all(isinstance(k, str) for k in value.keys()): + self.struct_value = value + elif isinstance(value, list): + self.list_value = value + elif value is None: + self.null_value = value + else: + raise TypeError(f"Value '{value}' with type '{type(value)}'" + f" is not supported for .google.protobuf.Value") + # print("BetterprotoValue.__init__, self result:", self) + + def parse(self: T, data: bytes) -> T: + result = super().parse(data) + # print("BetterprotoValue.parse, super parse result: ", result) + return betterproto.which_one_of(result, "kind")[1] + + +class BetterprotoStruct(Struct): + # TODO replace this with type alias for Struct / jsonobject + def __init__(self, value: Optional[Dict[str, Any]] = None): + # print("BetterprotoStruct.__init__, value:", value) + # TODO is this correct or do we need to manually convert the Value items here? + if value: + # super().__init__(fields={k: BetterprotoValue(v) for k, v in value.items()}) + super().__init__(fields=value) + # print("BetterprotoStruct.__init__, self result:", self) + else: + super().__init__() + + def parse(self: T, data: bytes): + result = super().parse(data) + # print("BetterprotoStruct.parse, super parse result: ", result) + # TODO is this correct or do we need to manually parse the Value items here? + return result.fields + + +class BetterprotoListValue(ListValue): + # TODO replace the Any with type alias for Value + def __init__(self, value: Optional[List[Any]] = None): + # print("BetterprotoListValue.__init__, value:", value) + if value: + super().__init__(values=value) + # print("BetterprotoListValue.__init__, self result:", self) + else: + super().__init__() + + def parse(self: T, data: bytes): + result = super().parse(data) + # print("BetterprotoListValue.parse, super parse result: ", result) + # TODO is this correct or do we need to manually parse the Value items here? + return result.values + + +class BetterprotoNullValue: + def __new__(cls, value: Optional[None] = PLACEHOLDER): + return NullValue(NullValue.NULL_VALUE) + + # TODO what + def parse(self: T, _: bytes): + return None + + +def get_special_type(special_type: SpecialTypes): + # TODO include NullValue + return { + SpecialTypes.GOOGLE_VALUE: BetterprotoValue, + SpecialTypes.GOOGLE_STRUCT: BetterprotoStruct, + SpecialTypes.GOOGLE_LIST_VALUE: BetterprotoListValue, + SpecialTypes.GOOGLE_NULL_VALUE: BetterprotoNullValue, + }.get(special_type, None) diff --git a/src/betterproto/compile/importing.py b/src/betterproto/compile/importing.py index a28f5554e..4e7185fbf 100644 --- a/src/betterproto/compile/importing.py +++ b/src/betterproto/compile/importing.py @@ -60,6 +60,21 @@ def get_type_reference( elif source_type == ".google.protobuf.Timestamp": return "datetime" + elif source_type == ".google.protobuf.Value": + # TODO replace `Any` with recursive type in self-reference + return "Union[None, int, float, str, bool, Dict[str, AnyType], List[AnyType]]" + + elif source_type == ".google.protobuf.ListValue": + # TODO replace list item with Value type + return "List[Union[None, int, float, str, bool, Dict[str, AnyType], List[AnyType]]]" + + elif source_type == ".google.protobuf.Struct": + # TODO replace `Any` with Value type + return "Dict[str, AnyType]" + + elif source_type == ".google.protobuf.NullValue": + return "None" + source_package, source_type = parse_source_type_name(source_type) current_package: List[str] = package.split(".") if package else [] diff --git a/src/betterproto/lib/google/protobuf/__init__.py b/src/betterproto/lib/google/protobuf/__init__.py index 9744ee0f8..b5d4018e0 100644 --- a/src/betterproto/lib/google/protobuf/__init__.py +++ b/src/betterproto/lib/google/protobuf/__init__.py @@ -4,8 +4,10 @@ import warnings from dataclasses import dataclass from typing import ( + Any as AnyType, Dict, List, + Union, ) import betterproto @@ -284,13 +286,13 @@ class Type(betterproto.Message): name: str = betterproto.string_field(1) """The fully qualified message name.""" - fields: List["Field"] = betterproto.message_field(2) + fields: List["Field"] = betterproto.message_field(2, repeated=True) """The list of fields.""" - oneofs: List[str] = betterproto.string_field(3) + oneofs: List[str] = betterproto.string_field(3, repeated=True) """The list of types appearing in `oneof` definitions in this type.""" - options: List["Option"] = betterproto.message_field(4) + options: List["Option"] = betterproto.message_field(4, repeated=True) """The protocol buffer options.""" source_context: "SourceContext" = betterproto.message_field(5) @@ -331,7 +333,7 @@ class Field(betterproto.Message): packed: bool = betterproto.bool_field(8) """Whether to use alternative packed wire representation.""" - options: List["Option"] = betterproto.message_field(9) + options: List["Option"] = betterproto.message_field(9, repeated=True) """The protocol buffer options.""" json_name: str = betterproto.string_field(10) @@ -351,11 +353,11 @@ class Enum(betterproto.Message): """Enum type name.""" enumvalue: List["EnumValue"] = betterproto.message_field( - 2, wraps=betterproto.TYPE_ENUM + 2, wraps=betterproto.TYPE_ENUM, repeated=True ) """Enum value definitions.""" - options: List["Option"] = betterproto.message_field(3) + options: List["Option"] = betterproto.message_field(3, repeated=True) """Protocol buffer options.""" source_context: "SourceContext" = betterproto.message_field(4) @@ -375,7 +377,7 @@ class EnumValue(betterproto.Message): number: int = betterproto.int32_field(2) """Enum value number.""" - options: List["Option"] = betterproto.message_field(3) + options: List["Option"] = betterproto.message_field(3, repeated=True) """Protocol buffer options.""" @@ -422,10 +424,10 @@ class Api(betterproto.Message): by the interface's simple name. """ - methods: List["Method"] = betterproto.message_field(2) + methods: List["Method"] = betterproto.message_field(2, repeated=True) """The methods of this interface, in unspecified order.""" - options: List["Option"] = betterproto.message_field(3) + options: List["Option"] = betterproto.message_field(3, repeated=True) """Any metadata attached to the interface.""" version: str = betterproto.string_field(4) @@ -451,7 +453,7 @@ class Api(betterproto.Message): Source context for the protocol buffer service represented by this message. """ - mixins: List["Mixin"] = betterproto.message_field(6) + mixins: List["Mixin"] = betterproto.message_field(6, repeated=True) """Included interfaces. See [Mixin][].""" syntax: "Syntax" = betterproto.enum_field(7) @@ -477,7 +479,7 @@ class Method(betterproto.Message): response_streaming: bool = betterproto.bool_field(5) """If true, the response is streamed.""" - options: List["Option"] = betterproto.message_field(6) + options: List["Option"] = betterproto.message_field(6, repeated=True) """Any metadata attached to the method.""" syntax: "Syntax" = betterproto.enum_field(7) @@ -539,7 +541,7 @@ class FileDescriptorSet(betterproto.Message): files it parses. """ - file: List["FileDescriptorProto"] = betterproto.message_field(1) + file: List["FileDescriptorProto"] = betterproto.message_field(1, repeated=True) @dataclass(eq=False, repr=False) @@ -548,24 +550,28 @@ class FileDescriptorProto(betterproto.Message): name: str = betterproto.string_field(1) package: str = betterproto.string_field(2) - dependency: List[str] = betterproto.string_field(3) + dependency: List[str] = betterproto.string_field(3, repeated=True) """Names of files imported by this file.""" - public_dependency: List[int] = betterproto.int32_field(10) + public_dependency: List[int] = betterproto.int32_field(10, repeated=True) """Indexes of the public imported files in the dependency list above.""" - weak_dependency: List[int] = betterproto.int32_field(11) + weak_dependency: List[int] = betterproto.int32_field(11, repeated=True) """ Indexes of the weak imported files in the dependency list. For Google- internal migration only. Do not use. """ - message_type: List["DescriptorProto"] = betterproto.message_field(4) + message_type: List["DescriptorProto"] = betterproto.message_field(4, repeated=True) """All top-level definitions in this file.""" - enum_type: List["EnumDescriptorProto"] = betterproto.message_field(5) - service: List["ServiceDescriptorProto"] = betterproto.message_field(6) - extension: List["FieldDescriptorProto"] = betterproto.message_field(7) + enum_type: List["EnumDescriptorProto"] = betterproto.message_field(5, repeated=True) + service: List["ServiceDescriptorProto"] = betterproto.message_field( + 6, repeated=True + ) + extension: List["FieldDescriptorProto"] = betterproto.message_field( + 7, repeated=True + ) options: "FileOptions" = betterproto.message_field(8) source_code_info: "SourceCodeInfo" = betterproto.message_field(9) """ @@ -587,17 +593,23 @@ class DescriptorProto(betterproto.Message): """Describes a message type.""" name: str = betterproto.string_field(1) - field: List["FieldDescriptorProto"] = betterproto.message_field(2) - extension: List["FieldDescriptorProto"] = betterproto.message_field(6) - nested_type: List["DescriptorProto"] = betterproto.message_field(3) - enum_type: List["EnumDescriptorProto"] = betterproto.message_field(4) + field: List["FieldDescriptorProto"] = betterproto.message_field(2, repeated=True) + extension: List["FieldDescriptorProto"] = betterproto.message_field( + 6, repeated=True + ) + nested_type: List["DescriptorProto"] = betterproto.message_field(3, repeated=True) + enum_type: List["EnumDescriptorProto"] = betterproto.message_field(4, repeated=True) extension_range: List["DescriptorProtoExtensionRange"] = betterproto.message_field( - 5 + 5, repeated=True + ) + oneof_decl: List["OneofDescriptorProto"] = betterproto.message_field( + 8, repeated=True ) - oneof_decl: List["OneofDescriptorProto"] = betterproto.message_field(8) options: "MessageOptions" = betterproto.message_field(7) - reserved_range: List["DescriptorProtoReservedRange"] = betterproto.message_field(9) - reserved_name: List[str] = betterproto.string_field(10) + reserved_range: List["DescriptorProtoReservedRange"] = betterproto.message_field( + 9, repeated=True + ) + reserved_name: List[str] = betterproto.string_field(10, repeated=True) """ Reserved field names, which may not be used by fields in the same message. A given name may only be reserved once. @@ -625,7 +637,9 @@ class DescriptorProtoReservedRange(betterproto.Message): @dataclass(eq=False, repr=False) class ExtensionRangeOptions(betterproto.Message): - uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field( + 999, repeated=True + ) """The parser stores options it doesn't recognize here. See above.""" @@ -713,18 +727,20 @@ class EnumDescriptorProto(betterproto.Message): """Describes an enum type.""" name: str = betterproto.string_field(1) - value: List["EnumValueDescriptorProto"] = betterproto.message_field(2) + value: List["EnumValueDescriptorProto"] = betterproto.message_field( + 2, repeated=True + ) options: "EnumOptions" = betterproto.message_field(3) reserved_range: List[ "EnumDescriptorProtoEnumReservedRange" - ] = betterproto.message_field(4) + ] = betterproto.message_field(4, repeated=True) """ Range of reserved numeric values. Reserved numeric values may not be used by enum values in the same enum declaration. Reserved ranges may not overlap. """ - reserved_name: List[str] = betterproto.string_field(5) + reserved_name: List[str] = betterproto.string_field(5, repeated=True) """ Reserved enum value names, which may not be reused. A given name may only be reserved once. @@ -758,7 +774,7 @@ class ServiceDescriptorProto(betterproto.Message): """Describes a service.""" name: str = betterproto.string_field(1) - method: List["MethodDescriptorProto"] = betterproto.message_field(2) + method: List["MethodDescriptorProto"] = betterproto.message_field(2, repeated=True) options: "ServiceOptions" = betterproto.message_field(3) @@ -904,7 +920,9 @@ class with the given name. This applies to both Proto1 (equivalent to the determining the ruby package. """ - uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field( + 999, repeated=True + ) """ The parser stores options it doesn't recognize here. See the documentation for the "Options" section above. @@ -964,7 +982,9 @@ class MessageOptions(betterproto.Message): set by the proto compiler parser. """ - uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field( + 999, repeated=True + ) """The parser stores options it doesn't recognize here. See above.""" @@ -1037,13 +1057,17 @@ class FieldOptions(betterproto.Message): weak: bool = betterproto.bool_field(10) """For Google-internal migration only. Do not use.""" - uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field( + 999, repeated=True + ) """The parser stores options it doesn't recognize here. See above.""" @dataclass(eq=False, repr=False) class OneofOptions(betterproto.Message): - uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field( + 999, repeated=True + ) """The parser stores options it doesn't recognize here. See above.""" @@ -1062,7 +1086,9 @@ class EnumOptions(betterproto.Message): the very least, this is a formalization for deprecating enums. """ - uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field( + 999, repeated=True + ) """The parser stores options it doesn't recognize here. See above.""" @@ -1076,7 +1102,9 @@ class EnumValueOptions(betterproto.Message): values. """ - uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field( + 999, repeated=True + ) """The parser stores options it doesn't recognize here. See above.""" @@ -1089,7 +1117,9 @@ class ServiceOptions(betterproto.Message): in the very least, this is a formalization for deprecating services. """ - uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field( + 999, repeated=True + ) """The parser stores options it doesn't recognize here. See above.""" @@ -1103,7 +1133,9 @@ class MethodOptions(betterproto.Message): """ idempotency_level: "MethodOptionsIdempotencyLevel" = betterproto.enum_field(34) - uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field(999) + uninterpreted_option: List["UninterpretedOption"] = betterproto.message_field( + 999, repeated=True + ) """The parser stores options it doesn't recognize here. See above.""" @@ -1118,7 +1150,9 @@ class UninterpretedOption(betterproto.Message): UninterpretedOptions in them. """ - name: List["UninterpretedOptionNamePart"] = betterproto.message_field(2) + name: List["UninterpretedOptionNamePart"] = betterproto.message_field( + 2, repeated=True + ) identifier_value: str = betterproto.string_field(3) """ The value of the uninterpreted option, in whatever type the tokenizer @@ -1153,7 +1187,9 @@ class SourceCodeInfo(betterproto.Message): FileDescriptorProto was generated. """ - location: List["SourceCodeInfoLocation"] = betterproto.message_field(1) + location: List["SourceCodeInfoLocation"] = betterproto.message_field( + 1, repeated=True + ) """ A Location identifies a piece of source code in a .proto file which corresponds to a particular definition. This information is intended to be @@ -1189,7 +1225,7 @@ class SourceCodeInfo(betterproto.Message): @dataclass(eq=False, repr=False) class SourceCodeInfoLocation(betterproto.Message): - path: List[int] = betterproto.int32_field(1) + path: List[int] = betterproto.int32_field(1, repeated=True) """ Identifies which part of the FileDescriptorProto was defined at this location. Each element is a field number or an index. They form a path @@ -1205,7 +1241,7 @@ class SourceCodeInfoLocation(betterproto.Message): declaration (from the beginning of the label to the terminating semicolon). """ - span: List[int] = betterproto.int32_field(2) + span: List[int] = betterproto.int32_field(2, repeated=True) """ Always has exactly three or four elements: start line, start column, end line (optional, otherwise assumed same as start line), end column. These @@ -1240,7 +1276,7 @@ class SourceCodeInfoLocation(betterproto.Message): """ trailing_comments: str = betterproto.string_field(4) - leading_detached_comments: List[str] = betterproto.string_field(6) + leading_detached_comments: List[str] = betterproto.string_field(6, repeated=True) @dataclass(eq=False, repr=False) @@ -1251,7 +1287,9 @@ class GeneratedCodeInfo(betterproto.Message): source file, but may contain references to different source .proto files. """ - annotation: List["GeneratedCodeInfoAnnotation"] = betterproto.message_field(1) + annotation: List["GeneratedCodeInfoAnnotation"] = betterproto.message_field( + 1, repeated=True + ) """ An Annotation connects some span of text in generated code to an element of its generating .proto file. @@ -1260,7 +1298,7 @@ class GeneratedCodeInfo(betterproto.Message): @dataclass(eq=False, repr=False) class GeneratedCodeInfoAnnotation(betterproto.Message): - path: List[int] = betterproto.int32_field(1) + path: List[int] = betterproto.int32_field(1, repeated=True) """ Identifies the element in the original source .proto file. This field is formatted the same as SourceCodeInfo.Location.path. @@ -1426,7 +1464,7 @@ class FieldMask(betterproto.Message): `INVALID_ARGUMENT` error if any path is unmappable. """ - paths: List[str] = betterproto.string_field(1) + paths: List[str] = betterproto.string_field(1, repeated=True) """The set of field mask paths.""" @@ -1441,8 +1479,13 @@ class Struct(betterproto.Message): representation for `Struct` is JSON object. """ - fields: Dict[str, "Value"] = betterproto.map_field( - 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE + fields: Dict[ + str, Union[None, int, float, str, bool, Dict[str, AnyType], List[AnyType]] + ] = betterproto.map_field( + 1, + betterproto.TYPE_STRING, + betterproto.TYPE_MESSAGE, + value_special=betterproto.SpecialTypes.GOOGLE_VALUE, ) """Unordered map of dynamically typed values.""" @@ -1457,7 +1500,9 @@ class Value(betterproto.Message): value. """ - null_value: "NullValue" = betterproto.enum_field(1, group="kind") + null_value: None = betterproto.enum_field( + 1, special=betterproto.SpecialTypes.GOOGLE_NULL_VALUE, group="kind" + ) """Represents a null value.""" number_value: float = betterproto.double_field(2, group="kind") @@ -1469,10 +1514,16 @@ class Value(betterproto.Message): bool_value: bool = betterproto.bool_field(4, group="kind") """Represents a boolean value.""" - struct_value: "Struct" = betterproto.message_field(5, group="kind") + struct_value: Dict[str, AnyType] = betterproto.message_field( + 5, special=betterproto.SpecialTypes.GOOGLE_STRUCT, group="kind" + ) """Represents a structured value.""" - list_value: "ListValue" = betterproto.message_field(6, group="kind") + list_value: List[ + Union[None, int, float, str, bool, Dict[str, AnyType], List[AnyType]] + ] = betterproto.message_field( + 6, special=betterproto.SpecialTypes.GOOGLE_LIST_VALUE, group="kind" + ) """Represents a repeated `Value`.""" @@ -1483,7 +1534,11 @@ class ListValue(betterproto.Message): representation for `ListValue` is JSON array. """ - values: List["Value"] = betterproto.message_field(1) + values: List[ + Union[None, int, float, str, bool, Dict[str, AnyType], List[AnyType]] + ] = betterproto.message_field( + 1, repeated=True, special=betterproto.SpecialTypes.GOOGLE_VALUE + ) """Repeated field of dynamically typed values.""" diff --git a/src/betterproto/plugin/models.py b/src/betterproto/plugin/models.py index 71c547145..c0d82c99b 100644 --- a/src/betterproto/plugin/models.py +++ b/src/betterproto/plugin/models.py @@ -403,6 +403,10 @@ def betterproto_field_args(self) -> List[str]: args = [] if self.field_wraps: args.append(f"wraps={self.field_wraps}") + if self.repeated: + args.append(f"repeated=True") + if self.field_special: + args.append(f"special={self.field_special}") if self.optional: args.append(f"optional=True") return args @@ -428,6 +432,11 @@ def typing_imports(self) -> Set[str]: imports.add("List") if "Dict[" in annotation: imports.add("Dict") + # TODO this is absolutely horrible, rewrite this to take it straight from get_type_reference + if "Union[" in annotation: + imports.add("Union") + if "AnyType" in annotation: + imports.add("Any as AnyType") return imports @property @@ -453,6 +462,14 @@ def field_wraps(self) -> Optional[str]: return f"betterproto.{wrapped_type}" return None + @property + def field_special(self) -> Optional[str]: + try: + special_type = betterproto.SpecialTypes(self.proto_obj.type_name) + return f"betterproto.SpecialTypes.{special_type.name}" + except ValueError: + return None + @property def repeated(self) -> bool: return ( @@ -573,6 +590,7 @@ class MapEntryCompiler(FieldCompiler): py_v_type: Type = PLACEHOLDER proto_k_type: str = PLACEHOLDER proto_v_type: str = PLACEHOLDER + proto_v_type_name: str = PLACEHOLDER def __post_init__(self) -> None: """Explore nested types and set k_type and v_type if unset.""" @@ -597,11 +615,18 @@ def __post_init__(self) -> None: # Get proto types self.proto_k_type = FieldDescriptorProtoType(nested.field[0].type).name self.proto_v_type = FieldDescriptorProtoType(nested.field[1].type).name + self.proto_v_type_name = nested.field[1].type_name super().__post_init__() # call FieldCompiler-> MessageCompiler __post_init__ @property def betterproto_field_args(self) -> List[str]: - return [f"betterproto.{self.proto_k_type}", f"betterproto.{self.proto_v_type}"] + field_args = [f"betterproto.{self.proto_k_type}", f"betterproto.{self.proto_v_type}"] + try: + special_type = betterproto.SpecialTypes(self.proto_v_type_name) + field_args.append(f"value_special=betterproto.SpecialTypes.{special_type.name}") + except ValueError: + pass + return field_args @property def field_type(self) -> str: diff --git a/tests/inputs/config.py b/tests/inputs/config.py index 49882b0d5..5c1ef2e25 100644 --- a/tests/inputs/config.py +++ b/tests/inputs/config.py @@ -2,8 +2,6 @@ # Remove from list when fixed. xfail = { "namespace_keywords", # 70 - "googletypes_struct", # 9 - "googletypes_value", # 9 "import_capitalized_package", "example", # This is the example in the readme. Not a test. } diff --git a/tests/inputs/googletypes_value/googletypes_value.json b/tests/inputs/googletypes_value/googletypes_value.json index db52d5c06..859af65ee 100644 --- a/tests/inputs/googletypes_value/googletypes_value.json +++ b/tests/inputs/googletypes_value/googletypes_value.json @@ -1,11 +1,3 @@ { - "value1": "hello world", - "value2": true, - "value3": 1, - "value4": null, - "value5": [ - 1, - 2, - 3 - ] + "value4": null } diff --git a/tests/inputs/googletypes_value/googletypes_value.proto b/tests/inputs/googletypes_value/googletypes_value.proto index d5089d5ef..976f135dc 100644 --- a/tests/inputs/googletypes_value/googletypes_value.proto +++ b/tests/inputs/googletypes_value/googletypes_value.proto @@ -7,9 +7,5 @@ import "google/protobuf/struct.proto"; // Tests that fields of type google.protobuf.Value can contain arbitrary JSON-values. message Test { - google.protobuf.Value value1 = 1; - google.protobuf.Value value2 = 2; - google.protobuf.Value value3 = 3; google.protobuf.Value value4 = 4; - google.protobuf.Value value5 = 5; } diff --git a/tests/inputs/googletypes_value/test_googletypes_value.py b/tests/inputs/googletypes_value/test_googletypes_value.py new file mode 100644 index 000000000..cb17fde7f --- /dev/null +++ b/tests/inputs/googletypes_value/test_googletypes_value.py @@ -0,0 +1,9 @@ +from tests.output_betterproto.googletypes_value import Test +from tests.util import get_test_case_json_data + + +def test_value(): + message = Test() + message.from_json(get_test_case_json_data("googletypes_value")[0].json) + assert message.value1 == "hello world" + assert message.value4 is None diff --git a/tests/test_get_ref_type.py b/tests/test_get_ref_type.py index 4883ead26..a6b702edd 100644 --- a/tests/test_get_ref_type.py +++ b/tests/test_get_ref_type.py @@ -16,18 +16,18 @@ ), ( ".google.protobuf.Struct", - '"betterproto_lib_google_protobuf.Struct"', - "import betterproto.lib.google.protobuf as betterproto_lib_google_protobuf", + 'Dict[str, AnyType]', + None, ), ( ".google.protobuf.ListValue", - '"betterproto_lib_google_protobuf.ListValue"', - "import betterproto.lib.google.protobuf as betterproto_lib_google_protobuf", + 'List[Union[None, int, float, str, bool, Dict[str, AnyType], List[AnyType]]]', + None, ), ( ".google.protobuf.Value", - '"betterproto_lib_google_protobuf.Value"', - "import betterproto.lib.google.protobuf as betterproto_lib_google_protobuf", + 'Union[None, int, float, str, bool, Dict[str, AnyType], List[AnyType]]', + None, ), ], ) @@ -38,9 +38,10 @@ def test_reference_google_wellknown_types_non_wrappers( name = get_type_reference(package="", imports=imports, source_type=google_type) assert name == expected_name - assert imports.__contains__( - expected_import - ), f"{expected_import} not found in {imports}" + if expected_import: + assert imports.__contains__( + expected_import + ), f"{expected_import} not found in {imports}" @pytest.mark.parametrize( diff --git a/tests/test_inputs.py b/tests/test_inputs.py index 2077fffd5..c5ac18bb6 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -209,14 +209,14 @@ def test_binary_compatibility(repeat, test_data: TestData) -> None: plugin_instance_from_json: betterproto.Message = ( plugin_module.Test().from_json(sample.json) ) - plugin_instance_from_binary = plugin_module.Test.FromString( - reference_binary_output - ) - # Generally this can't be relied on, but here we are aiming to match the # existing Python implementation and aren't doing anything tricky. # https://developers.google.com/protocol-buffers/docs/encoding#implications assert bytes(plugin_instance_from_json) == reference_binary_output + + plugin_instance_from_binary = plugin_module.Test.FromString( + reference_binary_output + ) assert bytes(plugin_instance_from_binary) == reference_binary_output assert plugin_instance_from_json == plugin_instance_from_binary From b43c882b2a11dd943d5c50e227a489b11517cfb0 Mon Sep 17 00:00:00 2001 From: ported Date: Wed, 8 Jun 2022 11:58:39 +0200 Subject: [PATCH 3/6] only some deserialization problems now --- src/betterproto/__init__.py | 436 +++++++++--------- src/betterproto/plugin/models.py | 4 +- .../googletypes_value/googletypes_value.json | 5 +- .../googletypes_value/googletypes_value.proto | 3 + .../test_proto3_field_presence_oneof.py | 6 +- 5 files changed, 234 insertions(+), 220 deletions(-) diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index 5d94eb694..abadafa03 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -6,7 +6,7 @@ import sys import typing import warnings -from abc import ABC +from abc import ABC, abstractmethod from base64 import ( b64decode, b64encode, @@ -150,7 +150,21 @@ class Casing(enum.Enum): SNAKE = snake_case #: A snake_case sterilization function. -PLACEHOLDER: Any = object() +class NotSetType: + def __repr__(self): + return "NOT_SET" + + +class PlaceholderType: + def __repr__(self): + return "PLACEHOLDER" + + +NOT_SET: Any = NotSetType() +PLACEHOLDER: Any = PlaceholderType() + +X = typing.TypeVar("X") +ProtoOptional = Union[X, NotSetType] @dataclasses.dataclass(frozen=True) @@ -193,7 +207,7 @@ def dataclass_field( ) -> dataclasses.Field: """Creates a dataclass field with attached protobuf metadata.""" return dataclasses.field( - default=None if optional else PLACEHOLDER, + default=NOT_SET if optional else PLACEHOLDER, metadata={ "betterproto": FieldMetadata( number, proto_type, map_types, repeated, group, wraps, special, optional @@ -374,7 +388,7 @@ def encode_varint(value: int) -> bytes: return bytes(b + [bits]) -def _preprocess_single(proto_type: str, wraps: str, special: Optional[SpecialTypes], value: Any) -> bytes: +def _preprocess_single(proto_type: str, wraps: str, value: Any) -> bytes: # print("_preprocess_single: proto_type:", proto_type, ", wraps:", wraps, ", special:", special, ", value:", value) """Adjusts values before serialization.""" if proto_type in ( @@ -406,11 +420,9 @@ def _preprocess_single(proto_type: str, wraps: str, special: Optional[SpecialTyp nanos = int((total_ms % 1e6) * 1e3) value = _Duration(seconds=seconds, nanos=nanos) elif wraps: - if value is None: + if value is NOT_SET: return b"" value = _get_wrapper(wraps)(value=value) - # elif special: - # value = get_special_type(special)(value) return bytes(value) return value @@ -423,10 +435,9 @@ def _serialize_single( *, serialize_empty: bool = False, wraps: str = "", - special: Optional[SpecialTypes] = None, ) -> bytes: """Serializes a single field and value.""" - value = _preprocess_single(proto_type, wraps, special, value) + value = _preprocess_single(proto_type, wraps, value) output = bytearray() if proto_type in WIRE_VARINT_TYPES: @@ -657,7 +668,7 @@ def __post_init__(self) -> None: group_current.setdefault(meta.group) value = self.__raw_get(field_name) - if value != PLACEHOLDER and not (meta.optional and value is None): + if value != PLACEHOLDER and not (meta.optional and value is NOT_SET): # Found a non-sentinel value all_sentinel = False @@ -712,17 +723,20 @@ def __repr__(self) -> str: ] return f"{self.__class__.__name__}({', '.join(parts)})" - if not TYPE_CHECKING: + def __get(self, name: str, notset_defaults: bool = False): + value = super().__getattribute__(name) + if value not in (PLACEHOLDER, NOT_SET,) or (value is NOT_SET and not notset_defaults): + return value + default_value = self._get_field_default(name) + if value is PLACEHOLDER: + super().__setattr__(name, default_value) + return default_value + if not TYPE_CHECKING: def __getattribute__(self, name: str) -> Any: - """ - Lazily initialize default values to avoid infinite recursion for recursive - message types - """ value = super().__getattribute__(name) if value is not PLACEHOLDER: return value - value = self._get_field_default(name) super().__setattr__(name, value) return value @@ -778,22 +792,22 @@ def __bytes__(self) -> bytes: """ output = bytearray() for field_name, meta in self._betterproto.meta_by_field_name.items(): - _value = getattr(self, field_name) # print("1. field_name:", field_name, ", meta:", meta, ", value: ", value) + if not self.is_set(field_name): + # Optional items should be skipped. This is used for the Google + # wrapper types and proto3 field presence/optional fields. + continue + + _value = self.__get(field_name) + # If this field is to be converted from/to a message type with special handling, convert it here # We skip this step if the value is repeated or a map to not infinitely recurse wrapping {} and [] if meta.special and not meta.map_types and not meta.repeated: - value = get_special_type(meta.special)(_value) + value = get_special_transform(meta.special).create_type(_value) else: value = _value - if value is None: - # Optional items should be skipped. This is used for the Google - # wrapper types and proto3 field presence/optional fields. - # print("2. field_name:", field_name, ", value is none, skipped") - continue - # Being selected in a group means this field is the one that is # currently set in a `oneof` group, so it must be serialized even # if the value is the default zero value. @@ -820,9 +834,6 @@ def __bytes__(self) -> bytes: # if this is the selected oneof item or if we know we have to # serialize an empty message (i.e. zero value was explicitly # set by the user). - - # TODO this skips google.protobuf.NullValue - # print("2. field_name:", field_name, ", value is default (", value, "), skipped") continue if isinstance(value, list): @@ -832,7 +843,7 @@ def __bytes__(self) -> bytes: # treat it like a field of raw bytes. buf = bytearray() for item in value: - buf += _preprocess_single(meta.proto_type, "", None, item) + buf += _preprocess_single(meta.proto_type, "", item) output += _serialize_single(meta.number, TYPE_BYTES, buf) else: for item in value: @@ -842,7 +853,6 @@ def __bytes__(self) -> bytes: meta.proto_type, item, wraps=meta.wraps or "", - special=meta.special, ) # if it's an empty message it still needs to be represented # as an item in the repeated list @@ -852,7 +862,7 @@ def __bytes__(self) -> bytes: elif isinstance(value, dict): for k, v in value.items(): if meta.special: - v = get_special_type(meta.special)(v) + v = get_special_transform(meta.special).create_type(v) # TODO get map_types from type referenced by meta.special if applicable? assert meta.map_types sk = _serialize_single(1, meta.map_types[0], k) @@ -875,8 +885,7 @@ def __bytes__(self) -> bytes: meta.proto_type, value, serialize_empty=serialize_empty or bool(selected_in_group), - wraps=meta.wraps or "", - special=meta.special, + wraps=meta.wraps or "" ) output += self._unknown_fields @@ -933,7 +942,7 @@ def _get_field_default_gen(cls, field: dataclasses.Field) -> Any: elif t.__origin__ in (list, List): # This is some kind of list (repeated) field. return list - elif t.__origin__ is Union and type(None) in t.__args__: + elif t.__origin__ is Union and (type(None) in t.__args__ or NotSetType in t.__args__): # This is an optional field (either wrapped, or using proto3 # field presence). For setting the default we really don't care # what kind of field it is. @@ -956,7 +965,10 @@ def _postprocess_single( ) -> Any: # print("wire_type: ", wire_type, ", meta: ", meta, ", field_name: ", field_name, ", value: ", value) """Adjusts values after parsing.""" - if wire_type == WIRE_VARINT: + if meta.special: + transform = get_special_transform(meta.special) + value = transform.parse(value) + elif wire_type == WIRE_VARINT: if meta.proto_type in (TYPE_INT32, TYPE_INT64): bits = int(meta.proto_type[3:]) value = value & ((1 << bits) - 1) @@ -976,9 +988,6 @@ def _postprocess_single( value = str(value, "utf-8") elif meta.proto_type == TYPE_MESSAGE: cls = self._betterproto.cls_by_field[field_name] - - # print("cls: ", cls, "meta: ", meta, ", value: ", value) - if cls == datetime: value = _Timestamp().parse(value).to_datetime() elif cls == timedelta: @@ -987,9 +996,6 @@ def _postprocess_single( # This is a Google wrapper value message around a single # scalar type. value = _get_wrapper(meta.wraps)().parse(value).value - elif meta.special: - # This is a Google well-known type that has special handling - value = get_special_type(meta.special)().parse(value) else: value = cls().parse(value) value._serialized_on_wire = True @@ -1055,7 +1061,7 @@ def parse(self: T, data: bytes) -> T: parsed.wire_type, meta, field_name, parsed.value ) - current = getattr(self, field_name) + current = self.__get(field_name) if meta.proto_type == TYPE_MAP: # Value represents a single key/value pair entry in the map. current[value.key] = value.value @@ -1113,16 +1119,15 @@ def to_dict( """ output: Dict[str, Any] = {} field_types = self._type_hints() - defaults = self._betterproto.default_gen for field_name, meta in self._betterproto.meta_by_field_name.items(): - field_is_repeated = defaults[field_name] is list - value = getattr(self, field_name) + value = self.__get(field_name, notset_defaults=include_default_values) + if not include_default_values and value is NOT_SET: + continue cased_name = casing(field_name).rstrip("_") # type: ignore if meta.proto_type == TYPE_MESSAGE: if isinstance(value, datetime): if ( value != DATETIME_ZERO - or include_default_values or self._include_default_value_for_oneof( field_name=field_name, meta=meta ) @@ -1131,7 +1136,6 @@ def to_dict( elif isinstance(value, timedelta): if ( value != timedelta(0) - or include_default_values or self._include_default_value_for_oneof( field_name=field_name, meta=meta ) @@ -1140,7 +1144,7 @@ def to_dict( elif meta.wraps or meta.special: if meta.special or value is not None or include_default_values: output[cased_name] = value - elif field_is_repeated: + elif meta.repeated: # Convert each item. cls = self._betterproto.cls_by_field[field_name] if cls == datetime: @@ -1151,11 +1155,9 @@ def to_dict( value = [ i.to_dict(casing, include_default_values) for i in value ] - if value or include_default_values: - output[cased_name] = value + output[cased_name] = value elif value is None: - if include_default_values: - output[cased_name] = value + output[cased_name] = value elif ( value._serialized_on_wire or include_default_values @@ -1170,8 +1172,7 @@ def to_dict( if hasattr(value[k], "to_dict"): output_map[k] = value[k].to_dict(casing, include_default_values) - if value or include_default_values: - output[cased_name] = output_map + output[cased_name] = output_map elif ( value != self._get_field_default(field_name) or include_default_values @@ -1180,24 +1181,23 @@ def to_dict( ) ): if meta.proto_type in INT_64_TYPES: - if field_is_repeated: + if meta.repeated: output[cased_name] = [str(n) for n in value] elif value is None: - if include_default_values: - output[cased_name] = value + output[cased_name] = value else: output[cased_name] = str(value) elif meta.proto_type == TYPE_BYTES: - if field_is_repeated: + if meta.repeated: output[cased_name] = [ b64encode(b).decode("utf8") for b in value ] - elif value is None and include_default_values: + elif value is None: output[cased_name] = value else: output[cased_name] = b64encode(value).decode("utf8") elif meta.proto_type == TYPE_ENUM: - if field_is_repeated: + if meta.repeated: enum_class = field_types[field_name].__args__[0] if isinstance(value, typing.Iterable) and not isinstance( value, str @@ -1207,8 +1207,7 @@ def to_dict( # transparently upgrade single value to repeated output[cased_name] = [enum_class(value).name] elif value is None: - if include_default_values: - output[cased_name] = value + output[cased_name] = value elif meta.optional: enum_class = field_types[field_name].__args__[0] output[cased_name] = enum_class(value).name @@ -1216,7 +1215,7 @@ def to_dict( enum_class = field_types[field_name] # noqa output[cased_name] = enum_class(value).name elif meta.proto_type in (TYPE_FLOAT, TYPE_DOUBLE): - if field_is_repeated: + if meta.repeated: output[cased_name] = [_dump_float(n) for n in value] else: output[cased_name] = _dump_float(value) @@ -1245,66 +1244,63 @@ def from_dict(self: T, value: Dict[str, Any]) -> T: meta = self._betterproto.meta_by_field_name.get(field_name) if not meta: continue - - # TODO none has to be an accepted value for google NullValue - # maybe it doesn't matter because the field will always be defaulted to None anyway? a bit confusing - if value[key] is not None: - if meta.proto_type == TYPE_MESSAGE: - v = getattr(self, field_name) - cls = self._betterproto.cls_by_field[field_name] - if isinstance(v, list): - if cls == datetime: - v = [isoparse(item) for item in value[key]] - elif cls == timedelta: - v = [ - timedelta(seconds=float(item[:-1])) - for item in value[key] - ] - else: - v = [cls().from_dict(item) for item in value[key]] - elif cls == datetime: - v = isoparse(value[key]) + if not meta.special and value[key] is None: + continue + if meta.proto_type == TYPE_MESSAGE: + v = self.__get(field_name, notset_defaults=True) + cls = self._betterproto.cls_by_field[field_name] + if isinstance(v, list): + if cls == datetime: + v = [isoparse(item) for item in value[key]] elif cls == timedelta: - v = timedelta(seconds=float(value[key][:-1])) - elif meta.wraps or meta.special: - v = value[key] - elif v is None: - v = cls().from_dict(value[key]) + v = [ + timedelta(seconds=float(item[:-1])) + for item in value[key] + ] else: - # NOTE: `from_dict` mutates the underlying message, so no - # assignment here is necessary. - v.from_dict(value[key]) - elif meta.map_types and meta.map_types[1] == TYPE_MESSAGE: - v = getattr(self, field_name) - cls = self._betterproto.cls_by_field[f"{field_name}.value"] - for k in value[key]: - v[k] = cls().from_dict(value[key][k]) - else: + v = [cls().from_dict(item) for item in value[key]] + elif cls == datetime: + v = isoparse(value[key]) + elif cls == timedelta: + v = timedelta(seconds=float(value[key][:-1])) + elif meta.wraps or meta.special: v = value[key] - if meta.proto_type in INT_64_TYPES: - if isinstance(value[key], list): - v = [int(n) for n in value[key]] - else: - v = int(value[key]) - elif meta.proto_type == TYPE_BYTES: - if isinstance(value[key], list): - v = [b64decode(n) for n in value[key]] - else: - v = b64decode(value[key]) - elif meta.proto_type == TYPE_ENUM: - enum_cls = self._betterproto.cls_by_field[field_name] - if isinstance(v, list): - v = [enum_cls.from_string(e) for e in v] - elif isinstance(v, str): - v = enum_cls.from_string(v) - elif meta.proto_type in (TYPE_FLOAT, TYPE_DOUBLE): - if isinstance(value[key], list): - v = [_parse_float(n) for n in value[key]] - else: - v = _parse_float(value[key]) - - if v is not None: - setattr(self, field_name, v) + elif v is None: + v = cls().from_dict(value[key]) + else: + # NOTE: `from_dict` mutates the underlying message, so no + # assignment here is necessary. + v.from_dict(value[key]) + elif meta.map_types and meta.map_types[1] == TYPE_MESSAGE: + v = self.__get(field_name, notset_defaults=True) + cls = self._betterproto.cls_by_field[f"{field_name}.value"] + for k in value[key]: + v[k] = cls().from_dict(value[key][k]) + else: + v = value[key] + if meta.proto_type in INT_64_TYPES: + if isinstance(value[key], list): + v = [int(n) for n in value[key]] + else: + v = int(value[key]) + elif meta.proto_type == TYPE_BYTES: + if isinstance(value[key], list): + v = [b64decode(n) for n in value[key]] + else: + v = b64decode(value[key]) + elif meta.proto_type == TYPE_ENUM: + enum_cls = self._betterproto.cls_by_field[field_name] + if isinstance(v, list): + v = [enum_cls.from_string(e) for e in v] + elif isinstance(v, str): + v = enum_cls.from_string(v) + elif meta.proto_type in (TYPE_FLOAT, TYPE_DOUBLE): + if isinstance(value[key], list): + v = [_parse_float(n) for n in value[key]] + else: + v = _parse_float(value[key]) + if v is not None: + setattr(self, field_name, v) return self def to_json(self, indent: Union[None, int, str] = None) -> str: @@ -1372,7 +1368,6 @@ def to_pydict( output: Dict[str, Any] = {} defaults = self._betterproto.default_gen for field_name, meta in self._betterproto.meta_by_field_name.items(): - field_is_repeated = defaults[field_name] is list value = getattr(self, field_name) cased_name = casing(field_name).rstrip("_") # type: ignore if meta.proto_type == TYPE_MESSAGE: @@ -1394,10 +1389,10 @@ def to_pydict( ) ): output[cased_name] = value - elif meta.wraps: - if value is not None or include_default_values: - output[cased_name] = value - elif field_is_repeated: + elif meta.wraps or meta.special: + if value is not NOT_SET or include_default_values: + output[cased_name] = None if NOT_SET else value + elif meta.repeated: # Convert each item. value = [i.to_pydict(casing, include_default_values) for i in value] if value or include_default_values: @@ -1449,33 +1444,31 @@ def from_pydict(self: T, value: Dict[str, Any]) -> T: if not meta: continue - if value[key] is not None: - if meta.proto_type == TYPE_MESSAGE: - v = getattr(self, field_name) - if isinstance(v, list): - cls = self._betterproto.cls_by_field[field_name] - for item in value[key]: - v.append(cls().from_pydict(item)) - elif isinstance(v, datetime): - v = value[key] - elif isinstance(v, timedelta): - v = value[key] - elif meta.wraps: - v = value[key] - else: - # NOTE: `from_pydict` mutates the underlying message, so no - # assignment here is necessary. - v.from_pydict(value[key]) - elif meta.map_types and meta.map_types[1] == TYPE_MESSAGE: - v = getattr(self, field_name) - cls = self._betterproto.cls_by_field[f"{field_name}.value"] - for k in value[key]: - v[k] = cls().from_pydict(value[key][k]) - else: + if meta.proto_type == TYPE_MESSAGE: + v = getattr(self, field_name) + if isinstance(v, list): + cls = self._betterproto.cls_by_field[field_name] + for item in value[key]: + v.append(cls().from_pydict(item)) + elif isinstance(v, datetime): + v = value[key] + elif isinstance(v, timedelta): v = value[key] + elif meta.wraps or meta.special: + v = value[key] + else: + # NOTE: `from_pydict` mutates the underlying message, so no + # assignment here is necessary. + v.from_pydict(value[key]) + elif meta.map_types and meta.map_types[1] == TYPE_MESSAGE: + v = getattr(self, field_name) + cls = self._betterproto.cls_by_field[f"{field_name}.value"] + for k in value[key]: + v[k] = cls().from_pydict(value[key][k]) + else: + v = value[key] - if v is not None: - setattr(self, field_name, v) + setattr(self, field_name, v) return self def is_set(self, name: str) -> bool: @@ -1495,7 +1488,7 @@ def is_set(self, name: str) -> bool: default = ( PLACEHOLDER if not self._betterproto.meta_by_field_name[name].optional - else None + else NOT_SET ) return self.__raw_get(name) is not default @@ -1603,85 +1596,100 @@ def _get_wrapper(proto_type: str) -> Type: }[proto_type] -class BetterprotoValue(Value): - # TODO replace this with type alias for Value - def __init__(self, value: Optional[Any] = None): - super().__init__() - # print("BetterprotoValue.__init__, value:", value) - if value: - if isinstance(value, str): - self.string_value = value - elif isinstance(value, bool): - self.bool_value = value - elif isinstance(value, int) or isinstance(value, float): - self.number_value = value - elif isinstance(value, dict) and all(isinstance(k, str) for k in value.keys()): - self.struct_value = value - elif isinstance(value, list): - self.list_value = value - elif value is None: - self.null_value = value - else: - raise TypeError(f"Value '{value}' with type '{type(value)}'" - f" is not supported for .google.protobuf.Value") - # print("BetterprotoValue.__init__, self result:", self) +class SpecialTransform(ABC): + @staticmethod + @abstractmethod + def create_type(value: Optional[Any]): + """ + Creates a specially handled type from the given value + e.g. dict -> .google.protobuf.Struct + """ + raise NotImplementedError - def parse(self: T, data: bytes) -> T: - result = super().parse(data) - # print("BetterprotoValue.parse, super parse result: ", result) - return betterproto.which_one_of(result, "kind")[1] + @staticmethod + @abstractmethod + def parse(data: bytes): + """ + Parses the given buffer as specially handled type down to the contained value + e.g. bytes -> .google.protobuf.Struct -> dict + """ + raise NotImplementedError -class BetterprotoStruct(Struct): - # TODO replace this with type alias for Struct / jsonobject - def __init__(self, value: Optional[Dict[str, Any]] = None): - # print("BetterprotoStruct.__init__, value:", value) - # TODO is this correct or do we need to manually convert the Value items here? - if value: - # super().__init__(fields={k: BetterprotoValue(v) for k, v in value.items()}) - super().__init__(fields=value) - # print("BetterprotoStruct.__init__, self result:", self) +class ValueTransform(SpecialTransform): + + # TODO replace type hint for value with JSONValue type + @staticmethod + def create_type(value: Optional[Any] = PLACEHOLDER) -> Value: + message = Value() + if value is PLACEHOLDER: + return message + if isinstance(value, str): + message.string_value = value + elif isinstance(value, bool): + message.bool_value = value + elif isinstance(value, int) or isinstance(value, float): + message.number_value = value + elif isinstance(value, dict) and all(isinstance(k, str) for k in value.keys()): + message.struct_value = value + elif isinstance(value, list): + message.list_value = value + elif value is None: + message.null_value = value else: - super().__init__() + raise TypeError(f"Value '{value}' with type '{type(value)}'" + f" is not supported for .google.protobuf.Value") + return message + + # TODO replace the return type with type alias for JSONValue + @staticmethod + def parse(data) -> Any: + value = Value().parse(data) + return betterproto.which_one_of(value, "kind")[1] - def parse(self: T, data: bytes): - result = super().parse(data) - # print("BetterprotoStruct.parse, super parse result: ", result) - # TODO is this correct or do we need to manually parse the Value items here? - return result.fields +class StructTransform(SpecialTransform): -class BetterprotoListValue(ListValue): - # TODO replace the Any with type alias for Value - def __init__(self, value: Optional[List[Any]] = None): - # print("BetterprotoListValue.__init__, value:", value) + @staticmethod + def create_type(value: Optional[Dict[str, Any]] = None): if value: - super().__init__(values=value) - # print("BetterprotoListValue.__init__, self result:", self) - else: - super().__init__() + return Struct(fields=value) + return Struct() + + @staticmethod + def parse(data): + return Struct().parse(data).fields + + +class ListValueTransform(SpecialTransform): + + @staticmethod + def create_type(value: Optional[List[Any]]): + if value: + return ListValue(values=value) + return ListValue() + + @staticmethod + def parse(data): + return ListValue().parse(data).values - def parse(self: T, data: bytes): - result = super().parse(data) - # print("BetterprotoListValue.parse, super parse result: ", result) - # TODO is this correct or do we need to manually parse the Value items here? - return result.values +class NullValueTransform(SpecialTransform): -class BetterprotoNullValue: - def __new__(cls, value: Optional[None] = PLACEHOLDER): + @staticmethod + def create_type(value: Optional[None] = PLACEHOLDER): return NullValue(NullValue.NULL_VALUE) - # TODO what - def parse(self: T, _: bytes): + @staticmethod + def parse(_): + # If a NullValue exists, the result is always None return None -def get_special_type(special_type: SpecialTypes): - # TODO include NullValue +def get_special_transform(special_type: SpecialTypes): return { - SpecialTypes.GOOGLE_VALUE: BetterprotoValue, - SpecialTypes.GOOGLE_STRUCT: BetterprotoStruct, - SpecialTypes.GOOGLE_LIST_VALUE: BetterprotoListValue, - SpecialTypes.GOOGLE_NULL_VALUE: BetterprotoNullValue, + SpecialTypes.GOOGLE_VALUE: ValueTransform, + SpecialTypes.GOOGLE_STRUCT: StructTransform, + SpecialTypes.GOOGLE_LIST_VALUE: ListValueTransform, + SpecialTypes.GOOGLE_NULL_VALUE: NullValueTransform, }.get(special_type, None) diff --git a/src/betterproto/plugin/models.py b/src/betterproto/plugin/models.py index c0d82c99b..33895ac51 100644 --- a/src/betterproto/plugin/models.py +++ b/src/betterproto/plugin/models.py @@ -501,7 +501,7 @@ def default_value_string(self) -> str: if self.repeated: return "[]" if self.optional: - return "None" + return "betterproto.NotSetType" if self.py_type == "int": return "0" if self.py_type == "float": @@ -570,7 +570,7 @@ def annotation(self) -> str: if self.repeated: return f"List[{py_type}]" if self.optional: - return f"Optional[{py_type}]" + return f"betterproto.ProtoOptional[{py_type}]" return py_type diff --git a/tests/inputs/googletypes_value/googletypes_value.json b/tests/inputs/googletypes_value/googletypes_value.json index 859af65ee..f8207e095 100644 --- a/tests/inputs/googletypes_value/googletypes_value.json +++ b/tests/inputs/googletypes_value/googletypes_value.json @@ -1,3 +1,6 @@ { + "value1": "hello world", + "value2": true, + "value3": 1, "value4": null -} +} \ No newline at end of file diff --git a/tests/inputs/googletypes_value/googletypes_value.proto b/tests/inputs/googletypes_value/googletypes_value.proto index 976f135dc..6cf2f3b0c 100644 --- a/tests/inputs/googletypes_value/googletypes_value.proto +++ b/tests/inputs/googletypes_value/googletypes_value.proto @@ -7,5 +7,8 @@ import "google/protobuf/struct.proto"; // Tests that fields of type google.protobuf.Value can contain arbitrary JSON-values. message Test { + google.protobuf.Value value1 = 1; + google.protobuf.Value value2 = 2; + google.protobuf.Value value3 = 3; google.protobuf.Value value4 = 4; } diff --git a/tests/inputs/proto3_field_presence_oneof/test_proto3_field_presence_oneof.py b/tests/inputs/proto3_field_presence_oneof/test_proto3_field_presence_oneof.py index d5f69d01a..9e5ce8a3b 100644 --- a/tests/inputs/proto3_field_presence_oneof/test_proto3_field_presence_oneof.py +++ b/tests/inputs/proto3_field_presence_oneof/test_proto3_field_presence_oneof.py @@ -1,3 +1,4 @@ +import betterproto from tests.output_betterproto.proto3_field_presence_oneof import ( InnerNested, Nested, @@ -17,8 +18,7 @@ def test_empty_nested(message: Test) -> None: assert bytes(message) == bytearray.fromhex("0a 00") test_empty_nested(Test(nested=Nested())) - test_empty_nested(Test(nested=Nested(inner=None))) - test_empty_nested(Test(nested=Nested(inner=InnerNested(a=None)))) + test_empty_nested(Test(nested=Nested(inner=InnerNested(a=betterproto.NOT_SET)))) def test_empty_with_optional(message: Test) -> None: # '12' => tag 2, length delimited @@ -26,4 +26,4 @@ def test_empty_with_optional(message: Test) -> None: assert bytes(message) == bytearray.fromhex("12 00") test_empty_with_optional(Test(with_optional=WithOptional())) - test_empty_with_optional(Test(with_optional=WithOptional(b=None))) + test_empty_with_optional(Test(with_optional=WithOptional(b=betterproto.NOT_SET))) From 1801b5592dce5d924f897823fe5eab0c1600373a Mon Sep 17 00:00:00 2001 From: ported Date: Wed, 8 Jun 2022 15:45:48 +0200 Subject: [PATCH 4/6] some cleanup, make ListValue work --- src/betterproto/__init__.py | 25 ++++++++----------- src/betterproto/plugin/models.py | 1 - .../googletypes_value/googletypes_value.json | 7 +++++- .../googletypes_value/googletypes_value.proto | 1 + tests/test_inputs.py | 2 +- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index abadafa03..4fbbeedb7 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -389,7 +389,6 @@ def encode_varint(value: int) -> bytes: def _preprocess_single(proto_type: str, wraps: str, value: Any) -> bytes: - # print("_preprocess_single: proto_type:", proto_type, ", wraps:", wraps, ", special:", special, ", value:", value) """Adjusts values before serialization.""" if proto_type in ( TYPE_ENUM, @@ -792,21 +791,17 @@ def __bytes__(self) -> bytes: """ output = bytearray() for field_name, meta in self._betterproto.meta_by_field_name.items(): - # print("1. field_name:", field_name, ", meta:", meta, ", value: ", value) - if not self.is_set(field_name): # Optional items should be skipped. This is used for the Google # wrapper types and proto3 field presence/optional fields. continue - _value = self.__get(field_name) + value = self.__get(field_name) # If this field is to be converted from/to a message type with special handling, convert it here # We skip this step if the value is repeated or a map to not infinitely recurse wrapping {} and [] if meta.special and not meta.map_types and not meta.repeated: - value = get_special_transform(meta.special).create_type(_value) - else: - value = _value + value = get_special_transform(meta.special).create_type(value) # Being selected in a group means this field is the one that is # currently set in a `oneof` group, so it must be serialized even @@ -843,10 +838,14 @@ def __bytes__(self) -> bytes: # treat it like a field of raw bytes. buf = bytearray() for item in value: + if meta.special: + item = get_special_transform(meta.special).create_type(item) buf += _preprocess_single(meta.proto_type, "", item) output += _serialize_single(meta.number, TYPE_BYTES, buf) else: for item in value: + if meta.special: + item = get_special_transform(meta.special).create_type(item) output += ( _serialize_single( meta.number, @@ -863,7 +862,6 @@ def __bytes__(self) -> bytes: for k, v in value.items(): if meta.special: v = get_special_transform(meta.special).create_type(v) - # TODO get map_types from type referenced by meta.special if applicable? assert meta.map_types sk = _serialize_single(1, meta.map_types[0], k) sv = _serialize_single(2, meta.map_types[1], v) @@ -963,12 +961,8 @@ def _get_field_default_gen(cls, field: dataclasses.Field) -> Any: def _postprocess_single( self, wire_type: int, meta: FieldMetadata, field_name: str, value: Any ) -> Any: - # print("wire_type: ", wire_type, ", meta: ", meta, ", field_name: ", field_name, ", value: ", value) """Adjusts values after parsing.""" - if meta.special: - transform = get_special_transform(meta.special) - value = transform.parse(value) - elif wire_type == WIRE_VARINT: + if wire_type == WIRE_VARINT: if meta.proto_type in (TYPE_INT32, TYPE_INT64): bits = int(meta.proto_type[3:]) value = value & ((1 << bits) - 1) @@ -1038,7 +1032,10 @@ def parse(self: T, data: bytes) -> T: meta = proto_meta.meta_by_field_name[field_name] value: Any - if parsed.wire_type == WIRE_LEN_DELIM and meta.proto_type in PACKED_TYPES: + if meta.special: + transform = get_special_transform(meta.special) + value = transform.parse(parsed.value) + elif parsed.wire_type == WIRE_LEN_DELIM and meta.proto_type in PACKED_TYPES: # This is a packed repeated field. pos = 0 value = [] diff --git a/src/betterproto/plugin/models.py b/src/betterproto/plugin/models.py index 33895ac51..01cd4c07b 100644 --- a/src/betterproto/plugin/models.py +++ b/src/betterproto/plugin/models.py @@ -432,7 +432,6 @@ def typing_imports(self) -> Set[str]: imports.add("List") if "Dict[" in annotation: imports.add("Dict") - # TODO this is absolutely horrible, rewrite this to take it straight from get_type_reference if "Union[" in annotation: imports.add("Union") if "AnyType" in annotation: diff --git a/tests/inputs/googletypes_value/googletypes_value.json b/tests/inputs/googletypes_value/googletypes_value.json index f8207e095..0b30752c9 100644 --- a/tests/inputs/googletypes_value/googletypes_value.json +++ b/tests/inputs/googletypes_value/googletypes_value.json @@ -2,5 +2,10 @@ "value1": "hello world", "value2": true, "value3": 1, - "value4": null + "value4": null, + "value5": [ + 1, + 2, + 3 + ] } \ No newline at end of file diff --git a/tests/inputs/googletypes_value/googletypes_value.proto b/tests/inputs/googletypes_value/googletypes_value.proto index 6cf2f3b0c..d5089d5ef 100644 --- a/tests/inputs/googletypes_value/googletypes_value.proto +++ b/tests/inputs/googletypes_value/googletypes_value.proto @@ -11,4 +11,5 @@ message Test { google.protobuf.Value value2 = 2; google.protobuf.Value value3 = 3; google.protobuf.Value value4 = 4; + google.protobuf.Value value5 = 5; } diff --git a/tests/test_inputs.py b/tests/test_inputs.py index c5ac18bb6..ea5430a63 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -214,7 +214,7 @@ def test_binary_compatibility(repeat, test_data: TestData) -> None: # https://developers.google.com/protocol-buffers/docs/encoding#implications assert bytes(plugin_instance_from_json) == reference_binary_output - plugin_instance_from_binary = plugin_module.Test.FromString( + plugin_instance_from_binary = plugin_module.Test().parse( reference_binary_output ) assert bytes(plugin_instance_from_binary) == reference_binary_output From d45f64a8ad3ab64611460371d870d8e82e221d54 Mon Sep 17 00:00:00 2001 From: ported Date: Thu, 29 Sep 2022 20:22:13 +0200 Subject: [PATCH 5/6] better test case for struct --- tests/inputs/googletypes_struct/googletypes_struct.json | 5 ++++- tests/inputs/googletypes_struct/googletypes_struct.proto | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/inputs/googletypes_struct/googletypes_struct.json b/tests/inputs/googletypes_struct/googletypes_struct.json index ecc175e06..27707aa28 100644 --- a/tests/inputs/googletypes_struct/googletypes_struct.json +++ b/tests/inputs/googletypes_struct/googletypes_struct.json @@ -1,5 +1,8 @@ { "struct": { - "key": true + "key": 1 + }, + "emptyStruct": { + } } diff --git a/tests/inputs/googletypes_struct/googletypes_struct.proto b/tests/inputs/googletypes_struct/googletypes_struct.proto index 2b8b5c55d..1fa0a957e 100644 --- a/tests/inputs/googletypes_struct/googletypes_struct.proto +++ b/tests/inputs/googletypes_struct/googletypes_struct.proto @@ -6,4 +6,5 @@ import "google/protobuf/struct.proto"; message Test { google.protobuf.Struct struct = 1; + google.protobuf.Struct empty_struct = 2; } From 00dce35ed5fb4ed8c6bfe12a41f79b32af9e56a9 Mon Sep 17 00:00:00 2001 From: ported Date: Thu, 29 Sep 2022 20:23:58 +0200 Subject: [PATCH 6/6] this is probably better? --- src/betterproto/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/betterproto/__init__.py b/src/betterproto/__init__.py index 4fbbeedb7..0f5dd1727 100644 --- a/src/betterproto/__init__.py +++ b/src/betterproto/__init__.py @@ -1032,10 +1032,7 @@ def parse(self: T, data: bytes) -> T: meta = proto_meta.meta_by_field_name[field_name] value: Any - if meta.special: - transform = get_special_transform(meta.special) - value = transform.parse(parsed.value) - elif parsed.wire_type == WIRE_LEN_DELIM and meta.proto_type in PACKED_TYPES: + if parsed.wire_type == WIRE_LEN_DELIM and meta.proto_type in PACKED_TYPES: # This is a packed repeated field. pos = 0 value = [] @@ -1053,6 +1050,9 @@ def parse(self: T, data: bytes) -> T: wire_type, meta, field_name, decoded ) value.append(decoded) + elif meta.special: + transform = get_special_transform(meta.special) + value = transform.parse(parsed.value) else: value = self._postprocess_single( parsed.wire_type, meta, field_name, parsed.value