From 126fe12b73c712920b80c96431b603c9c9a871bc Mon Sep 17 00:00:00 2001 From: Julien Cassis Date: Tue, 5 Dec 2023 14:16:37 -0500 Subject: [PATCH] Clean up marshaller added hex support --- cmd/tools/print/tools_print.go | 6 +--- go.mod | 6 ++-- go.sum | 10 ++----- jsonencoder/encoder.go | 9 ++---- jsonencoder/marshallers.go | 45 ++++++++++++++++++++++++++++ jsonencoder/options.go | 32 -------------------- jsonencoder/proto.go | 21 ------------- protoregistry/generator/generator.go | 2 +- protoregistry/registry.go | 10 ++++--- unsafe_extensions.go | 4 --- 10 files changed, 60 insertions(+), 85 deletions(-) create mode 100644 jsonencoder/marshallers.go delete mode 100644 jsonencoder/options.go delete mode 100644 jsonencoder/proto.go diff --git a/cmd/tools/print/tools_print.go b/cmd/tools/print/tools_print.go index 6f35031..1b99e17 100644 --- a/cmd/tools/print/tools_print.go +++ b/cmd/tools/print/tools_print.go @@ -277,9 +277,5 @@ func SetupJsonEncoder(cmd *cobra.Command) (*jsonencoder.Encoder, error) { } pbregistry.Extends(protoregistry.WellKnownRegistry) - - options := []jsonencoder.Option{ - jsonencoder.WithBytesAsHex(), - } - return jsonencoder.New(pbregistry, options...), nil + return jsonencoder.New(pbregistry), nil } diff --git a/go.mod b/go.mod index 6adc290..fe5616a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( buf.build/gen/go/bufbuild/reflect/connectrpc/go v1.12.0-20230822193137-310c9c4845dd.1 buf.build/gen/go/bufbuild/reflect/protocolbuffers/go v1.31.0-20230822193137-310c9c4845dd.2 + connectrpc.com/connect v1.12.0 github.com/ShinyTrinkets/overseer v0.3.0 github.com/dustin/go-humanize v1.0.1 github.com/go-json-experiment/json v0.0.0-20231013223334-54c864be5b8d @@ -40,7 +41,6 @@ require ( ) require ( - connectrpc.com/connect v1.12.0 // indirect github.com/bufbuild/protocompile v0.4.0 // indirect github.com/google/s2a-go v0.1.4 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect @@ -75,7 +75,7 @@ require ( github.com/bits-and-blooms/bitset v1.3.1 // indirect github.com/blendle/zapdriver v1.3.2-0.20200203083823-9200777f8a3d // indirect github.com/bobg/go-generics/v2 v2.1.1 // indirect - github.com/bufbuild/connect-go v1.10.0 + github.com/bufbuild/connect-go v1.10.0 // indirect github.com/bufbuild/connect-grpchealth-go v1.1.1 // indirect github.com/bufbuild/connect-grpcreflect-go v1.0.0 // indirect github.com/bufbuild/connect-opentelemetry-go v0.3.0 // indirect @@ -228,5 +228,5 @@ require ( replace ( github.com/ShinyTrinkets/overseer => github.com/streamingfast/overseer v0.2.1-0.20210326144022-ee491780e3ef github.com/bytecodealliance/wasmtime-go/v4 => github.com/streamingfast/wasmtime-go/v4 v4.0.0-freemem3 - github.com/jhump/protoreflect => github.com/streamingfast/protoreflect v0.0.0-20230414203421-018294174fdc + github.com/jhump/protoreflect => github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d ) diff --git a/go.sum b/go.sum index 8933791..d1d2077 100644 --- a/go.sum +++ b/go.sum @@ -578,12 +578,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/streamingfast/bstream v0.0.2-0.20231205163051-ade2f311eca3 h1:u8orpRssS8rYceziOQ/mbBQHlYh5w06oOtTXK90/yMc= -github.com/streamingfast/bstream v0.0.2-0.20231205163051-ade2f311eca3/go.mod h1:08GVb+DXyz6jVNIsbf+2zlaC81UeEGu5o1h49KrSR3Y= -github.com/streamingfast/bstream v0.0.2-0.20231205174934-869fb7d64fd2 h1:TQPPxjBXflVinpSSbYaiMuNgw1HB1YnMcFiR52M8EVo= -github.com/streamingfast/bstream v0.0.2-0.20231205174934-869fb7d64fd2/go.mod h1:08GVb+DXyz6jVNIsbf+2zlaC81UeEGu5o1h49KrSR3Y= -github.com/streamingfast/bstream v0.0.2-0.20231205175345-609448673b00 h1:U/8aQZOpOzLTVcuEVdbEVffVu00ixotkTe8DRhEXxao= -github.com/streamingfast/bstream v0.0.2-0.20231205175345-609448673b00/go.mod h1:08GVb+DXyz6jVNIsbf+2zlaC81UeEGu5o1h49KrSR3Y= github.com/streamingfast/bstream v0.0.2-0.20231205185208-7e21cc7e64bc h1:ioohiLa+d59fqToa2OhbUx418YMrqt2bLT+m+fmjOG8= github.com/streamingfast/bstream v0.0.2-0.20231205185208-7e21cc7e64bc/go.mod h1:08GVb+DXyz6jVNIsbf+2zlaC81UeEGu5o1h49KrSR3Y= github.com/streamingfast/cli v0.0.4-0.20230825151644-8cc84512cd80 h1:UxJUTcEVkdZy8N77E3exz0iNlgQuxl4m220GPvzdZ2s= @@ -619,8 +613,8 @@ github.com/streamingfast/overseer v0.2.1-0.20210326144022-ee491780e3ef h1:9IVFHR github.com/streamingfast/overseer v0.2.1-0.20210326144022-ee491780e3ef/go.mod h1:cq8CvbZ3ioFmGrHokSAJalS0lC+pVXLKhITScItUGXY= github.com/streamingfast/pbgo v0.0.6-0.20231120172814-537d034aad5e h1:8hoT2QUwh+YNgIcCPux9xd4u9XojHR8hbyAzz7rQuEM= github.com/streamingfast/pbgo v0.0.6-0.20231120172814-537d034aad5e/go.mod h1:fZuijmeFrqxW2YnnXmGrkQpUTHx3eHCaJUKwdvXAYKM= -github.com/streamingfast/protoreflect v0.0.0-20230414203421-018294174fdc h1:poYChURzYXislOzzeo44FKipd3wWvxhlz966qzO9kZk= -github.com/streamingfast/protoreflect v0.0.0-20230414203421-018294174fdc/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= +github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d h1:33VIARqUqBUKXJcuQoOS1rVSms54tgxhhNCmrLptpLg= +github.com/streamingfast/protoreflect v0.0.0-20231205191344-4b629d20ce8d/go.mod h1:aBJivEdekmFWYSQ29EE/fN9IanJWJXbtjy3ky0XD/jE= github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9 h1:YRwpVvLYa+FEJlTy0S7mk4UptYjk5zac+A+ZE1phOeA= github.com/streamingfast/sf-tracing v0.0.0-20230616174903-cd2ade641ca9/go.mod h1:ktzt1BUj3GF+SKQHEmn3ShryJ7y87JeCHtaTGaDVATs= github.com/streamingfast/shutter v1.5.0 h1:NpzDYzj0HVpSiDJVO/FFSL6QIK/YKOxY0gJAtyaTOgs= diff --git a/jsonencoder/encoder.go b/jsonencoder/encoder.go index ea667b5..744ddb2 100644 --- a/jsonencoder/encoder.go +++ b/jsonencoder/encoder.go @@ -14,17 +14,12 @@ type Encoder struct { marshallers []*json.Marshalers } -func New(files *protoregistry.Registry, opts ...Option) *Encoder { +func New(files *protoregistry.Registry) *Encoder { e := &Encoder{ protoRegistry: files, } - e.marshallers = []*json.Marshalers{ - json.MarshalFuncV2(e.protoAny), - } - - for _, opt := range opts { - opt(e) + json.MarshalFuncV2(e.anypb), } return e } diff --git a/jsonencoder/marshallers.go b/jsonencoder/marshallers.go new file mode 100644 index 0000000..b389bff --- /dev/null +++ b/jsonencoder/marshallers.go @@ -0,0 +1,45 @@ +package jsonencoder + +import ( + "encoding/hex" + "fmt" + "strings" + + "github.com/jhump/protoreflect/dynamic" + + "github.com/mr-tron/base58" + + "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" + "google.golang.org/protobuf/types/known/anypb" +) + +func (e *Encoder) anypb(encoder *jsontext.Encoder, t *anypb.Any, options json.Options) error { + msg, err := e.protoRegistry.Unmarshall(t) + if err != nil { + return fmt.Errorf("unmarshalling proto any: %w", err) + } + setBytesEncoder(t.TypeUrl) + cnt, err := json.Marshal(msg, json.WithMarshalers(json.NewMarshalers(e.marshallers...))) + if err != nil { + return fmt.Errorf("json marshalling proto any: %w", err) + } + return encoder.WriteValue(cnt) +} + +func (e *Encoder) base58Bytes(encoder *jsontext.Encoder, t []byte, options json.Options) error { + return encoder.WriteToken(jsontext.String(base58.Encode(t))) +} + +func (e *Encoder) hexBytes(encoder *jsontext.Encoder, t []byte, options json.Options) error { + return encoder.WriteToken(jsontext.String(hex.EncodeToString(t))) +} + +func setBytesEncoder(typeURL string) { + if strings.Contains(typeURL, "solana") { + dynamic.SetDefaultBytesRepresentation(dynamic.BytesAsBase58) + return + } else { + dynamic.SetDefaultBytesRepresentation(dynamic.BytesAsHex) + } +} diff --git a/jsonencoder/options.go b/jsonencoder/options.go deleted file mode 100644 index 3b6c550..0000000 --- a/jsonencoder/options.go +++ /dev/null @@ -1,32 +0,0 @@ -package jsonencoder - -import ( - "encoding/hex" - "fmt" - - "github.com/go-json-experiment/json" - "github.com/go-json-experiment/json/jsontext" - "github.com/mr-tron/base58" -) - -type Option func(c *Encoder) - -func WithBytesAsBase58() Option { - return func(c *Encoder) { - m := json.MarshalFuncV2(func(encoder *jsontext.Encoder, t []byte, options json.Options) error { - fmt.Println("base58", hex.EncodeToString(t)) - return encoder.WriteToken(jsontext.String(hex.EncodeToString(t))) - }) - c.marshallers = append(c.marshallers, m) - } -} - -func WithBytesAsHex() Option { - return func(c *Encoder) { - m := json.MarshalFuncV2(func(encoder *jsontext.Encoder, t []byte, options json.Options) error { - fmt.Println("hex", hex.EncodeToString(t)) - return encoder.WriteToken(jsontext.String(base58.Encode(t))) - }) - c.marshallers = append(c.marshallers, m) - } -} diff --git a/jsonencoder/proto.go b/jsonencoder/proto.go deleted file mode 100644 index a787e6c..0000000 --- a/jsonencoder/proto.go +++ /dev/null @@ -1,21 +0,0 @@ -package jsonencoder - -import ( - "fmt" - - "github.com/go-json-experiment/json" - "github.com/go-json-experiment/json/jsontext" - "google.golang.org/protobuf/types/known/anypb" -) - -func (e *Encoder) protoAny(encoder *jsontext.Encoder, t *anypb.Any, options json.Options) error { - msg, err := e.protoRegistry.Unmarshall(t.TypeUrl, t.Value) - if err != nil { - return fmt.Errorf("unmarshalling proto any: %w", err) - } - cnt, err := json.Marshal(msg, json.WithMarshalers(json.NewMarshalers(e.marshallers...))) - if err != nil { - return fmt.Errorf("json marshalling proto any: %w", err) - } - return encoder.WriteValue(cnt) -} diff --git a/protoregistry/generator/generator.go b/protoregistry/generator/generator.go index 7fd9f67..25de309 100644 --- a/protoregistry/generator/generator.go +++ b/protoregistry/generator/generator.go @@ -67,7 +67,7 @@ func main() { for _, file := range fileDescriptorSet.Msg.FileDescriptorSet.File { cnt, err := proto.Marshal(file) if err != nil { - log.Fatalf("failed to marshall proto file %s: %v", file.Name, err) + log.Fatalf("failed to marshall proto file %s: %v", file.GetName(), err) return } name := "" diff --git a/protoregistry/registry.go b/protoregistry/registry.go index 574c5e5..e8b7775 100644 --- a/protoregistry/registry.go +++ b/protoregistry/registry.go @@ -4,6 +4,8 @@ import ( "fmt" "strings" + "google.golang.org/protobuf/types/known/anypb" + "github.com/jhump/protoreflect/desc" "github.com/jhump/protoreflect/dynamic" ) @@ -37,18 +39,18 @@ func (r *Registry) RegisterFileDescriptor(f *desc.FileDescriptor) { r.filesDescriptors = append(r.filesDescriptors, f) } -func (r *Registry) Unmarshall(typeURL string, value []byte) (*dynamic.Message, error) { +func (r *Registry) Unmarshall(t *anypb.Any) (*dynamic.Message, error) { for _, fd := range r.filesDescriptors { - md := fd.FindSymbol(cleanTypeURL(typeURL)) + md := fd.FindSymbol(cleanTypeURL(t.TypeUrl)) if md != nil { dynMsg := dynamic.NewMessageFactoryWithDefaults().NewDynamicMessage(md.(*desc.MessageDescriptor)) - if err := dynMsg.Unmarshal(value); err != nil { + if err := dynMsg.Unmarshal(t.Value); err != nil { return nil, fmt.Errorf("unmarshalling proto: %w", err) } return dynMsg, nil } } - return nil, fmt.Errorf("no message descriptor in registry for type url: %s", typeURL) + return nil, fmt.Errorf("no message descriptor in registry for type url: %s", t.TypeUrl) } func (r *Registry) Extends(registry *Registry) { diff --git a/unsafe_extensions.go b/unsafe_extensions.go index c014d8d..9a847c5 100644 --- a/unsafe_extensions.go +++ b/unsafe_extensions.go @@ -3,14 +3,10 @@ package firecore import ( "context" - pbbstream "github.com/streamingfast/bstream/pb/sf/bstream/v1" "github.com/streamingfast/dlauncher/launcher" "go.uber.org/zap" ) -var UnsafePayloadKind pbbstream.Protocol = pbbstream.Protocol_UNKNOWN -var UnsafeJsonBytesEncoder = "hex" - // UnsafeResolveReaderNodeStartBlock is a function that resolved the reader node start block num, by default it simply // returns the value of the 'reader-node-start-block-num'. However, the function may be overwritten in certain chains // to perform a more complex resolution logic.