From 1ca402fccca3dd7b4480c0fd1dbbae2c7d550ca3 Mon Sep 17 00:00:00 2001 From: Joshua Williams Date: Wed, 15 Dec 2021 17:44:33 -0500 Subject: [PATCH] Proto fix (#509) * Fixed protobuf conversion logic * Added tests --- operator/builtin/output/googlecloud/entry.go | 3 +- operator/builtin/output/googlecloud/proto.go | 116 ++++++++++++++++++ .../builtin/output/googlecloud/proto_test.go | 97 +++++++++++++++ 3 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 operator/builtin/output/googlecloud/proto.go create mode 100644 operator/builtin/output/googlecloud/proto_test.go diff --git a/operator/builtin/output/googlecloud/entry.go b/operator/builtin/output/googlecloud/entry.go index 6094fedc9..1a738a637 100644 --- a/operator/builtin/output/googlecloud/entry.go +++ b/operator/builtin/output/googlecloud/entry.go @@ -10,7 +10,6 @@ import ( pstruct "github.com/golang/protobuf/ptypes/struct" "google.golang.org/genproto/googleapis/logging/v2" "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -168,7 +167,7 @@ func (g *GoogleEntryBuilder) setPayload(entry *entry.Entry, logEntry *logging.Lo logEntry.Payload = &logging.LogEntry_TextPayload{TextPayload: string(value)} return nil case map[string]interface{}: - structValue, err := structpb.NewValue(value) + structValue, err := toProto(value) if err != nil { return fmt.Errorf("failed to convert record of type map[string]interface: %w", err) } diff --git a/operator/builtin/output/googlecloud/proto.go b/operator/builtin/output/googlecloud/proto.go new file mode 100644 index 000000000..b9561dab7 --- /dev/null +++ b/operator/builtin/output/googlecloud/proto.go @@ -0,0 +1,116 @@ +package googlecloud + +import ( + "encoding/base64" + "fmt" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/structpb" +) + +// toProto converts a value to a protobuf equivalent +func toProto(v interface{}) (*structpb.Value, error) { + switch v := v.(type) { + case nil: + return structpb.NewNullValue(), nil + case bool: + return structpb.NewBoolValue(v), nil + case int: + return structpb.NewNumberValue(float64(v)), nil + case int8: + return structpb.NewNumberValue(float64(v)), nil + case int16: + return structpb.NewNumberValue(float64(v)), nil + case int32: + return structpb.NewNumberValue(float64(v)), nil + case int64: + return structpb.NewNumberValue(float64(v)), nil + case uint: + return structpb.NewNumberValue(float64(v)), nil + case uint8: + return structpb.NewNumberValue(float64(v)), nil + case uint16: + return structpb.NewNumberValue(float64(v)), nil + case uint32: + return structpb.NewNumberValue(float64(v)), nil + case uint64: + return structpb.NewNumberValue(float64(v)), nil + case float32: + return structpb.NewNumberValue(float64(v)), nil + case float64: + return structpb.NewNumberValue(v), nil + case string: + return toProtoString(v) + case []byte: + s := base64.StdEncoding.EncodeToString(v) + return structpb.NewStringValue(s), nil + case map[string]interface{}: + v2, err := toProtoStruct(v) + if err != nil { + return nil, err + } + return structpb.NewStructValue(v2), nil + case map[string]string: + fields := map[string]*structpb.Value{} + for key, value := range v { + fields[key] = structpb.NewStringValue(value) + } + return structpb.NewStructValue(&structpb.Struct{ + Fields: fields, + }), nil + case []interface{}: + v2, err := toProtoList(v) + if err != nil { + return nil, err + } + return structpb.NewListValue(v2), nil + case []string: + values := []*structpb.Value{} + for _, str := range v { + values = append(values, structpb.NewStringValue(str)) + } + + return structpb.NewListValue(&structpb.ListValue{ + Values: values, + }), nil + default: + return nil, fmt.Errorf("invalid type: %T", v) + } +} + +// toProtoStruct converts a map to a protobuf equivalent +func toProtoStruct(v map[string]interface{}) (*structpb.Struct, error) { + x := &structpb.Struct{Fields: make(map[string]*structpb.Value, len(v))} + for k, v := range v { + if !utf8.ValidString(k) { + return nil, fmt.Errorf("invalid UTF-8 in string: %q", k) + } + var err error + x.Fields[k], err = toProto(v) + if err != nil { + return nil, err + } + } + return x, nil +} + +// toProtoList converts a slice of interface a protobuf equivalent +func toProtoList(v []interface{}) (*structpb.ListValue, error) { + x := &structpb.ListValue{Values: make([]*structpb.Value, len(v))} + for i, v := range v { + var err error + x.Values[i], err = toProto(v) + if err != nil { + return nil, err + } + } + return x, nil +} + +// toProtoString converts a string to a proto string +func toProtoString(v string) (*structpb.Value, error) { + if !utf8.ValidString(v) { + return nil, fmt.Errorf("invalid UTF-8 in string: %q", v) + } + return structpb.NewStringValue(v), nil +} diff --git a/operator/builtin/output/googlecloud/proto_test.go b/operator/builtin/output/googlecloud/proto_test.go new file mode 100644 index 000000000..9aecf4493 --- /dev/null +++ b/operator/builtin/output/googlecloud/proto_test.go @@ -0,0 +1,97 @@ +package googlecloud + +import ( + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" +) + +func TestToProto(t *testing.T) { + testCases := []struct { + name string + value interface{} + expectedValue *structpb.Value + expectedErr string + }{ + { + name: "nil", + value: nil, + expectedValue: structpb.NewNullValue(), + }, + { + name: "numbers", + value: map[string]interface{}{ + "int": int(1), + "int8": int8(1), + "int16": int16(1), + "int32": int32(1), + "int64": int64(1), + "uint": uint(1), + "uint8": uint8(1), + "uint16": uint16(1), + "uint32": uint32(1), + "uint64": uint64(1), + "float32": float32(1), + "float64": float64(1), + }, + expectedValue: structpb.NewStructValue(&structpb.Struct{ + Fields: map[string]*structpb.Value{ + "int": structpb.NewNumberValue(float64(1)), + "int8": structpb.NewNumberValue(float64(1)), + "int16": structpb.NewNumberValue(float64(1)), + "int32": structpb.NewNumberValue(float64(1)), + "int64": structpb.NewNumberValue(float64(1)), + "uint": structpb.NewNumberValue(float64(1)), + "uint8": structpb.NewNumberValue(float64(1)), + "uint16": structpb.NewNumberValue(float64(1)), + "uint32": structpb.NewNumberValue(float64(1)), + "uint64": structpb.NewNumberValue(float64(1)), + "float32": structpb.NewNumberValue(float64(1)), + "float64": structpb.NewNumberValue(float64(1)), + }, + }), + }, + { + name: "map[string]string", + value: map[string]string{ + "test": "value", + }, + expectedValue: structpb.NewStructValue(&structpb.Struct{ + Fields: map[string]*structpb.Value{ + "test": structpb.NewStringValue("value"), + }, + }), + }, + { + name: "interface list", + value: []interface{}{"value", 1}, + expectedValue: structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{ + structpb.NewStringValue("value"), + structpb.NewNumberValue(float64(1)), + }}), + }, + { + name: "string list", + value: []string{"value", "test"}, + expectedValue: structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{ + structpb.NewStringValue("value"), + structpb.NewStringValue("test"), + }}), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + proto, err := toProto(tc.value) + if tc.expectedErr != "" { + require.Error(t, err) + require.Contains(t, err, tc.expectedErr) + return + } + + require.NoError(t, err) + require.Equal(t, tc.expectedValue, proto) + }) + } +}