fix: INT64 numbers are rounded for arrow.Array objects during json ma…

…rshal process (#128) * fix: INT64 numbers are rounded for arrow.Array objects during json marshal process --------- Co-authored-by: Bhargav Dodla <[email protected]>
ExpediaGroup · Aug 17, 2024 · 4fd7b29 · 4fd7b29
1 parent 049157e
commit 4fd7b29
Show file tree

Hide file tree

Showing 18 changed files with 182 additions and 602 deletions.
diff --git a/go.mod b/go.mod
@@ -1,22 +1,20 @@
 module github.com/feast-dev/feast
 
-go 1.19
+go 1.21
 
-replace github.com/go-python/gopy v0.4.4 => github.com/feast-dev/gopy v0.4.1-0.20220714211711-252048177d85
+toolchain go1.22.1
 
 require (
-	github.com/apache/arrow/go/v8 v8.0.0
 	github.com/ghodss/yaml v1.0.0
-	// github.com/go-redis/redis/v8 v8.11.5
-	github.com/golang/protobuf v1.5.3
-	github.com/google/uuid v1.3.0
+	github.com/golang/protobuf v1.5.4
+	github.com/google/uuid v1.6.0
 	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/pkg/errors v0.9.1
 	github.com/redis/go-redis/v9 v9.5.1
 	github.com/spaolacci/murmur3 v1.1.0
-	github.com/stretchr/testify v1.8.4
-	google.golang.org/grpc v1.54.0
-	google.golang.org/protobuf v1.31.0
+	github.com/stretchr/testify v1.9.0
+	google.golang.org/grpc v1.63.2
+	google.golang.org/protobuf v1.34.2
 	gopkg.in/DataDog/dd-trace-go.v1 v1.54.0
 )
 
@@ -41,36 +39,40 @@ require (
 	inet.af/netaddr v0.0.0-20220811202034-502d2d690317 // indirect
 )
 
-require github.com/rs/zerolog v1.21.0
+require (
+	github.com/apache/arrow/go/v17 v17.0.0
+	github.com/rs/zerolog v1.21.0
+)
 
 require (
 	github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
-	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/apache/thrift v0.16.0 // indirect
+	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/apache/thrift v0.20.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/flatbuffers v2.0.8+incompatible // indirect
+	github.com/google/flatbuffers v24.3.25+incompatible // indirect
 	github.com/klauspost/asmfmt v1.3.2 // indirect
-	github.com/klauspost/compress v1.16.3 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
-	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
+	golang.org/x/mod v0.18.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/tools v0.22.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
diff --git a/go/embedded/online_features.go b/go/embedded/online_features.go
@@ -3,7 +3,6 @@ package embedded
 import (
 	"context"
 	"fmt"
-	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 	"log"
 	"net"
 	"os"
@@ -12,10 +11,12 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/apache/arrow/go/v8/arrow"
-	"github.com/apache/arrow/go/v8/arrow/array"
-	"github.com/apache/arrow/go/v8/arrow/cdata"
-	"github.com/apache/arrow/go/v8/arrow/memory"
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/arrow/array"
+	"github.com/apache/arrow/go/v17/arrow/cdata"
+	"github.com/apache/arrow/go/v17/arrow/memory"
 	"google.golang.org/grpc"
 
 	"github.com/feast-dev/feast/go/internal/feast"

diff --git a/go/internal/feast/featurestore.go b/go/internal/feast/featurestore.go
@@ -7,7 +7,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/apache/arrow/go/v8/arrow/memory"
+	"github.com/apache/arrow/go/v17/arrow/memory"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 
 	"github.com/feast-dev/feast/go/internal/feast/model"

diff --git a/go/internal/feast/onlineserving/serving.go b/go/internal/feast/onlineserving/serving.go
@@ -7,8 +7,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/apache/arrow/go/v8/arrow"
-	"github.com/apache/arrow/go/v8/arrow/memory"
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/arrow/memory"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/timestamppb"

diff --git a/go/internal/feast/server/grpc_server_test.go b/go/internal/feast/server/grpc_server_test.go
@@ -15,10 +15,10 @@ import (
 
 	"github.com/feast-dev/feast/go/internal/feast/registry"
 
-	"github.com/apache/arrow/go/v8/arrow/array"
-	"github.com/apache/arrow/go/v8/arrow/memory"
-	"github.com/apache/arrow/go/v8/parquet/file"
-	"github.com/apache/arrow/go/v8/parquet/pqarrow"
+	"github.com/apache/arrow/go/v17/arrow/array"
+	"github.com/apache/arrow/go/v17/arrow/memory"
+	"github.com/apache/arrow/go/v17/parquet/file"
+	"github.com/apache/arrow/go/v17/parquet/pqarrow"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/test/bufconn"

diff --git a/go/internal/feast/server/http_server_test.go b/go/internal/feast/server/http_server_test.go
@@ -1,8 +1,13 @@
 package server
 
 import (
-	"github.com/stretchr/testify/assert"
+	"encoding/json"
 	"testing"
+
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/arrow/array"
+	"github.com/apache/arrow/go/v17/arrow/memory"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestUnmarshalJSON(t *testing.T) {
@@ -38,3 +43,36 @@ func TestUnmarshalJSON(t *testing.T) {
 	assert.Nil(t, u.UnmarshalJSON([]byte("[[true, false, true], [false, true, false]]")))
 	assert.Equal(t, [][]bool{{true, false, true}, {false, true, false}}, u.boolListVal)
 }
+func TestMarshalInt32JSON(t *testing.T) {
+	var arrowArray arrow.Array
+	memoryPool := memory.NewGoAllocator()
+	builder := array.NewInt32Builder(memoryPool)
+	defer builder.Release()
+	builder.AppendValues([]int32{1, 2, 3, 4}, nil)
+	arrowArray = builder.NewArray()
+	defer arrowArray.Release()
+	expectedJSON := `[1,2,3,4]`
+
+	jsonData, err := json.Marshal(arrowArray)
+	assert.NoError(t, err, "Error marshaling Arrow array")
+
+	assert.Equal(t, expectedJSON, string(jsonData), "JSON output does not match expected")
+	assert.IsType(t, &array.Int32{}, arrowArray, "arrowArray is not of type *array.Int32")
+}
+
+func TestMarshalInt64JSON(t *testing.T) {
+	var arrowArray arrow.Array
+	memoryPool := memory.NewGoAllocator()
+	builder := array.NewInt64Builder(memoryPool)
+	defer builder.Release()
+	builder.AppendValues([]int64{-9223372036854775808, 9223372036854775807}, nil)
+	arrowArray = builder.NewArray()
+	defer arrowArray.Release()
+	expectedJSON := `[-9223372036854775808,9223372036854775807]`
+
+	jsonData, err := json.Marshal(arrowArray)
+	assert.NoError(t, err, "Error marshaling Arrow array")
+
+	assert.Equal(t, expectedJSON, string(jsonData), "JSON output does not match expected")
+	assert.IsType(t, &array.Int64{}, arrowArray, "arrowArray is not of type *array.Int64")
+}
diff --git a/go/internal/feast/server/logging/filelogsink.go b/go/internal/feast/server/logging/filelogsink.go
@@ -8,12 +8,12 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/apache/arrow/go/v8/arrow"
+	"github.com/apache/arrow/go/v17/arrow"
 	"github.com/google/uuid"
 
-	"github.com/apache/arrow/go/v8/arrow/array"
-	"github.com/apache/arrow/go/v8/parquet"
-	"github.com/apache/arrow/go/v8/parquet/pqarrow"
+	"github.com/apache/arrow/go/v17/arrow/array"
+	"github.com/apache/arrow/go/v17/parquet"
+	"github.com/apache/arrow/go/v17/parquet/pqarrow"
 )
 
 type FileLogSink struct {

diff --git a/go/internal/feast/server/logging/logger.go b/go/internal/feast/server/logging/logger.go
@@ -8,7 +8,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/apache/arrow/go/v8/arrow"
+	"github.com/apache/arrow/go/v17/arrow"
 	"github.com/pkg/errors"
 	"google.golang.org/protobuf/types/known/timestamppb"
 

diff --git a/go/internal/feast/server/logging/logger_test.go b/go/internal/feast/server/logging/logger_test.go
@@ -7,11 +7,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/apache/arrow/go/v8/arrow"
-	"github.com/apache/arrow/go/v8/arrow/array"
-	"github.com/apache/arrow/go/v8/arrow/memory"
-	"github.com/apache/arrow/go/v8/parquet/file"
-	"github.com/apache/arrow/go/v8/parquet/pqarrow"
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/arrow/array"
+	"github.com/apache/arrow/go/v17/arrow/memory"
+	"github.com/apache/arrow/go/v17/parquet/file"
+	"github.com/apache/arrow/go/v17/parquet/pqarrow"
 	"github.com/stretchr/testify/require"
 
 	"github.com/feast-dev/feast/go/protos/feast/types"

diff --git a/go/internal/feast/server/logging/memorybuffer.go b/go/internal/feast/server/logging/memorybuffer.go
@@ -3,9 +3,9 @@ package logging
 import (
 	"fmt"
 
-	"github.com/apache/arrow/go/v8/arrow"
-	"github.com/apache/arrow/go/v8/arrow/array"
-	"github.com/apache/arrow/go/v8/arrow/memory"
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/arrow/array"
+	"github.com/apache/arrow/go/v17/arrow/memory"
 
 	"github.com/feast-dev/feast/go/protos/feast/types"
 	gotypes "github.com/feast-dev/feast/go/types"

diff --git a/go/internal/feast/server/logging/memorybuffer_test.go b/go/internal/feast/server/logging/memorybuffer_test.go
@@ -5,9 +5,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/apache/arrow/go/v8/arrow"
-	"github.com/apache/arrow/go/v8/arrow/array"
-	"github.com/apache/arrow/go/v8/arrow/memory"
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/arrow/array"
+	"github.com/apache/arrow/go/v17/arrow/memory"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/protobuf/types/known/timestamppb"
 

diff --git a/go/internal/feast/server/logging/offlinestoresink.go b/go/internal/feast/server/logging/offlinestoresink.go
@@ -8,10 +8,10 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/apache/arrow/go/v8/arrow"
-	"github.com/apache/arrow/go/v8/arrow/array"
-	"github.com/apache/arrow/go/v8/parquet"
-	"github.com/apache/arrow/go/v8/parquet/pqarrow"
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/arrow/array"
+	"github.com/apache/arrow/go/v17/parquet"
+	"github.com/apache/arrow/go/v17/parquet/pqarrow"
 	"github.com/google/uuid"
 )
 

diff --git a/go/internal/feast/transformation/transformation.go b/go/internal/feast/transformation/transformation.go
@@ -6,8 +6,8 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/apache/arrow/go/v8/arrow"
-	"github.com/apache/arrow/go/v8/arrow/memory"
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/arrow/memory"
 	"github.com/rs/zerolog/log"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 

diff --git a/go/internal/feast/transformation/transformation_service.go b/go/internal/feast/transformation/transformation_service.go
@@ -6,14 +6,15 @@ import (
 	"fmt"
 	"strings"
 
+	"io"
+
 	"github.com/feast-dev/feast/go/internal/feast/registry"
 	"google.golang.org/protobuf/types/known/timestamppb"
-	"io"
 
-	"github.com/apache/arrow/go/v8/arrow"
-	"github.com/apache/arrow/go/v8/arrow/array"
-	"github.com/apache/arrow/go/v8/arrow/ipc"
-	"github.com/apache/arrow/go/v8/arrow/memory"
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/arrow/array"
+	"github.com/apache/arrow/go/v17/arrow/ipc"
+	"github.com/apache/arrow/go/v17/arrow/memory"
 	"github.com/feast-dev/feast/go/internal/feast/model"
 	"github.com/feast-dev/feast/go/internal/feast/onlineserving"
 	"github.com/feast-dev/feast/go/protos/feast/serving"

diff --git a/go/internal/test/go_integration_test_utils.go b/go/internal/test/go_integration_test_utils.go
@@ -5,20 +5,20 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/apache/arrow/go/v8/arrow/memory"
+	"github.com/apache/arrow/go/v17/arrow/memory"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
-	"github.com/apache/arrow/go/v8/arrow"
-	"github.com/apache/arrow/go/v8/parquet/file"
-	"github.com/apache/arrow/go/v8/parquet/pqarrow"
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/parquet/file"
+	"github.com/apache/arrow/go/v17/parquet/pqarrow"
 
 	"os"
 	"os/exec"
 	"path/filepath"
 	"time"
 
-	"github.com/apache/arrow/go/v8/arrow/array"
+	"github.com/apache/arrow/go/v17/arrow/array"
 
 	"github.com/feast-dev/feast/go/internal/feast/model"
 	"github.com/feast-dev/feast/go/protos/feast/types"

diff --git a/go/types/typeconversion.go b/go/types/typeconversion.go
@@ -3,9 +3,9 @@ package types
 import (
 	"fmt"
 
-	"github.com/apache/arrow/go/v8/arrow"
-	"github.com/apache/arrow/go/v8/arrow/array"
-	"github.com/apache/arrow/go/v8/arrow/memory"
+	"github.com/apache/arrow/go/v17/arrow"
+	"github.com/apache/arrow/go/v17/arrow/array"
+	"github.com/apache/arrow/go/v17/arrow/memory"
 
 	"github.com/feast-dev/feast/go/protos/feast/types"
 )

diff --git a/go/types/typeconversion_test.go b/go/types/typeconversion_test.go
@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/apache/arrow/go/v8/arrow/memory"
+	"github.com/apache/arrow/go/v17/arrow/memory"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/protobuf/proto"
 
@@ -47,8 +47,8 @@ var (
 			{Val: &types.Value_Int32ListVal{&types.Int32List{Val: []int32{3, 4, 5}}}},
 		},
 		{
-			{Val: &types.Value_Int64ListVal{&types.Int64List{Val: []int64{0, 1, 2}}}},
-			{Val: &types.Value_Int64ListVal{&types.Int64List{Val: []int64{3, 4, 5}}}},
+			{Val: &types.Value_Int64ListVal{&types.Int64List{Val: []int64{0, 1, 2, 553248634761893728}}}},
+			{Val: &types.Value_Int64ListVal{&types.Int64List{Val: []int64{3, 4, 5, 553248634761893729}}}},
 		},
 		{
 			{Val: &types.Value_FloatListVal{&types.FloatList{Val: []float32{0.5, 1.5, 2}}}},