Skip to content

Commit

Permalink
Runtime: otel fix for db close leaks (#3690)
Browse files Browse the repository at this point in the history 
* otel fix for db close leaks

* comment when to revert

* fix test case - wal doesn't exist

* otel pkgs upgrade and code changes

* deployment not found fix

(cherry picked from commit c24f3b4)
  • Loading branch information
@k-anshul @esevastyanov
k-anshul authored and esevastyanov committed Dec 19, 2023
1 parent 0471d72 commit f265e64
Show file tree
Hide file tree
Showing 14 changed files with 128 additions and 185 deletions.
3 changes: 1 addition & 2 deletions admin/client/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@ func New(adminHost, bearerToken, userAgent string) (*Client, error) {

opts := []grpc.DialOption{
grpc.WithUserAgent(userAgent),
grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()),
grpc.WithStreamInterceptor(otelgrpc.StreamClientInterceptor()),
grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
}

if uri.Scheme == "http" {
Expand Down
2 changes: 1 addition & 1 deletion admin/database/postgres/postgres.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"github.com/jackc/pgconn"
"github.com/jmoiron/sqlx"
"github.com/rilldata/rill/admin/database"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"

// Load postgres driver
_ "github.com/jackc/pgx/v4/stdlib"
Expand Down
2 changes: 1 addition & 1 deletion admin/server/auth/middleware.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
"github.com/grpc-ecosystem/go-grpc-middleware/util/metautils"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
Expand Down
4 changes: 2 additions & 2 deletions admin/server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/rilldata/rill/runtime/pkg/ratelimit"
runtimeauth "github.com/rilldata/rill/runtime/server/auth"
"github.com/rs/cors"
"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.uber.org/zap"
"google.golang.org/grpc"
Expand Down Expand Up @@ -121,7 +122,6 @@ func (s *Server) ServeGRPC(ctx context.Context) error {
server := grpc.NewServer(
grpc.ChainStreamInterceptor(
middleware.TimeoutStreamServerInterceptor(timeoutSelector),
observability.TracingStreamServerInterceptor(),
observability.LoggingStreamServerInterceptor(s.logger),
errorMappingStreamServerInterceptor(),
grpc_auth.StreamServerInterceptor(checkUserAgent),
Expand All @@ -131,14 +131,14 @@ func (s *Server) ServeGRPC(ctx context.Context) error {
),
grpc.ChainUnaryInterceptor(
middleware.TimeoutUnaryServerInterceptor(timeoutSelector),
observability.TracingUnaryServerInterceptor(),
observability.LoggingUnaryServerInterceptor(s.logger),
errorMappingUnaryServerInterceptor(),
grpc_auth.UnaryServerInterceptor(checkUserAgent),
grpc_validator.UnaryServerInterceptor(),
s.authenticator.UnaryServerInterceptor(),
grpc_auth.UnaryServerInterceptor(s.checkRateLimit),
),
grpc.StatsHandler(otelgrpc.NewServerHandler()),
)

adminv1.RegisterAdminServiceServer(server, s)
Expand Down
4 changes: 4 additions & 0 deletions cli/cmd/project/refresh.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ func RefreshCmd(ch *cmdutil.Helper) *cobra.Command {
return err
}

if resp.ProdDeployment == nil {
return fmt.Errorf("no production deployment found for project %q", project)
}

_, err = client.TriggerRefreshSources(ctx, &adminv1.TriggerRefreshSourcesRequest{DeploymentId: resp.ProdDeployment.Id, Sources: source})
if err != nil {
return fmt.Errorf("failed to trigger refresh: %w", err)
Expand Down
73 changes: 37 additions & 36 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/Masterminds/sprig/v3 v3.2.3
github.com/MicahParks/keyfunc v1.9.0
github.com/NYTimes/gziphandler v1.1.1
github.com/XSAM/otelsql v0.23.0
github.com/XSAM/otelsql v0.26.0
github.com/alicebob/miniredis v2.5.0+incompatible
github.com/apache/arrow/go/v14 v14.0.1
github.com/apache/calcite-avatica-go/v5 v5.2.0
Expand All @@ -39,7 +39,7 @@ require (
github.com/gorilla/securecookie v1.1.1
github.com/gorilla/sessions v1.2.1
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1
github.com/hashicorp/go-version v1.6.0
github.com/hashicorp/golang-lru v0.6.0
github.com/jackc/pgconn v1.14.0
Expand All @@ -52,7 +52,7 @@ require (
github.com/marcboeker/go-duckdb v1.5.4
github.com/mazznoer/csscolorparser v0.1.3
github.com/mitchellh/mapstructure v1.5.0
github.com/prometheus/client_golang v1.15.1
github.com/prometheus/client_golang v1.17.0
github.com/redis/go-redis/v9 v9.0.2
github.com/rs/cors v1.9.0
github.com/snowflakedb/gosnowflake v1.7.0
Expand All @@ -61,28 +61,28 @@ require (
github.com/stretchr/testify v1.8.4
github.com/testcontainers/testcontainers-go v0.19.0
github.com/xuri/excelize/v2 v2.7.1
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0
go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0
go.opentelemetry.io/otel v1.16.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.39.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0
go.opentelemetry.io/otel/exporters/prometheus v0.39.0
go.opentelemetry.io/otel/metric v1.16.0
go.opentelemetry.io/otel/sdk v1.16.0
go.opentelemetry.io/otel/sdk/metric v0.39.0
go.opentelemetry.io/otel/trace v1.16.0
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1
go.opentelemetry.io/contrib/instrumentation/runtime v0.46.1
go.opentelemetry.io/otel v1.21.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0
go.opentelemetry.io/otel/exporters/prometheus v0.44.0
go.opentelemetry.io/otel/metric v1.21.0
go.opentelemetry.io/otel/sdk v1.21.0
go.opentelemetry.io/otel/sdk/metric v1.21.0
go.opentelemetry.io/otel/trace v1.21.0
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.25.0
go.uber.org/zap/exp v0.1.0
gocloud.dev v0.34.0
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
golang.org/x/oauth2 v0.13.0
golang.org/x/sync v0.4.0
golang.org/x/sys v0.14.0
golang.org/x/sys v0.15.0
google.golang.org/api v0.149.0
google.golang.org/grpc v1.59.0
google.golang.org/grpc v1.60.0
google.golang.org/protobuf v1.31.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/square/go-jose.v2 v2.6.0
Expand All @@ -109,17 +109,18 @@ require (
github.com/jackc/pgtype v1.12.0 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
github.com/mtibben/percent v0.2.1 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 // indirect
)

require (
cloud.google.com/go v0.110.8 // indirect
cloud.google.com/go v0.110.10 // indirect
cloud.google.com/go/bigquery v1.57.1
cloud.google.com/go/compute v1.23.1 // indirect
cloud.google.com/go/compute v1.23.3 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.3 // indirect
cloud.google.com/go/iam v1.1.5 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
Expand Down Expand Up @@ -163,13 +164,13 @@ require (
github.com/docker/go-units v0.5.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.7.0
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.4.1 // indirect
github.com/go-jose/go-jose/v3 v3.0.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
Expand Down Expand Up @@ -217,7 +218,6 @@ require (
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
Expand All @@ -235,9 +235,9 @@ require (
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.10.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.45.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/richardlehane/mscfb v1.0.4 // indirect
github.com/richardlehane/msoleps v1.0.3 // indirect
Expand All @@ -254,19 +254,17 @@ require (
github.com/yuin/gopher-lua v1.1.0 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
golang.org/x/crypto v0.15.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
golang.org/x/crypto v0.16.0 // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/net v0.18.0 // indirect
golang.org/x/term v0.14.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.14.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0
google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
lukechampine.com/uint128 v1.3.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
Expand All @@ -290,3 +288,6 @@ replace github.com/marcboeker/go-duckdb v1.5.4 => github.com/rilldata/go-duckdb
replace github.com/snowflakedb/gosnowflake v1.7.0 => github.com/esevastyanov/gosnowflake v0.0.0-20231129090721-012eca4e3448

exclude modernc.org/sqlite v1.18.1

// revert to original SDK once this PR is merged and released : https://github.com/XSAM/otelsql/pull/199
replace github.com/XSAM/otelsql v0.26.0 => github.com/rilldata/otelsql v0.0.0-20231213102114-31132d178f10
Loading

0 comments on commit f265e64

Please sign in to comment.