From 716213caa197fe509495f14211881bb1cd180383 Mon Sep 17 00:00:00 2001 From: Yu Long Date: Wed, 6 Nov 2024 14:15:06 -0800 Subject: [PATCH 1/6] Bump objstore to latest master to get metrics fix --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 565c325300..376b71d5d5 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( github.com/prometheus/prometheus v0.54.2-0.20240920164404-6f0d6038b7f9 github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.9.0 - github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 + github.com/thanos-io/objstore v0.0.0-20241105144332-b598dceacb13 github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect diff --git a/go.sum b/go.sum index 0e75a24782..50102487e0 100644 --- a/go.sum +++ b/go.sum @@ -2245,6 +2245,8 @@ github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1 github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 h1:TUPZ6euAh8I62KrpDnBIg7k2C5HjgXQnVHoUUMacGwM= github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069/go.mod h1:Cba80S8NbVBBdyZKzra7San/jXvpAxArbpFymWzIZhg= +github.com/thanos-io/objstore v0.0.0-20241105144332-b598dceacb13 h1:PQd6xZs18KGoCZJgL9eyYsrRGzzRwYCr4iXuehZm++w= +github.com/thanos-io/objstore v0.0.0-20241105144332-b598dceacb13/go.mod h1:/ZMUxFcp/nT6oYV5WslH9k07NU/+86+aibgZRmMMr/4= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 h1:xPaP58g+3EPohdw4cv+6jv5+LcX6LynhHvQcYwTAMxQ= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= From 84da98cf24078d84b068629415e6e81f32760ea4 Mon Sep 17 00:00:00 2001 From: Yu Long Date: Wed, 6 Nov 2024 16:25:41 -0800 Subject: [PATCH 2/6] Fix api break --- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/receive.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- cmd/thanos/tools_bucket.go | 20 ++++++++++---------- pkg/block/fetcher.go | 2 +- pkg/cache/groupcache.go | 2 +- pkg/replicate/replicator.go | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index ddaf702d85..741fc664ef 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -205,7 +205,7 @@ func runCompact( return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index a5cd5c38ed..556369b0a1 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -84,7 +84,7 @@ func RunDownsample( return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Downsample.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Downsample.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 569ec1e57e..5db1d51631 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -195,7 +195,7 @@ func runReceive( } // The background shipper continuously scans the data directory and uploads // new blocks to object storage service. - bkt, err = client.NewBucket(logger, confContentYaml, comp.String()) + bkt, err = client.NewBucket(logger, confContentYaml, comp.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index ad793d17a4..62b42761a7 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -843,7 +843,7 @@ func runRule( if len(confContentYaml) > 0 { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, component.Rule.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Rule.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 91f7feee54..3df6612259 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -367,7 +367,7 @@ func runSidecar( if uploads { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, component.Sidecar.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Sidecar.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index d626fe55d8..0c5eb780a1 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -309,7 +309,7 @@ func runStore( return err } - bkt, err := client.NewBucket(logger, confContentYaml, conf.component.String()) + bkt, err := client.NewBucket(logger, confContentYaml, conf.component.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 7e711168cf..7875d9c0a0 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -327,7 +327,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -346,7 +346,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path } } else { // nil Prometheus registerer: don't create conflicting metrics. - backupBkt, err = client.NewBucket(logger, backupconfContentYaml, component.Bucket.String()) + backupBkt, err = client.NewBucket(logger, backupconfContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -411,7 +411,7 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -519,7 +519,7 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -629,7 +629,7 @@ func registerBucketWeb(app extkingpin.AppClause, objStoreConfig *extflag.PathOrC return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return errors.Wrap(err, "bucket client") } @@ -826,7 +826,7 @@ func registerBucketCleanup(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Cleanup.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Cleanup.String(), nil) if err != nil { return err } @@ -1083,7 +1083,7 @@ func registerBucketMarkBlock(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Mark.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Mark.String(), nil) if err != nil { return err } @@ -1163,7 +1163,7 @@ func registerBucketRewrite(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Rewrite.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Rewrite.String(), nil) if err != nil { return err } @@ -1371,7 +1371,7 @@ func registerBucketRetention(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Retention.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Retention.String(), nil) if err != nil { return err } @@ -1460,7 +1460,7 @@ func registerBucketUploadBlocks(app extkingpin.AppClause, objStoreConfig *extfla return errors.Wrap(err, "unable to parse objstore config") } - bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String(), nil) if err != nil { return errors.Wrap(err, "unable to create bucket") } diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go index f3204b651f..28fdf17a6a 100644 --- a/pkg/block/fetcher.go +++ b/pkg/block/fetcher.go @@ -252,7 +252,7 @@ func (f *RecursiveLister) GetActiveAndPartialBlockIDs(ctx context.Context, ch ch case ch <- id: } return nil - }, objstore.WithRecursiveIter) + }, objstore.WithRecursiveIter()) return partialBlocks, err } diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index 609959d0cb..d105dcb022 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -208,7 +208,7 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf if err := bucket.Iter(ctx, parsedData.Name, func(s string) error { list = append(list, s) return nil - }, objstore.WithRecursiveIter); err != nil { + }, objstore.WithRecursiveIter()); err != nil { return err } diff --git a/pkg/replicate/replicator.go b/pkg/replicate/replicator.go index 668d64afce..dc9e804f56 100644 --- a/pkg/replicate/replicator.go +++ b/pkg/replicate/replicator.go @@ -115,7 +115,7 @@ func RunReplicate( return errors.New("No supported bucket was configured to replicate from") } - bkt, err := client.NewBucket(logger, fromConfContentYaml, component.Replicate.String()) + bkt, err := client.NewBucket(logger, fromConfContentYaml, component.Replicate.String(), nil) if err != nil { return err } @@ -136,7 +136,7 @@ func RunReplicate( return errors.New("No supported bucket was configured to replicate to") } - toBkt, err := client.NewBucket(logger, toConfContentYaml, component.Replicate.String()) + toBkt, err := client.NewBucket(logger, toConfContentYaml, component.Replicate.String(), nil) if err != nil { return err } From 2ed10af422c39bdb9af2c791e48b80e3397a23b2 Mon Sep 17 00:00:00 2001 From: Yu Long Date: Wed, 6 Nov 2024 16:51:41 -0800 Subject: [PATCH 3/6] Tidy up mod --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 50102487e0..19f12791e2 100644 --- a/go.sum +++ b/go.sum @@ -2243,8 +2243,6 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 h1:TUPZ6euAh8I62KrpDnBIg7k2C5HjgXQnVHoUUMacGwM= -github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069/go.mod h1:Cba80S8NbVBBdyZKzra7San/jXvpAxArbpFymWzIZhg= github.com/thanos-io/objstore v0.0.0-20241105144332-b598dceacb13 h1:PQd6xZs18KGoCZJgL9eyYsrRGzzRwYCr4iXuehZm++w= github.com/thanos-io/objstore v0.0.0-20241105144332-b598dceacb13/go.mod h1:/ZMUxFcp/nT6oYV5WslH9k07NU/+86+aibgZRmMMr/4= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 h1:xPaP58g+3EPohdw4cv+6jv5+LcX6LynhHvQcYwTAMxQ= From c0984d861f44e301a4c97435b871d251b416de48 Mon Sep 17 00:00:00 2001 From: Yu Long Date: Wed, 6 Nov 2024 16:54:31 -0800 Subject: [PATCH 4/6] Fix integration test --- test/e2e/compact_test.go | 6 +++--- test/e2e/query_test.go | 6 +++--- test/e2e/store_gateway_test.go | 16 ++++++++-------- test/e2e/tools_bucket_web_test.go | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go index 54c04fe91a..975edb51c5 100644 --- a/test/e2e/compact_test.go +++ b/test/e2e/compact_test.go @@ -351,7 +351,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) @@ -896,7 +896,7 @@ func TestCompactorDownsampleIgnoresMarked(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bktCfg := e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()) - bkt, err := s3.NewBucketWithConfig(logger, bktCfg, "test") + bkt, err := s3.NewBucketWithConfig(logger, bktCfg, "test", nil) testutil.Ok(t, err) downsampledBase := blockDesc{ @@ -944,7 +944,7 @@ func TestCompactorIssue6775(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) baseBlockDesc := blockDesc{ diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 763e5cd9cb..0351cc9bc8 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -619,7 +619,7 @@ func TestQueryStoreMetrics(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) // Preparing 3 different blocks for the tests. @@ -807,7 +807,7 @@ func TestQueryStoreDedup(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) storeGW := e2ethanos.NewStoreGW( @@ -2035,7 +2035,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) // Add series from different tenants diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 912712f0a8..8977aa46bb 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -127,7 +127,7 @@ metafile_content_ttl: 0s`, memcached.InternalEndpoint("memcached")) testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -453,7 +453,7 @@ func TestStoreGatewayNoCacheFile(t *testing.T) { testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -672,7 +672,7 @@ blocks_iter_ttl: 0s`, memcached.InternalEndpoint("memcached")) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -807,7 +807,7 @@ metafile_content_ttl: 0s` testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -934,7 +934,7 @@ config: testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -1068,7 +1068,7 @@ config: l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -1180,7 +1180,7 @@ func TestStoreGatewayLazyExpandedPostingsEnabled(t *testing.T) { l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -1354,7 +1354,7 @@ func TestStoreGatewayLazyExpandedPostingsPromQLSmithFuzz(t *testing.T) { l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) diff --git a/test/e2e/tools_bucket_web_test.go b/test/e2e/tools_bucket_web_test.go index a72a14a3db..e679d2b924 100644 --- a/test/e2e/tools_bucket_web_test.go +++ b/test/e2e/tools_bucket_web_test.go @@ -156,7 +156,7 @@ func TestToolsBucketWebWithTimeAndRelabelFilter(t *testing.T) { // Create bucket. logger := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "tools") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "tools", nil) testutil.Ok(t, err) // Create share dir for upload. From a0a952ea0e2f05712b161f8861e93ab07c01fc50 Mon Sep 17 00:00:00 2001 From: Yu Long Date: Wed, 6 Nov 2024 17:02:52 -0800 Subject: [PATCH 5/6] revert to db_main --- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/receive.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- cmd/thanos/tools_bucket.go | 20 ++++++++++---------- go.mod | 2 +- go.sum | 4 ++-- pkg/block/fetcher.go | 2 +- pkg/cache/groupcache.go | 2 +- pkg/replicate/replicator.go | 4 ++-- test/e2e/compact_test.go | 6 +++--- test/e2e/query_test.go | 6 +++--- test/e2e/store_gateway_test.go | 16 ++++++++-------- test/e2e/tools_bucket_web_test.go | 2 +- 16 files changed, 38 insertions(+), 38 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index 741fc664ef..ddaf702d85 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -205,7 +205,7 @@ func runCompact( return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.String()) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index 556369b0a1..a5cd5c38ed 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -84,7 +84,7 @@ func RunDownsample( return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Downsample.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Downsample.String()) if err != nil { return err } diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 5db1d51631..569ec1e57e 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -195,7 +195,7 @@ func runReceive( } // The background shipper continuously scans the data directory and uploads // new blocks to object storage service. - bkt, err = client.NewBucket(logger, confContentYaml, comp.String(), nil) + bkt, err = client.NewBucket(logger, confContentYaml, comp.String()) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 62b42761a7..ad793d17a4 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -843,7 +843,7 @@ func runRule( if len(confContentYaml) > 0 { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, component.Rule.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Rule.String()) if err != nil { return err } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 3df6612259..91f7feee54 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -367,7 +367,7 @@ func runSidecar( if uploads { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, component.Sidecar.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Sidecar.String()) if err != nil { return err } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index 0c5eb780a1..d626fe55d8 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -309,7 +309,7 @@ func runStore( return err } - bkt, err := client.NewBucket(logger, confContentYaml, conf.component.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, conf.component.String()) if err != nil { return err } diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 7875d9c0a0..7e711168cf 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -327,7 +327,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) if err != nil { return err } @@ -346,7 +346,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path } } else { // nil Prometheus registerer: don't create conflicting metrics. - backupBkt, err = client.NewBucket(logger, backupconfContentYaml, component.Bucket.String(), nil) + backupBkt, err = client.NewBucket(logger, backupconfContentYaml, component.Bucket.String()) if err != nil { return err } @@ -411,7 +411,7 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) if err != nil { return err } @@ -519,7 +519,7 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) if err != nil { return err } @@ -629,7 +629,7 @@ func registerBucketWeb(app extkingpin.AppClause, objStoreConfig *extflag.PathOrC return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) if err != nil { return errors.Wrap(err, "bucket client") } @@ -826,7 +826,7 @@ func registerBucketCleanup(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Cleanup.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Cleanup.String()) if err != nil { return err } @@ -1083,7 +1083,7 @@ func registerBucketMarkBlock(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Mark.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Mark.String()) if err != nil { return err } @@ -1163,7 +1163,7 @@ func registerBucketRewrite(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Rewrite.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Rewrite.String()) if err != nil { return err } @@ -1371,7 +1371,7 @@ func registerBucketRetention(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Retention.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Retention.String()) if err != nil { return err } @@ -1460,7 +1460,7 @@ func registerBucketUploadBlocks(app extkingpin.AppClause, objStoreConfig *extfla return errors.Wrap(err, "unable to parse objstore config") } - bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String(), nil) + bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String()) if err != nil { return errors.Wrap(err, "unable to create bucket") } diff --git a/go.mod b/go.mod index 376b71d5d5..565c325300 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( github.com/prometheus/prometheus v0.54.2-0.20240920164404-6f0d6038b7f9 github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.9.0 - github.com/thanos-io/objstore v0.0.0-20241105144332-b598dceacb13 + github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect diff --git a/go.sum b/go.sum index 19f12791e2..0e75a24782 100644 --- a/go.sum +++ b/go.sum @@ -2243,8 +2243,8 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-io/objstore v0.0.0-20241105144332-b598dceacb13 h1:PQd6xZs18KGoCZJgL9eyYsrRGzzRwYCr4iXuehZm++w= -github.com/thanos-io/objstore v0.0.0-20241105144332-b598dceacb13/go.mod h1:/ZMUxFcp/nT6oYV5WslH9k07NU/+86+aibgZRmMMr/4= +github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 h1:TUPZ6euAh8I62KrpDnBIg7k2C5HjgXQnVHoUUMacGwM= +github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069/go.mod h1:Cba80S8NbVBBdyZKzra7San/jXvpAxArbpFymWzIZhg= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 h1:xPaP58g+3EPohdw4cv+6jv5+LcX6LynhHvQcYwTAMxQ= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= diff --git a/pkg/block/fetcher.go b/pkg/block/fetcher.go index 28fdf17a6a..f3204b651f 100644 --- a/pkg/block/fetcher.go +++ b/pkg/block/fetcher.go @@ -252,7 +252,7 @@ func (f *RecursiveLister) GetActiveAndPartialBlockIDs(ctx context.Context, ch ch case ch <- id: } return nil - }, objstore.WithRecursiveIter()) + }, objstore.WithRecursiveIter) return partialBlocks, err } diff --git a/pkg/cache/groupcache.go b/pkg/cache/groupcache.go index d105dcb022..609959d0cb 100644 --- a/pkg/cache/groupcache.go +++ b/pkg/cache/groupcache.go @@ -208,7 +208,7 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf if err := bucket.Iter(ctx, parsedData.Name, func(s string) error { list = append(list, s) return nil - }, objstore.WithRecursiveIter()); err != nil { + }, objstore.WithRecursiveIter); err != nil { return err } diff --git a/pkg/replicate/replicator.go b/pkg/replicate/replicator.go index dc9e804f56..668d64afce 100644 --- a/pkg/replicate/replicator.go +++ b/pkg/replicate/replicator.go @@ -115,7 +115,7 @@ func RunReplicate( return errors.New("No supported bucket was configured to replicate from") } - bkt, err := client.NewBucket(logger, fromConfContentYaml, component.Replicate.String(), nil) + bkt, err := client.NewBucket(logger, fromConfContentYaml, component.Replicate.String()) if err != nil { return err } @@ -136,7 +136,7 @@ func RunReplicate( return errors.New("No supported bucket was configured to replicate to") } - toBkt, err := client.NewBucket(logger, toConfContentYaml, component.Replicate.String(), nil) + toBkt, err := client.NewBucket(logger, toConfContentYaml, component.Replicate.String()) if err != nil { return err } diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go index 975edb51c5..54c04fe91a 100644 --- a/test/e2e/compact_test.go +++ b/test/e2e/compact_test.go @@ -351,7 +351,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") testutil.Ok(t, err) ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) @@ -896,7 +896,7 @@ func TestCompactorDownsampleIgnoresMarked(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bktCfg := e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()) - bkt, err := s3.NewBucketWithConfig(logger, bktCfg, "test", nil) + bkt, err := s3.NewBucketWithConfig(logger, bktCfg, "test") testutil.Ok(t, err) downsampledBase := blockDesc{ @@ -944,7 +944,7 @@ func TestCompactorIssue6775(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") testutil.Ok(t, err) baseBlockDesc := blockDesc{ diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 0351cc9bc8..763e5cd9cb 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -619,7 +619,7 @@ func TestQueryStoreMetrics(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") testutil.Ok(t, err) // Preparing 3 different blocks for the tests. @@ -807,7 +807,7 @@ func TestQueryStoreDedup(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") testutil.Ok(t, err) storeGW := e2ethanos.NewStoreGW( @@ -2035,7 +2035,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") testutil.Ok(t, err) // Add series from different tenants diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 8977aa46bb..912712f0a8 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -127,7 +127,7 @@ metafile_content_ttl: 0s`, memcached.InternalEndpoint("memcached")) testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -453,7 +453,7 @@ func TestStoreGatewayNoCacheFile(t *testing.T) { testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -672,7 +672,7 @@ blocks_iter_ttl: 0s`, memcached.InternalEndpoint("memcached")) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -807,7 +807,7 @@ metafile_content_ttl: 0s` testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -934,7 +934,7 @@ config: testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -1068,7 +1068,7 @@ config: l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -1180,7 +1180,7 @@ func TestStoreGatewayLazyExpandedPostingsEnabled(t *testing.T) { l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -1354,7 +1354,7 @@ func TestStoreGatewayLazyExpandedPostingsPromQLSmithFuzz(t *testing.T) { l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) diff --git a/test/e2e/tools_bucket_web_test.go b/test/e2e/tools_bucket_web_test.go index e679d2b924..a72a14a3db 100644 --- a/test/e2e/tools_bucket_web_test.go +++ b/test/e2e/tools_bucket_web_test.go @@ -156,7 +156,7 @@ func TestToolsBucketWebWithTimeAndRelabelFilter(t *testing.T) { // Create bucket. logger := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "tools", nil) + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "tools") testutil.Ok(t, err) // Create share dir for upload. From 0b2bff92a6235e9990b6ce144e11372c88f0f549 Mon Sep 17 00:00:00 2001 From: Yu Long Date: Wed, 6 Nov 2024 17:11:17 -0800 Subject: [PATCH 6/6] Fix API breakage --- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/receive.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- cmd/thanos/tools_bucket.go | 20 ++++++++++---------- go.mod | 2 +- go.sum | 4 ++-- pkg/replicate/replicator.go | 4 ++-- test/e2e/compact_test.go | 6 +++--- test/e2e/query_test.go | 6 +++--- test/e2e/store_gateway_test.go | 16 ++++++++-------- test/e2e/tools_bucket_web_test.go | 2 +- 14 files changed, 36 insertions(+), 36 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index ddaf702d85..741fc664ef 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -205,7 +205,7 @@ func runCompact( return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index a5cd5c38ed..556369b0a1 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -84,7 +84,7 @@ func RunDownsample( return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Downsample.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Downsample.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 569ec1e57e..5db1d51631 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -195,7 +195,7 @@ func runReceive( } // The background shipper continuously scans the data directory and uploads // new blocks to object storage service. - bkt, err = client.NewBucket(logger, confContentYaml, comp.String()) + bkt, err = client.NewBucket(logger, confContentYaml, comp.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index ad793d17a4..62b42761a7 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -843,7 +843,7 @@ func runRule( if len(confContentYaml) > 0 { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, component.Rule.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Rule.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 91f7feee54..3df6612259 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -367,7 +367,7 @@ func runSidecar( if uploads { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, component.Sidecar.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Sidecar.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index d626fe55d8..0c5eb780a1 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -309,7 +309,7 @@ func runStore( return err } - bkt, err := client.NewBucket(logger, confContentYaml, conf.component.String()) + bkt, err := client.NewBucket(logger, confContentYaml, conf.component.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 7e711168cf..7875d9c0a0 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -327,7 +327,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -346,7 +346,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path } } else { // nil Prometheus registerer: don't create conflicting metrics. - backupBkt, err = client.NewBucket(logger, backupconfContentYaml, component.Bucket.String()) + backupBkt, err = client.NewBucket(logger, backupconfContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -411,7 +411,7 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -519,7 +519,7 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -629,7 +629,7 @@ func registerBucketWeb(app extkingpin.AppClause, objStoreConfig *extflag.PathOrC return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return errors.Wrap(err, "bucket client") } @@ -826,7 +826,7 @@ func registerBucketCleanup(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Cleanup.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Cleanup.String(), nil) if err != nil { return err } @@ -1083,7 +1083,7 @@ func registerBucketMarkBlock(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Mark.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Mark.String(), nil) if err != nil { return err } @@ -1163,7 +1163,7 @@ func registerBucketRewrite(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Rewrite.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Rewrite.String(), nil) if err != nil { return err } @@ -1371,7 +1371,7 @@ func registerBucketRetention(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Retention.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Retention.String(), nil) if err != nil { return err } @@ -1460,7 +1460,7 @@ func registerBucketUploadBlocks(app extkingpin.AppClause, objStoreConfig *extfla return errors.Wrap(err, "unable to parse objstore config") } - bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String(), nil) if err != nil { return errors.Wrap(err, "unable to create bucket") } diff --git a/go.mod b/go.mod index 565c325300..5438621f51 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( github.com/prometheus/prometheus v0.54.2-0.20240920164404-6f0d6038b7f9 github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.9.0 - github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 + github.com/thanos-io/objstore v0.0.0-20241028150459-cfdd0e50390d github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect diff --git a/go.sum b/go.sum index 0e75a24782..660b4e37d4 100644 --- a/go.sum +++ b/go.sum @@ -2243,8 +2243,8 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 h1:TUPZ6euAh8I62KrpDnBIg7k2C5HjgXQnVHoUUMacGwM= -github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069/go.mod h1:Cba80S8NbVBBdyZKzra7San/jXvpAxArbpFymWzIZhg= +github.com/thanos-io/objstore v0.0.0-20241028150459-cfdd0e50390d h1:k+SLTP1mjNqXxsCiq4UYeKCe07le0ieffyuHm/YfmH8= +github.com/thanos-io/objstore v0.0.0-20241028150459-cfdd0e50390d/go.mod h1:/ZMUxFcp/nT6oYV5WslH9k07NU/+86+aibgZRmMMr/4= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 h1:xPaP58g+3EPohdw4cv+6jv5+LcX6LynhHvQcYwTAMxQ= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= diff --git a/pkg/replicate/replicator.go b/pkg/replicate/replicator.go index 668d64afce..dc9e804f56 100644 --- a/pkg/replicate/replicator.go +++ b/pkg/replicate/replicator.go @@ -115,7 +115,7 @@ func RunReplicate( return errors.New("No supported bucket was configured to replicate from") } - bkt, err := client.NewBucket(logger, fromConfContentYaml, component.Replicate.String()) + bkt, err := client.NewBucket(logger, fromConfContentYaml, component.Replicate.String(), nil) if err != nil { return err } @@ -136,7 +136,7 @@ func RunReplicate( return errors.New("No supported bucket was configured to replicate to") } - toBkt, err := client.NewBucket(logger, toConfContentYaml, component.Replicate.String()) + toBkt, err := client.NewBucket(logger, toConfContentYaml, component.Replicate.String(), nil) if err != nil { return err } diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go index 54c04fe91a..975edb51c5 100644 --- a/test/e2e/compact_test.go +++ b/test/e2e/compact_test.go @@ -351,7 +351,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) @@ -896,7 +896,7 @@ func TestCompactorDownsampleIgnoresMarked(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bktCfg := e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()) - bkt, err := s3.NewBucketWithConfig(logger, bktCfg, "test") + bkt, err := s3.NewBucketWithConfig(logger, bktCfg, "test", nil) testutil.Ok(t, err) downsampledBase := blockDesc{ @@ -944,7 +944,7 @@ func TestCompactorIssue6775(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) baseBlockDesc := blockDesc{ diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 763e5cd9cb..0351cc9bc8 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -619,7 +619,7 @@ func TestQueryStoreMetrics(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) // Preparing 3 different blocks for the tests. @@ -807,7 +807,7 @@ func TestQueryStoreDedup(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) storeGW := e2ethanos.NewStoreGW( @@ -2035,7 +2035,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) // Add series from different tenants diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 912712f0a8..8977aa46bb 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -127,7 +127,7 @@ metafile_content_ttl: 0s`, memcached.InternalEndpoint("memcached")) testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -453,7 +453,7 @@ func TestStoreGatewayNoCacheFile(t *testing.T) { testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -672,7 +672,7 @@ blocks_iter_ttl: 0s`, memcached.InternalEndpoint("memcached")) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -807,7 +807,7 @@ metafile_content_ttl: 0s` testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -934,7 +934,7 @@ config: testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -1068,7 +1068,7 @@ config: l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -1180,7 +1180,7 @@ func TestStoreGatewayLazyExpandedPostingsEnabled(t *testing.T) { l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -1354,7 +1354,7 @@ func TestStoreGatewayLazyExpandedPostingsPromQLSmithFuzz(t *testing.T) { l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) diff --git a/test/e2e/tools_bucket_web_test.go b/test/e2e/tools_bucket_web_test.go index a72a14a3db..e679d2b924 100644 --- a/test/e2e/tools_bucket_web_test.go +++ b/test/e2e/tools_bucket_web_test.go @@ -156,7 +156,7 @@ func TestToolsBucketWebWithTimeAndRelabelFilter(t *testing.T) { // Create bucket. logger := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "tools") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "tools", nil) testutil.Ok(t, err) // Create share dir for upload.