From e5603e272e6a892dbc9d19621285c35c5bcdefd2 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Thu, 30 Nov 2023 11:53:54 +0800 Subject: [PATCH 01/12] sync: fix temporary name when create a file (#4215) --- pkg/object/file.go | 18 +++++++++++------- pkg/object/filesystem_test.go | 6 ++++++ pkg/object/hdfs.go | 14 +++++++++++--- pkg/object/sftp.go | 15 ++++++++++++--- 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/pkg/object/file.go b/pkg/object/file.go index 1df31b4c4100..fbf57a5e33f8 100644 --- a/pkg/object/file.go +++ b/pkg/object/file.go @@ -132,14 +132,23 @@ func (d *filestore) Get(key string, off, limit int64) (io.ReadCloser, error) { return f, nil } -func (d *filestore) Put(key string, in io.Reader) error { +func (d *filestore) Put(key string, in io.Reader) (err error) { p := d.path(key) if strings.HasSuffix(key, dirSuffix) || key == "" && strings.HasSuffix(d.root, dirSuffix) { return os.MkdirAll(p, os.FileMode(0755)) } - tmp := filepath.Join(filepath.Dir(p), "."+filepath.Base(p)+".tmp"+strconv.Itoa(rand.Int())) + name := filepath.Base(p) + if len(name) > 200 { + name = name[:200] + } + tmp := filepath.Join(filepath.Dir(p), "."+name+".tmp"+strconv.Itoa(rand.Int())) + defer func() { + if err != nil { + _ = os.Remove(tmp) + } + }() f, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil && os.IsNotExist(err) { if err := os.MkdirAll(filepath.Dir(p), os.FileMode(0755)); err != nil { @@ -150,11 +159,6 @@ func (d *filestore) Put(key string, in io.Reader) error { if err != nil { return err } - defer func() { - if err != nil { - _ = os.Remove(tmp) - } - }() if TryCFR { _, err = io.Copy(f, in) diff --git a/pkg/object/filesystem_test.go b/pkg/object/filesystem_test.go index de37e78f4522..ccbda7b061d0 100644 --- a/pkg/object/filesystem_test.go +++ b/pkg/object/filesystem_test.go @@ -195,4 +195,10 @@ func testFileSystem(t *testing.T, s ObjectStorage) { t.Fatalf("size of target(file) should be 10") } } + + // put a file with very long name + longName := strings.Repeat("a", 255) + if err := s.Put("dir/"+longName, bytes.NewReader([]byte{0})); err != nil { + t.Fatalf("PUT a file with long name `%s` failed: %q", longName, err) + } } diff --git a/pkg/object/hdfs.go b/pkg/object/hdfs.go index faeccec433b6..2e3fac2e1695 100644 --- a/pkg/object/hdfs.go +++ b/pkg/object/hdfs.go @@ -114,14 +114,22 @@ func (h *hdfsclient) Get(key string, off, limit int64) (io.ReadCloser, error) { const abcException = "org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException" -func (h *hdfsclient) Put(key string, in io.Reader) error { +func (h *hdfsclient) Put(key string, in io.Reader) (err error) { path := h.path(key) if strings.HasSuffix(path, dirSuffix) { return h.c.MkdirAll(path, os.FileMode(0755)) } - tmp := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s.tmp.%d", filepath.Base(path), rand.Int())) + name := filepath.Base(path) + if len(name) > 200 { + name = name[:200] + } + tmp := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s.tmp.%d", name, rand.Int())) + defer func() { + if err != nil { + _ = h.c.Remove(tmp) + } + }() f, err := h.c.CreateFile(tmp, 3, 128<<20, 0755) - defer func() { _ = h.c.Remove(tmp) }() if err != nil { if pe, ok := err.(*os.PathError); ok && pe.Err == os.ErrNotExist { _ = h.c.MkdirAll(filepath.Dir(path), 0755) diff --git a/pkg/object/sftp.go b/pkg/object/sftp.go index d4793dfa0042..2808bb2bad8a 100644 --- a/pkg/object/sftp.go +++ b/pkg/object/sftp.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "io/ioutil" + "math/rand" "net" "net/url" "os" @@ -202,7 +203,7 @@ func (f *sftpStore) Get(key string, off, limit int64) (io.ReadCloser, error) { return ff, err } -func (f *sftpStore) Put(key string, in io.Reader) error { +func (f *sftpStore) Put(key string, in io.Reader) (err error) { c, err := f.getSftpConnection() if err != nil { return err @@ -216,7 +217,16 @@ func (f *sftpStore) Put(key string, in io.Reader) error { if err := c.sftpClient.MkdirAll(filepath.Dir(p)); err != nil { return err } - tmp := filepath.Join(filepath.Dir(p), "."+filepath.Base(p)+".tmp") + name := filepath.Base(p) + if len(name) > 200 { + name = name[:200] + } + tmp := filepath.Join(filepath.Dir(p), fmt.Sprintf(".%s.tmp.%d", name, rand.Int())) + defer func() { + if err != nil { + _ = c.sftpClient.Remove(tmp) + } + }() if runtime.GOOS == "windows" { tmp = strings.Replace(tmp, "\\", "/", -1) } @@ -225,7 +235,6 @@ func (f *sftpStore) Put(key string, in io.Reader) error { if err != nil { return err } - defer func() { _ = c.sftpClient.Remove(tmp) }() buf := bufPool.Get().(*[]byte) defer bufPool.Put(buf) _, err = io.CopyBuffer(ff, in, *buf) From 8ec2941108912fa8150768531366a12d61a55f62 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Thu, 30 Nov 2023 12:26:28 +0800 Subject: [PATCH 02/12] object: fix thread-safety in list of ceph (#4216) --- pkg/object/ceph.go | 75 ++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/pkg/object/ceph.go b/pkg/object/ceph.go index ce8d40879e20..33e74432efb1 100644 --- a/pkg/object/ceph.go +++ b/pkg/object/ceph.go @@ -198,46 +198,49 @@ func (c *ceph) Head(key string) (Object, error) { } func (c *ceph) ListAll(prefix, marker string) (<-chan Object, error) { - var objs = make(chan Object, 1000) - err := c.do(func(ctx *rados.IOContext) error { - iter, err := ctx.Iter() - if err != nil { - close(objs) - return err - } - defer iter.Close() + ctx, err := c.newContext() + if err != nil { + return nil, err + } + iter, err := ctx.Iter() + if err != nil { + ctx.Destroy() + return nil, err + } + defer iter.Close() - // FIXME: this will be really slow for many objects - keys := make([]string, 0, 1000) - for iter.Next() { - key := iter.Value() - if key <= marker || !strings.HasPrefix(key, prefix) { - continue - } - keys = append(keys, key) + // FIXME: this will be really slow for many objects + keys := make([]string, 0, 1000) + for iter.Next() { + key := iter.Value() + if key <= marker || !strings.HasPrefix(key, prefix) { + continue } - // the keys are not ordered, sort them first - sort.Strings(keys) - // TODO: parallel - go func() { - defer close(objs) - for _, key := range keys { - st, err := ctx.Stat(key) - if err != nil { - if errors.Is(err, rados.ErrNotFound) { - logger.Warnf("Skip non-existent key: %s", key) - continue - } - objs <- nil - logger.Errorf("Stat key %s: %s", key, err) - return + keys = append(keys, key) + } + // the keys are not ordered, sort them first + sort.Strings(keys) + + var objs = make(chan Object, 1000) + // TODO: parallel + go func() { + defer close(objs) + defer ctx.Destroy() + for _, key := range keys { + st, err := ctx.Stat(key) + if err != nil { + if errors.Is(err, rados.ErrNotFound) { + logger.Debugf("Skip non-existent key: %s", key) + continue } - objs <- &obj{key, int64(st.Size), st.ModTime, strings.HasSuffix(key, "/")} + objs <- nil + logger.Errorf("Stat key %s: %s", key, err) + return } - }() - return nil - }) - return objs, err + objs <- &obj{key, int64(st.Size), st.ModTime, strings.HasSuffix(key, "/")} + } + }() + return objs, nil } func newCeph(endpoint, cluster, user, token string) (ObjectStorage, error) { From adaa733122d33fe9f87e51aa1f531169464e4971 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Wed, 13 Dec 2023 11:56:21 +0800 Subject: [PATCH 03/12] object/cos: fix multipart upload with long key (#4258) --- go.mod | 2 +- go.sum | 8 ++++---- pkg/object/cos.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5f3e28ae4107..d8c878375318 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/sirupsen/logrus v1.9.0 github.com/smartystreets/goconvey v1.7.2 github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 - github.com/tencentyun/cos-go-sdk-v5 v0.7.34 + github.com/tencentyun/cos-go-sdk-v5 v0.7.45 github.com/tikv/client-go/v2 v2.0.2 github.com/upyun/go-sdk/v3 v3.0.2 github.com/urfave/cli/v2 v2.4.0 diff --git a/go.sum b/go.sum index 06f5b4a1a8f7..2af54ab2edf0 100644 --- a/go.sum +++ b/go.sum @@ -924,10 +924,10 @@ github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9H github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= -github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= -github.com/tencentyun/cos-go-sdk-v5 v0.7.34 h1:xm+Pg+6m486y4eugRI7/E4WasbVmpY1hp9QBSRErgp8= -github.com/tencentyun/cos-go-sdk-v5 v0.7.34/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= +github.com/tencentyun/cos-go-sdk-v5 v0.7.45 h1:5/ZGOv846tP6+2X7w//8QjLgH2KcUK+HciFbfjWquFU= +github.com/tencentyun/cos-go-sdk-v5 v0.7.45/go.mod h1:DH9US8nB+AJXqwu/AMOrCFN1COv3dpytXuJWHgdg7kE= github.com/tidwall/gjson v1.6.7/go.mod h1:zeFuBCIqD4sN/gmqBzZ4j7Jd6UcA2Fc56x7QFsv+8fI= github.com/tidwall/gjson v1.9.3 h1:hqzS9wAHMO+KVBBkLxYdkEeeFHuqr95GfClRLKlgK0E= github.com/tidwall/gjson v1.9.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= diff --git a/pkg/object/cos.go b/pkg/object/cos.go index b383767b2b57..6c09376dae97 100644 --- a/pkg/object/cos.go +++ b/pkg/object/cos.go @@ -174,7 +174,7 @@ func (c *COS) AbortUpload(key string, uploadID string) { func (c *COS) CompleteUpload(key string, uploadID string, parts []*Part) error { var cosParts []cos.Object for i := range parts { - cosParts = append(cosParts, cos.Object{Key: key, ETag: parts[i].ETag, PartNumber: parts[i].Num}) + cosParts = append(cosParts, cos.Object{ETag: parts[i].ETag, PartNumber: parts[i].Num}) } _, _, err := c.c.Object.CompleteMultipartUpload(ctx, key, uploadID, &cos.CompleteMultipartUploadOptions{Parts: cosParts}) return err From c4d52701022225b9fc7b0597f3a1f22a575a8f4d Mon Sep 17 00:00:00 2001 From: Shan Huang <1171399+mtnk2005@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:40:02 +0800 Subject: [PATCH 04/12] meta/redis: use TxPipelined() instead of Pipelined() (#4264) --- pkg/meta/redis.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index cd367908b054..a1c8be5fb7ba 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -2101,7 +2101,7 @@ func (m *redisMeta) CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, vals = append(vals, val) } - _, err = tx.Pipelined(ctx, func(pipe redis.Pipeliner) error { + _, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error { coff := offIn / ChunkSize * ChunkSize for _, sv := range vals { // Add a zero chunk for hole @@ -2261,7 +2261,7 @@ func (m *redisMeta) cleanupZeroRef(key string) { if v != 0 { return syscall.EINVAL } - _, err = tx.Pipelined(ctx, func(p redis.Pipeliner) error { + _, err = tx.TxPipelined(ctx, func(p redis.Pipeliner) error { p.HDel(ctx, m.sliceRefs(), key) return nil }) @@ -2456,7 +2456,7 @@ func (r *redisMeta) doCleanupDelayedSlices(edge int64) (int, error) { if len(ss) == 0 { return fmt.Errorf("invalid value for delSlices %s: %v", key, buf) } - _, e = tx.Pipelined(ctx, func(pipe redis.Pipeliner) error { + _, e = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error { for _, s := range ss { rs = append(rs, pipe.HIncrBy(ctx, r.sliceRefs(), r.sliceKey(s.Id, s.Size), -1)) } @@ -2562,7 +2562,7 @@ func (m *redisMeta) compactChunk(inode Ino, indx uint32, force bool) { } } - _, err = tx.Pipelined(ctx, func(pipe redis.Pipeliner) error { + _, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error { pipe.LTrim(ctx, key, int64(len(vals)), -1) pipe.LPush(ctx, key, marshalSlice(pos, id, size, 0, size)) for i := skipped; i > 0; i-- { From 17595498dc24ed4b80df2b1b73ced10c1fe9756b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Dec 2023 10:59:14 +0800 Subject: [PATCH 05/12] Build(deps): bump golang.org/x/crypto from 0.14.0 to 0.17.0 (#4287) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index d8c878375318..0893b48fb59e 100644 --- a/go.mod +++ b/go.mod @@ -65,12 +65,12 @@ require ( github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a go.etcd.io/etcd v3.3.27+incompatible go.etcd.io/etcd/client/v3 v3.5.2 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.17.0 golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 - golang.org/x/sys v0.13.0 - golang.org/x/term v0.13.0 - golang.org/x/text v0.13.0 + golang.org/x/sys v0.15.0 + golang.org/x/term v0.15.0 + golang.org/x/text v0.14.0 google.golang.org/api v0.70.0 gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216 xorm.io/xorm v1.0.7 diff --git a/go.sum b/go.sum index 2af54ab2edf0..2ff375bc25e5 100644 --- a/go.sum +++ b/go.sum @@ -1056,8 +1056,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1275,12 +1275,12 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1291,8 +1291,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From ac2934505600560c024d2c0973400bdbe3bef86c Mon Sep 17 00:00:00 2001 From: tangyoupeng Date: Fri, 29 Dec 2023 11:54:16 +0800 Subject: [PATCH 06/12] sdk/java: fix guid update (#4299) --- sdk/java/libjfs/guid.go | 52 +++++++++++++++++++++++++---------------- sdk/java/libjfs/main.go | 5 ++++ 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/sdk/java/libjfs/guid.go b/sdk/java/libjfs/guid.go index 386f634e9a18..db251624a671 100644 --- a/sdk/java/libjfs/guid.go +++ b/sdk/java/libjfs/guid.go @@ -80,8 +80,8 @@ func (m *mapping) lookupUser(name string) uint32 { } else { id = m.genGuid(name) } - m.usernames[name] = id - m.userIDs[id] = name + logger.Debugf("update user to %s:%d by lookup user", name, id) + m.updateUser(name, id) return id } @@ -102,8 +102,8 @@ func (m *mapping) lookupGroup(name string) uint32 { id_, _ := strconv.ParseUint(g.Gid, 10, 32) id = uint32(id_) } - m.groups[name] = id - m.groupIDs[id] = name + logger.Debugf("update group to %s:%d by lookup group", name, id) + m.updateGroup(name, id) return 0 } @@ -124,8 +124,8 @@ func (m *mapping) lookupUserID(id uint32) string { if len(name) > 49 { name = name[:49] } - m.usernames[name] = id - m.userIDs[id] = name + logger.Debugf("update user to %s:%d by lookup user id", name, id) + m.updateUser(name, id) return name } @@ -146,8 +146,8 @@ func (m *mapping) lookupGroupID(id uint32) string { if len(name) > 49 { name = name[:49] } - m.groups[name] = id - m.groupIDs[id] = name + logger.Debugf("update group to %s:%d by lookup group id", name, id) + m.updateGroup(name, id) return name } @@ -156,19 +156,31 @@ func (m *mapping) update(uids []pwent, gids []pwent, local bool) { defer m.Unlock() m.local = local for _, u := range uids { - oldId := m.usernames[u.name] - oldName := m.userIDs[u.id] - delete(m.userIDs, oldId) - delete(m.usernames, oldName) - m.usernames[u.name] = u.id - m.userIDs[u.id] = u.name + m.updateUser(u.name, u.id) } for _, g := range gids { - oldId := m.groups[g.name] - oldName := m.groupIDs[g.id] - delete(m.groupIDs, oldId) - delete(m.groups, oldName) - m.groups[g.name] = g.id - m.groupIDs[g.id] = g.name + m.updateGroup(g.name, g.id) } + logger.Debugf("users:\n%+v", m.usernames) + logger.Debugf("userids:\n%+v", m.userIDs) + logger.Debugf("groups:\n%+v", m.groups) + logger.Debugf("gorupids:\n%+v", m.groupIDs) +} + +func (m *mapping) updateUser(name string, id uint32) { + oldId := m.usernames[name] + oldName := m.userIDs[id] + delete(m.userIDs, oldId) + delete(m.usernames, oldName) + m.usernames[name] = id + m.userIDs[id] = name +} + +func (m *mapping) updateGroup(name string, id uint32) { + oldId := m.groups[name] + oldName := m.groupIDs[id] + delete(m.groupIDs, oldId) + delete(m.groups, oldName) + m.groups[name] = id + m.groupIDs[id] = name } diff --git a/sdk/java/libjfs/main.go b/sdk/java/libjfs/main.go index bc4574360eb5..4f43d119ac96 100644 --- a/sdk/java/libjfs/main.go +++ b/sdk/java/libjfs/main.go @@ -571,6 +571,11 @@ func jfs_update_uid_grouping(h uintptr, uidstr *C.char, grouping *C.char) { } } logger.Debugf("Update groups of %s to %s", w.user, strings.Join(groups, ",")) + var buffer bytes.Buffer + for _, g := range gids { + buffer.WriteString(fmt.Sprintf("\t%v:%v\n", g.name, g.id)) + } + logger.Debugf("Update gids mapping\n %s", buffer.String()) } w.m.update(uids, gids, false) From a9c631a0fc6f1cbe6d3bdbaef1b24d24a6cfce01 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Mon, 8 Jan 2024 17:38:17 +0800 Subject: [PATCH 07/12] meta: trigger compaction more aggresively when there are too many slices (#4309) --- pkg/meta/redis.go | 2 +- pkg/meta/sql.go | 3 ++- pkg/meta/tkv.go | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index a1c8be5fb7ba..1a7d30da60ae 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -2028,7 +2028,7 @@ func (m *redisMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice return nil }) if err == nil { - needCompact = rpush.Val()%100 == 99 + needCompact = rpush.Val()%100 == 99 || rpush.Val() > 350 } return err }, m.inodeKey(inode)) diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index c819c98b1dcd..1fcb74df496e 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2048,7 +2048,8 @@ func (m *dbMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Sl } _, err = s.Cols("length", "mtime", "ctime").Update(&n, &node{Inode: inode}) if err == nil { - needCompact = (len(ck.Slices)/sliceBytes)%100 == 99 + ns := len(ck.Slices) / sliceBytes // number of slices + needCompact = ns%100 == 99 || ns > 350 } return err }, inode) diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index f3a30c507d94..dfa1ff1792a0 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -1716,7 +1716,8 @@ func (m *kvMeta) Write(ctx Context, inode Ino, indx uint32, off uint32, slice Sl attr.Ctimensec = uint32(now.Nanosecond()) val := tx.append(m.chunkKey(inode, indx), marshalSlice(off, slice.Id, slice.Size, slice.Off, slice.Len)) tx.set(m.inodeKey(inode), m.marshal(&attr)) - needCompact = (len(val)/sliceBytes)%100 == 99 + ns := len(val) / sliceBytes // number of slices + needCompact = ns%100 == 99 || ns > 350 return nil }, inode) if err == nil { From 53f47312bcad9a8f8ec6d6597cb180ba117cc7a7 Mon Sep 17 00:00:00 2001 From: zhijian Date: Wed, 10 Jan 2024 20:45:52 +0800 Subject: [PATCH 08/12] object/obs: don't decode the object key in list result (#4331) --- pkg/object/obs.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/object/obs.go b/pkg/object/obs.go index 40c0e7020cb7..e43c1860e842 100644 --- a/pkg/object/obs.go +++ b/pkg/object/obs.go @@ -31,8 +31,6 @@ import ( "os" "strings" - "github.com/pkg/errors" - "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" "github.com/juicedata/juicefs/pkg/utils" "golang.org/x/net/http/httpproxy" @@ -182,12 +180,9 @@ func (s *obsClient) List(prefix, marker string, limit int64) ([]Object, error) { n := len(resp.Contents) objs := make([]Object, n) for i := 0; i < n; i++ { + // Obs SDK listObjects method already decodes the object key. o := resp.Contents[i] - key, err := obs.UrlDecode(o.Key) - if err != nil { - return nil, errors.WithMessagef(err, "failed to decode key %s", o.Key) - } - objs[i] = &obj{key, o.Size, o.LastModified, strings.HasSuffix(key, "/")} + objs[i] = &obj{o.Key, o.Size, o.LastModified, strings.HasSuffix(o.Key, "/")} } return objs, nil } From b1c1e3c04a245ae57ddf04114458a9e442ba3ccf Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Wed, 10 Jan 2024 20:47:38 +0800 Subject: [PATCH 09/12] meta: fix the issue that truncated slices may never be compacted (#4320) --- pkg/meta/base.go | 2 ++ pkg/meta/redis.go | 9 ++++++++- pkg/meta/slice.go | 4 +++- pkg/meta/sql.go | 10 ++++++++++ pkg/meta/tkv.go | 13 ++++++++++--- pkg/meta/tkv_etcd.go | 1 + 6 files changed, 34 insertions(+), 5 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index d356051a933e..280a7f8211db 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -40,6 +40,8 @@ const ( nlocks = 1024 ) +var maxCompactSlices = 1000 + type engine interface { // Get the value of counter name. getCounter(name string) (int64, error) diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index 1a7d30da60ae..34b5d5a0520b 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -2505,7 +2505,7 @@ func (m *redisMeta) compactChunk(inode Ino, indx uint32, force bool) { } var ctx = Background - vals, err := m.rdb.LRange(ctx, m.chunkKey(inode, indx), 0, 1000).Result() + vals, err := m.rdb.LRange(ctx, m.chunkKey(inode, indx), 0, int64(maxCompactSlices)).Result() if err != nil { return } @@ -2516,11 +2516,18 @@ func (m *redisMeta) compactChunk(inode Ino, indx uint32, force bool) { return } skipped := skipSome(ss) + var last *slice + if skipped > 0 { + last = ss[skipped-1] + } ss = ss[skipped:] pos, size, slices := compactChunk(ss) if len(ss) < 2 || size == 0 { return } + if last != nil && last.pos+last.len > pos { + panic(fmt.Sprintf("invalid compaction: last skipped slice %+v, pos %d", last, pos)) + } var id uint64 st := m.NewSlice(ctx, &id) diff --git a/pkg/meta/slice.go b/pkg/meta/slice.go index 16936bf533bb..601e8f1d7662 100644 --- a/pkg/meta/slice.go +++ b/pkg/meta/slice.go @@ -157,7 +157,9 @@ func buildSlice(ss []*slice) []Slice { func compactChunk(ss []*slice) (uint32, uint32, []Slice) { var chunk = buildSlice(ss) var pos uint32 - if len(chunk) > 0 && chunk[0].Id == 0 { + if len(chunk) == 1 && chunk[0].Id == 0 { + chunk[0].Len = 1 + } else if len(chunk) > 1 && chunk[0].Id == 0 { pos = chunk[0].Len chunk = chunk[1:] } diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index 1fcb74df496e..b56f4d037495 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2384,6 +2384,9 @@ func (m *dbMeta) compactChunk(inode Ino, indx uint32, force bool) { if err != nil { return } + if len(c.Slices) > sliceBytes*maxCompactSlices { + c.Slices = c.Slices[:sliceBytes*maxCompactSlices] + } ss := readSliceBuf(c.Slices) if ss == nil { @@ -2391,11 +2394,18 @@ func (m *dbMeta) compactChunk(inode Ino, indx uint32, force bool) { return } skipped := skipSome(ss) + var last *slice + if skipped > 0 { + last = ss[skipped-1] + } ss = ss[skipped:] pos, size, slices := compactChunk(ss) if len(ss) < 2 || size == 0 { return } + if last != nil && last.pos+last.len > pos { + panic(fmt.Sprintf("invalid compaction: last skipped slice %+v, pos %d", last, pos)) + } var id uint64 st := m.NewSlice(Background, &id) diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index dfa1ff1792a0..5c5d78624289 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -2026,21 +2026,28 @@ func (m *kvMeta) compactChunk(inode Ino, indx uint32, force bool) { if err != nil { return } - - if len(buf) > sliceBytes*100 { - buf = buf[:sliceBytes*100] + if len(buf) > sliceBytes*maxCompactSlices { + buf = buf[:sliceBytes*maxCompactSlices] } + ss := readSliceBuf(buf) if ss == nil { logger.Errorf("Corrupt value for inode %d chunk indx %d", inode, indx) return } skipped := skipSome(ss) + var last *slice + if skipped > 0 { + last = ss[skipped-1] + } ss = ss[skipped:] pos, size, slices := compactChunk(ss) if len(ss) < 2 || size == 0 { return } + if last != nil && last.pos+last.len > pos { + panic(fmt.Sprintf("invalid compaction: last skipped slice %+v, pos %d", last, pos)) + } var id uint64 st := m.NewSlice(Background, &id) diff --git a/pkg/meta/tkv_etcd.go b/pkg/meta/tkv_etcd.go index bdbc7e7c1ac3..62b8e9f1dfcd 100644 --- a/pkg/meta/tkv_etcd.go +++ b/pkg/meta/tkv_etcd.go @@ -388,6 +388,7 @@ func newEtcdClient(addr string) (tkvClient, error) { if err != nil { return nil, err } + maxCompactSlices = 100 var prefix string = u.Path + "\xFD" return withPrefix(&etcdClient{c, etcd.NewKV(c)}, []byte(prefix)), nil } From b895fe7eef4216d4baaf3cc21329836e63039934 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Tue, 16 Jan 2024 20:44:51 +0800 Subject: [PATCH 10/12] meta: fix protection check for compaction (#4349) --- pkg/meta/base_test.go | 47 +++++++++++++++++++++++++++++++++++++++++++ pkg/meta/redis.go | 8 ++++---- pkg/meta/slice.go | 7 ++++--- pkg/meta/sql.go | 8 ++++---- pkg/meta/tkv.go | 8 ++++---- 5 files changed, 63 insertions(+), 15 deletions(-) diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go index 15e2e4f36734..0af654e2a068 100644 --- a/pkg/meta/base_test.go +++ b/pkg/meta/base_test.go @@ -935,6 +935,53 @@ func testCompaction(t *testing.T, m Meta, trash bool) { if deletes < 200 { t.Fatalf("deleted slices %d is less than 200", deletes) } + + // truncate to 0 + if st := m.Truncate(ctx, inode, 0, 0, attr, false); st != 0 { + t.Fatalf("truncate file: %s", st) + } + if c, ok := m.(compactor); ok { + c.compactChunk(inode, 0, true) + } + if st := m.Read(ctx, inode, 0, &slices); st != 0 { + t.Fatalf("read 0: %s", st) + } + if len(slices) != 1 || slices[0].Len != 1 { + t.Fatalf("inode %d should be compacted, but have %d slices, size %d", inode, len(slices), slices[0].Len) + } + + if st := m.Truncate(ctx, inode, 0, 64<<10, attr, false); st != 0 { + t.Fatalf("truncate file: %s", st) + } + m.NewSlice(ctx, &sliceId) + _ = m.Write(ctx, inode, 0, uint32(1<<20), Slice{Id: sliceId, Size: 2 << 20, Len: 2 << 20}, time.Now()) + if c, ok := m.(compactor); ok { + c.compactChunk(inode, 0, true) + } + if st := m.Read(ctx, inode, 0, &slices); st != 0 { + t.Fatalf("read 0: %s", st) + } + if len(slices) != 2 || slices[0].Id != 0 || slices[1].Len != 2<<20 { + t.Fatalf("inode %d should be compacted, but have %d slices, id %d size %d", + inode, len(slices), slices[0].Id, slices[1].Len) + } + + m.NewSlice(ctx, &sliceId) + _ = m.Write(ctx, inode, 0, uint32(512<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 64 << 10}, time.Now()) + m.NewSlice(ctx, &sliceId) + _ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: sliceId, Size: 1 << 20, Len: 64 << 10}, time.Now()) + m.NewSlice(ctx, &sliceId) + _ = m.Write(ctx, inode, 0, uint32(128<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 128 << 10}, time.Now()) + _ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: 0, Size: 1 << 20, Len: 1 << 20}, time.Now()) + if c, ok := m.(compactor); ok { + c.compactChunk(inode, 0, true) + } + if st := m.Read(ctx, inode, 0, &slices); st != 0 { + t.Fatalf("read 0: %s", st) + } + if len(slices) != 1 || slices[0].Len != 3<<20 { + t.Fatalf("inode %d should be compacted, but have %d slices, size %d", inode, len(slices), slices[0].Len) + } } func testConcurrentWrite(t *testing.T, m Meta) { diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index 34b5d5a0520b..7e76a1197b95 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -2516,17 +2516,17 @@ func (m *redisMeta) compactChunk(inode Ino, indx uint32, force bool) { return } skipped := skipSome(ss) - var last *slice + var first, last *slice if skipped > 0 { - last = ss[skipped-1] + first, last = ss[0], ss[skipped-1] } ss = ss[skipped:] pos, size, slices := compactChunk(ss) if len(ss) < 2 || size == 0 { return } - if last != nil && last.pos+last.len > pos { - panic(fmt.Sprintf("invalid compaction: last skipped slice %+v, pos %d", last, pos)) + if first != nil && last != nil && pos+size > first.pos && last.pos+last.len > pos { + panic(fmt.Sprintf("invalid compaction: skipped slices [%+v, %+v], pos %d, size %d", *first, *last, pos, size)) } var id uint64 diff --git a/pkg/meta/slice.go b/pkg/meta/slice.go index 601e8f1d7662..d036954dba80 100644 --- a/pkg/meta/slice.go +++ b/pkg/meta/slice.go @@ -157,11 +157,12 @@ func buildSlice(ss []*slice) []Slice { func compactChunk(ss []*slice) (uint32, uint32, []Slice) { var chunk = buildSlice(ss) var pos uint32 + for len(chunk) > 1 && chunk[0].Id == 0 { + pos += chunk[0].Len + chunk = chunk[1:] + } if len(chunk) == 1 && chunk[0].Id == 0 { chunk[0].Len = 1 - } else if len(chunk) > 1 && chunk[0].Id == 0 { - pos = chunk[0].Len - chunk = chunk[1:] } var size uint32 for _, c := range chunk { diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index b56f4d037495..59a663e741e5 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2394,17 +2394,17 @@ func (m *dbMeta) compactChunk(inode Ino, indx uint32, force bool) { return } skipped := skipSome(ss) - var last *slice + var first, last *slice if skipped > 0 { - last = ss[skipped-1] + first, last = ss[0], ss[skipped-1] } ss = ss[skipped:] pos, size, slices := compactChunk(ss) if len(ss) < 2 || size == 0 { return } - if last != nil && last.pos+last.len > pos { - panic(fmt.Sprintf("invalid compaction: last skipped slice %+v, pos %d", last, pos)) + if first != nil && last != nil && pos+size > first.pos && last.pos+last.len > pos { + panic(fmt.Sprintf("invalid compaction: skipped slices [%+v, %+v], pos %d, size %d", *first, *last, pos, size)) } var id uint64 diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index 5c5d78624289..56a2afb3eb1c 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -2036,17 +2036,17 @@ func (m *kvMeta) compactChunk(inode Ino, indx uint32, force bool) { return } skipped := skipSome(ss) - var last *slice + var first, last *slice if skipped > 0 { - last = ss[skipped-1] + first, last = ss[0], ss[skipped-1] } ss = ss[skipped:] pos, size, slices := compactChunk(ss) if len(ss) < 2 || size == 0 { return } - if last != nil && last.pos+last.len > pos { - panic(fmt.Sprintf("invalid compaction: last skipped slice %+v, pos %d", last, pos)) + if first != nil && last != nil && pos+size > first.pos && last.pos+last.len > pos { + panic(fmt.Sprintf("invalid compaction: skipped slices [%+v, %+v], pos %d, size %d", *first, *last, pos, size)) } var id uint64 From 20e3950ec62cc7af280e82d57e01e45ce375f857 Mon Sep 17 00:00:00 2001 From: zhijian Date: Wed, 31 Jan 2024 16:18:12 +0800 Subject: [PATCH 11/12] cmd/sync: fix the src password is not removed (#4390) --- cmd/main.go | 18 +++++++++++------- cmd/sync.go | 3 +-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index f0f4d395eac1..6aa790657e48 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -284,15 +284,19 @@ func setup(c *cli.Context, n int) { } } -func removePassword(uri string) { +func removePassword(uris ...string) { args := make([]string, len(os.Args)) copy(args, os.Args) - uri2 := utils.RemovePassword(uri) - if uri2 != uri { - for i, a := range os.Args { - if a == uri { - args[i] = uri2 - break + var idx int + for _, uri := range uris { + uri2 := utils.RemovePassword(uri) + if uri2 != uri { + for i := idx; i < len(os.Args); i++ { + if os.Args[i] == uri { + args[i] = uri2 + idx = i + 1 + break + } } } } diff --git a/cmd/sync.go b/cmd/sync.go index dc384a21f66b..a90993fad603 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -315,8 +315,7 @@ func doSync(c *cli.Context) error { // Windows support `\` and `/` as its separator, Unix only use `/` srcURL := c.Args().Get(0) dstURL := c.Args().Get(1) - removePassword(srcURL) - removePassword(dstURL) + removePassword(srcURL, dstURL) if runtime.GOOS == "windows" { if !strings.Contains(srcURL, "://") { srcURL = strings.Replace(srcURL, "\\", "/", -1) From d7c287767c4f7385b1a93d396dee459e0daab41e Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Sat, 3 Feb 2024 18:24:12 +0800 Subject: [PATCH 12/12] CI: fix unit tests for backports --- .github/scripts/prepare_db.sh | 3 +++ .github/scripts/setup-hdfs.sh | 5 +++-- pkg/meta/base_test.go | 14 +++++++------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/scripts/prepare_db.sh b/.github/scripts/prepare_db.sh index 80c0c98ab407..4ad08a20807a 100755 --- a/.github/scripts/prepare_db.sh +++ b/.github/scripts/prepare_db.sh @@ -1,4 +1,7 @@ #!/bin/bash -e + +set -x + source .github/scripts/start_meta_engine.sh [ -z "$TEST" ] && echo "TEST is not set" && exit 1 diff --git a/.github/scripts/setup-hdfs.sh b/.github/scripts/setup-hdfs.sh index 3c3ea96decf4..dc46a50f9392 100755 --- a/.github/scripts/setup-hdfs.sh +++ b/.github/scripts/setup-hdfs.sh @@ -15,12 +15,13 @@ # limitations under the License. # +set -x set -e sudo apt-get update sudo apt-get install openjdk-8-jdk -y -HADOOP_VERSION="2.10.1" -wget -q https://dlcdn.apache.org/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz +HADOOP_VERSION="2.10.2" +wget -q https://dlcdn.apache.org/hadoop/common/hadoop-2.10.2/hadoop-2.10.2.tar.gz mkdir ~/app tar -zxf hadoop-${HADOOP_VERSION}.tar.gz -C ~/app diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go index 0af654e2a068..f2a6bbaaf697 100644 --- a/pkg/meta/base_test.go +++ b/pkg/meta/base_test.go @@ -937,7 +937,7 @@ func testCompaction(t *testing.T, m Meta, trash bool) { } // truncate to 0 - if st := m.Truncate(ctx, inode, 0, 0, attr, false); st != 0 { + if st := m.Truncate(ctx, inode, 0, 0, attr); st != 0 { t.Fatalf("truncate file: %s", st) } if c, ok := m.(compactor); ok { @@ -950,11 +950,11 @@ func testCompaction(t *testing.T, m Meta, trash bool) { t.Fatalf("inode %d should be compacted, but have %d slices, size %d", inode, len(slices), slices[0].Len) } - if st := m.Truncate(ctx, inode, 0, 64<<10, attr, false); st != 0 { + if st := m.Truncate(ctx, inode, 0, 64<<10, attr); st != 0 { t.Fatalf("truncate file: %s", st) } m.NewSlice(ctx, &sliceId) - _ = m.Write(ctx, inode, 0, uint32(1<<20), Slice{Id: sliceId, Size: 2 << 20, Len: 2 << 20}, time.Now()) + _ = m.Write(ctx, inode, 0, uint32(1<<20), Slice{Id: sliceId, Size: 2 << 20, Len: 2 << 20}) if c, ok := m.(compactor); ok { c.compactChunk(inode, 0, true) } @@ -967,12 +967,12 @@ func testCompaction(t *testing.T, m Meta, trash bool) { } m.NewSlice(ctx, &sliceId) - _ = m.Write(ctx, inode, 0, uint32(512<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 64 << 10}, time.Now()) + _ = m.Write(ctx, inode, 0, uint32(512<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 64 << 10}) m.NewSlice(ctx, &sliceId) - _ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: sliceId, Size: 1 << 20, Len: 64 << 10}, time.Now()) + _ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: sliceId, Size: 1 << 20, Len: 64 << 10}) m.NewSlice(ctx, &sliceId) - _ = m.Write(ctx, inode, 0, uint32(128<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 128 << 10}, time.Now()) - _ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: 0, Size: 1 << 20, Len: 1 << 20}, time.Now()) + _ = m.Write(ctx, inode, 0, uint32(128<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 128 << 10}) + _ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: 0, Size: 1 << 20, Len: 1 << 20}) if c, ok := m.(compactor); ok { c.compactChunk(inode, 0, true) }