diff --git a/cmd/fsck.go b/cmd/fsck.go index 53eb5a97d198..abb3390ac5df 100644 --- a/cmd/fsck.go +++ b/cmd/fsck.go @@ -51,172 +51,203 @@ $ juicefs fsck redis://localhost --path /d1/d2 --repair # recursively check $ juicefs fsck redis://localhost --path /d1/d2 --recursive`, Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "check", + Value: "both", + Usage: `check "meta", "data" or "both"`, + }, &cli.StringFlag{ Name: "path", + Value: "/", Usage: "absolute path within JuiceFS to check", }, - &cli.BoolFlag{ - Name: "repair", - Usage: "repair specified path if it's broken", - }, &cli.BoolFlag{ Name: "recursive", Aliases: []string{"r"}, + Value: true, Usage: "recursively check or repair", }, + &cli.BoolFlag{ + Name: "repair", + Usage: "repair specified path if it's broken (Note: repair attr and stat only)", + }, &cli.BoolFlag{ Name: "sync-dir-stat", Usage: "sync stat of all directories, even if they are existed and not broken (NOTE: it may take a long time for huge trees)", }, + &cli.UintFlag{ + Name: "threads", + Value: 20, + Usage: "number of concurrent threads", + }, }, } } +const ( + checkMetaFlag = 1 << iota + checkDataFlag +) + func fsck(ctx *cli.Context) error { setup(ctx, 1) + removePassword(ctx.Args().Get(0)) + + var flag int + switch ctx.String("check") { + case "meta": + flag = checkMetaFlag + case "data": + flag = checkDataFlag + case "both": + flag = checkMetaFlag | checkDataFlag + default: + logger.Fatalf("invalid check flag: %s", ctx.String("check")) + } + if ctx.Bool("repair") && ctx.String("path") == "" { logger.Fatalf("Please provide the path to repair with `--path` option") } - removePassword(ctx.Args().Get(0)) + + path := ctx.String("path") + if path == "" { + path = "/" + } + if !strings.HasPrefix(path, "/") { + logger.Fatalf("File path should be the absolute path within JuiceFS") + } + defer logger.Infof("Check %s [%s] finished, check log for more details", path, ctx.String("check")) + m := meta.NewClient(ctx.Args().Get(0), nil) format, err := m.Load(true) if err != nil { logger.Fatalf("load setting: %s", err) } - var c = meta.NewContext(0, 0, []uint32{0}) - if p := ctx.String("path"); p != "" { - if !strings.HasPrefix(p, "/") { - logger.Fatalf("File path should be the absolute path within JuiceFS") - } - return m.Check(c, p, ctx.Bool("repair"), ctx.Bool("recursive"), ctx.Bool("sync-dir-stat")) - } - chunkConf := chunk.Config{ - BlockSize: format.BlockSize * 1024, - Compress: format.Compression, - GetTimeout: time.Second * 60, - PutTimeout: time.Second * 60, - MaxUpload: 20, - BufferSize: 300 << 20, - CacheDir: "memory", + if ctx.Bool("sync-dir-stat") && !format.DirStats { + logger.Warn("dir stats is disabled, flag '--sync-dir-stat' will be ignored") } - blob, err := createStorage(*format) - if err != nil { - logger.Fatalf("object storage: %s", err) - } - logger.Infof("Data use %s", blob) - blob = object.WithPrefix(blob, "chunks/") - objs, err := osync.ListAll(blob, "", "", "", true) - if err != nil { - logger.Fatalf("list all blocks: %s", err) + threads := ctx.Uint("threads") + if threads == 0 { + threads = 20 + logger.Warnf("threads number set to %d", threads) } - // Find all blocks in object storage + var c = meta.NewContext(0, 0, []uint32{0}) progress := utils.NewProgress(false) - blockDSpin := progress.AddDoubleSpinner("Found blocks") - var blocks = make(map[string]int64) - for obj := range objs { - if obj == nil { - break // failed listing - } - if obj.IsDir() { - continue - } - logger.Debugf("found block %s", obj.Key()) - parts := strings.Split(obj.Key(), "/") - if len(parts) != 3 { - continue - } - name := parts[2] - blocks[name] = obj.Size() - blockDSpin.IncrInt64(obj.Size()) - } - blockDSpin.Done() - if progress.Quiet { - c, b := blockDSpin.Current() - logger.Infof("Found %d blocks (%d bytes)", c, b) - } - - // List all slices in metadata engine - sliceCSpin := progress.AddCountSpinner("Listed slices") - slices := make(map[meta.Ino][]meta.Slice) - r := m.ListSlices(c, slices, false, sliceCSpin.Increment) - if r != 0 { - logger.Fatalf("list all slices: %s", r) - } - sliceCSpin.Done() - delfilesSpin := progress.AddCountSpinner("Deleted files") - delfiles := make(map[meta.Ino]bool) - err = m.ScanDeletedObject(c, nil, nil, nil, func(ino meta.Ino, size uint64, ts int64) (clean bool, err error) { - delfilesSpin.Increment() - delfiles[ino] = true - return false, nil - }) - if err != nil { - logger.Warnf("scan deleted objects: %s", err) + var metaChecker func(meta.Ino, string, *meta.Attr) error + if flag&checkMetaFlag != 0 { + metaChecker = m.GetMetaChecker(c, ctx.Bool("repair"), ctx.Bool("sync-dir-stat")) } - delfilesSpin.Done() - // Scan all slices to find lost blocks - skippedSlices := progress.AddCountSpinner("Skipped slices") - sliceCBar := progress.AddCountBar("Scanned slices", sliceCSpin.Current()) - sliceBSpin := progress.AddByteSpinner("Scanned slices") - lostDSpin := progress.AddDoubleSpinner("Lost blocks") + var dataChecker func(meta.Ino, []meta.Slice) error brokens := make(map[meta.Ino]string) - for inode, ss := range slices { - if delfiles[inode] { - skippedSlices.IncrBy(len(ss)) - continue + if flag&checkDataFlag != 0 { + sliceCBar := progress.AddCountBar("Scanned slices", 0) + sliceBSpin := progress.AddByteSpinner("Scanned slices") + lostDSpin := progress.AddDoubleSpinner("Lost blocks") + defer func() { + if progress.Quiet { + logger.Infof("Used by %d slices (%d bytes)", sliceCBar.Current(), sliceBSpin.Current()) + } + if lc, lb := lostDSpin.Current(); lc > 0 { + msg := fmt.Sprintf("%d objects are lost (%d bytes), %d broken files:\n", lc, lb, len(brokens)) + msg += fmt.Sprintf("%13s: PATH\n", "INODE") + var fileList []string + for i, p := range brokens { + fileList = append(fileList, fmt.Sprintf("%13d: %s", i, p)) + } + sort.Strings(fileList) + msg += strings.Join(fileList, "\n") + logger.Error(msg) + } + }() + + chunkConf := chunk.Config{ + BlockSize: format.BlockSize * 1024, + Compress: format.Compression, + GetTimeout: time.Second * 60, + PutTimeout: time.Second * 60, + MaxUpload: 20, + BufferSize: 300 << 20, + CacheDir: "memory", } - for _, s := range ss { - n := (s.Size - 1) / uint32(chunkConf.BlockSize) - for i := uint32(0); i <= n; i++ { - sz := chunkConf.BlockSize - if i == n { - sz = int(s.Size) - int(i)*chunkConf.BlockSize + + blob, err := createStorage(*format) + if err != nil { + logger.Fatalf("object storage: %s", err) + } + logger.Infof("Data use %s", blob) + blob = object.WithPrefix(blob, "chunks/") + + var blocks = make(map[string]int64) + if path != "/" { + objs, err := osync.ListAll(blob, "", "", "", true) + if err != nil { + logger.Fatalf("list all blocks: %s", err) + } + + // Find all blocks in object storage + blockDSpin := progress.AddDoubleSpinner("Found blocks") + for obj := range objs { + if obj == nil { + break // failed listing + } + if obj.IsDir() { + continue + } + + parts := strings.Split(obj.Key(), "/") + if len(parts) != 3 { + continue } - key := fmt.Sprintf("%d_%d_%d", s.Id, i, sz) - if _, ok := blocks[key]; !ok { - var objKey string - if format.HashPrefix { - objKey = fmt.Sprintf("%02X/%v/%s", s.Id%256, s.Id/1000/1000, key) - } else { - objKey = fmt.Sprintf("%v/%v/%s", s.Id/1000/1000, s.Id/1000, key) + name := parts[2] + blocks[name] = obj.Size() + blockDSpin.IncrInt64(obj.Size()) + } + blockDSpin.Done() + } + + dataChecker = func(inode meta.Ino, ss []meta.Slice) error { + sliceCBar.IncrTotal(int64(len(ss))) + for _, s := range ss { + n := (s.Size - 1) / uint32(chunkConf.BlockSize) + for i := uint32(0); i <= n; i++ { + sz := chunkConf.BlockSize + if i == n { + sz = int(s.Size) - int(i)*chunkConf.BlockSize } - if _, err := blob.Head(objKey); err != nil { - if _, ok := brokens[inode]; !ok { - if ps := m.GetPaths(meta.Background, inode); len(ps) > 0 { - brokens[inode] = ps[0] - } else { - brokens[inode] = fmt.Sprintf("inode:%d", inode) + key := fmt.Sprintf("%d_%d_%d", s.Id, i, sz) + if _, ok := blocks[key]; !ok { + var objKey string + if format.HashPrefix { + objKey = fmt.Sprintf("%02X/%v/%s", s.Id%256, s.Id/1000/1000, key) + } else { + objKey = fmt.Sprintf("%v/%v/%s", s.Id/1000/1000, s.Id/1000, key) + } + if _, err := blob.Head(objKey); err != nil { + if _, ok := brokens[inode]; !ok { + if ps := m.GetPaths(meta.Background, inode); len(ps) > 0 { + brokens[inode] = ps[0] + } else { + brokens[inode] = fmt.Sprintf("inode:%d", inode) + } } + logger.Errorf("can't find block %s for file %s: %s", objKey, brokens[inode], err) + lostDSpin.IncrInt64(int64(sz)) } - logger.Errorf("can't find block %s for file %s: %s", objKey, brokens[inode], err) - lostDSpin.IncrInt64(int64(sz)) } } + sliceCBar.Increment() + sliceBSpin.IncrInt64(int64(s.Size)) } - sliceCBar.Increment() - sliceBSpin.IncrInt64(int64(s.Size)) - } - } - progress.Done() - if progress.Quiet { - logger.Infof("Used by %d slices (%d bytes)", sliceCBar.Current(), sliceBSpin.Current()) - } - if lc, lb := lostDSpin.Current(); lc > 0 { - msg := fmt.Sprintf("%d objects are lost (%d bytes), %d broken files:\n", lc, lb, len(brokens)) - msg += fmt.Sprintf("%13s: PATH\n", "INODE") - var fileList []string - for i, p := range brokens { - fileList = append(fileList, fmt.Sprintf("%13d: %s", i, p)) + return nil } - sort.Strings(fileList) - msg += strings.Join(fileList, "\n") - logger.Fatal(msg) } - return nil + err = m.Check(c, path, ctx.Bool("recursive"), threads, metaChecker, dataChecker, progress) + progress.Done() + return err } diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 8234152395d6..0764daa61ae4 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -121,6 +121,8 @@ type engine interface { doSetFacl(ctx Context, ino Ino, aclType uint8, rule *aclAPI.Rule) syscall.Errno doGetFacl(ctx Context, ino Ino, aclType uint8, aclId uint32, rule *aclAPI.Rule) syscall.Errno cacheACLs(ctx Context) error + + ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno } type trashSliceScan func(ss []Slice, ts int64) (clean bool, err error) @@ -1749,12 +1751,161 @@ func (m *baseMeta) walk(ctx Context, inode Ino, p string, attr *Attr, walkFn met return 0 } -func (m *baseMeta) Check(ctx Context, fpath string, repair bool, recursive bool, statAll bool) error { +func (m *baseMeta) GetMetaChecker(ctx Context, repair, statAll bool) func(Ino, string, *Attr) error { + format := m.GetFormat() + return func(inode Ino, path string, attr *Attr) error { + var err error + if attr.Typ != TypeDirectory { + return err + } + + var attrBroken, statBroken bool + if attr.Full { + nlink, st := m.countDirNlink(ctx, inode) + if st == syscall.ENOENT { + return err + } + if st != 0 { + return errors.Errorf("count nlink for inode %d: %v", inode, st) + } + if attr.Nlink != nlink { + logger.Warnf("nlink of %s should be %d, but got %d", path, nlink, attr.Nlink) + attrBroken = true + } + } else { + logger.Warnf("attribute of %s is missing", path) + attrBroken = true + } + + if attrBroken { + if repair { + if !attr.Full { + now := time.Now().Unix() + attr.Mode = 0644 + attr.Uid = ctx.Uid() + attr.Gid = ctx.Gid() + attr.Atime = now + attr.Mtime = now + attr.Ctime = now + attr.Length = 4 << 10 + } + if st1 := m.en.doRepair(ctx, inode, attr); st1 == 0 || st1 == syscall.ENOENT { + logger.Debugf("Path %s (inode %d) is successfully repaired", path, inode) + } else { + err = errors.Errorf("repair path %s inode %d: %s", path, inode, st1) + } + } else { + err = errors.Errorf("Path %s (inode %d) can be repaired, please re-run with '--path %s --repair' to fix it", path, inode, path) + } + } + + if format.DirStats { + stat, st := m.en.doGetDirStat(ctx, inode, false) + if st == syscall.ENOENT { + return err + } + if st != 0 { + return errors.Errorf("get dir stat for inode %d: %v", inode, st) + } + if stat == nil || stat.space < 0 || stat.inodes < 0 { + logger.Warnf("usage stat of %s is missing or broken", path) + statBroken = true + } + + if !repair && statAll { + s, st := m.calcDirStat(ctx, inode) + if st != 0 { + return errors.Errorf("calc dir stat for inode %d: %v", inode, st) + } + if stat.space != s.space || stat.inodes != s.inodes { + logger.Warnf("usage stat of %s should be %v, but got %v", path, s, stat) + statBroken = true + } + } + + if repair { + if statBroken || statAll { + if _, st := m.en.doSyncDirStat(ctx, inode); st == 0 || st == syscall.ENOENT { + logger.Debugf("Stat of path %s (inode %d) is successfully synced", path, inode) + } else { + err = errors.Errorf("sync stat of path %s inode %d: %s", path, inode, st) + } + } + } else if statBroken { + err = errors.Errorf("Stat of path %s (inode %d) should be synced, please re-run with '--path %s --repair --sync-dir-stat' to fix it", path, inode, path) + } + } + return err + } +} + +func (m *baseMeta) Check(ctx Context, fpath string, recursive bool, threads uint, metaChecker func(Ino, string, *Attr) error, dataChecker func(Ino, []Slice) error, progress *utils.Progress) error { + if metaChecker == nil && dataChecker == nil { + return nil + } else if metaChecker == nil && dataChecker != nil { + var sliceCSpin *utils.Bar + if progress != nil { + sliceCSpin = progress.AddCountSpinner("Listed slices") + } + var showProgress func() + if sliceCSpin != nil { + showProgress = sliceCSpin.Increment + } + + slices := make(map[Ino][]Slice) + r := m.en.ListSlices(ctx, slices, false, showProgress) + if r != 0 { + return errors.Errorf("list all slices: %s", r) + } + if sliceCSpin != nil { + sliceCSpin.Done() + } + + var delfilesSpin *utils.Bar + if progress != nil { + delfilesSpin = progress.AddCountSpinner("Deleted files") + } + delfiles := make(map[Ino]bool) + err := m.ScanDeletedObject(ctx, nil, nil, nil, func(ino Ino, size uint64, ts int64) (clean bool, err error) { + if delfilesSpin != nil { + delfilesSpin.Increment() + } + delfiles[ino] = true + return false, nil + }) + if err != nil { + logger.Warnf("scan deleted objects: %s", err) + } + if delfilesSpin != nil { + delfilesSpin.Done() + } + + var skippedSlices *utils.Bar + if progress != nil { + skippedSlices = progress.AddCountSpinner("Skipped slices") + } + for inode, ss := range slices { + if delfiles[inode] { + if skippedSlices != nil { + skippedSlices.IncrBy(len(ss)) + } + continue + } + if err := dataChecker(inode, ss); err != nil { + logger.Error(err) + } + } + if skippedSlices != nil { + skippedSlices.Done() + } + return nil + } + var attr Attr var inode = RootInode var parent = RootInode attr.Typ = TypeDirectory - if fpath == "/" { + if fpath == "/" || fpath == "" { if st := m.GetAttr(ctx, inode, &attr); st != 0 && st != syscall.ENOENT { logger.Errorf("GetAttr inode %d: %s", inode, st) return st @@ -1766,8 +1917,7 @@ func (m *baseMeta) Check(ctx Context, fpath string, repair bool, recursive bool, for i, name := range ps { parent = inode if st := m.Lookup(ctx, parent, name, &inode, &attr, false); st != 0 { - logger.Errorf("Lookup parent %d name %s: %s", parent, name, st) - return st + return errors.Errorf("Lookup parent %s name %s err: %s", parent, name, st) } if !attr.Full && i < len(ps)-1 { // missing attribute @@ -1805,117 +1955,42 @@ func (m *baseMeta) Check(ctx Context, fpath string, repair bool, recursive bool, } }() - format, err := m.Load(false) - if err != nil { - return errors.Wrap(err, "load meta format") - } - if statAll && !format.DirStats { - logger.Warn("dir stats is disabled, flag '--sync-dir-stat' will be ignored") - } - var wg sync.WaitGroup - for i := 0; i < 20; i++ { + for i := 0; i < int(threads); i++ { wg.Add(1) go func() { defer wg.Done() - for e := range nodes { - inode := e.inode - path := e.path - attr := e.attr - if attr.Typ != TypeDirectory { - // TODO - continue - } - - var attrBroken, statBroken bool - if attr.Full { - nlink, st := m.countDirNlink(ctx, inode) - if st == syscall.ENOENT { - continue - } - if st != 0 { - hasError = true - logger.Errorf("Count nlink for inode %d: %s", inode, st) - continue - } - if attr.Nlink != nlink { - logger.Warnf("nlink of %s should be %d, but got %d", path, nlink, attr.Nlink) - attrBroken = true - } - } else { - logger.Warnf("attribute of %s is missing", path) - attrBroken = true - } - - if attrBroken { - if repair { - if !attr.Full { - now := time.Now().Unix() - attr.Mode = 0644 - attr.Uid = ctx.Uid() - attr.Gid = ctx.Gid() - attr.Atime = now - attr.Mtime = now - attr.Ctime = now - attr.Length = 4 << 10 - } - if st1 := m.en.doRepair(ctx, inode, attr); st1 == 0 || st1 == syscall.ENOENT { - logger.Debugf("Path %s (inode %d) is successfully repaired", path, inode) - } else { - hasError = true - logger.Errorf("Repair path %s inode %d: %s", path, inode, st1) - } - } else { - logger.Warnf("Path %s (inode %d) can be repaired, please re-run with '--path %s --repair' to fix it", path, inode, path) + for n := range nodes { + if metaChecker != nil { + if err := metaChecker(n.inode, n.path, n.attr); err != nil { + logger.Error(err) hasError = true } } - - if format.DirStats { - stat, st := m.en.doGetDirStat(ctx, inode, false) - if st == syscall.ENOENT { - continue - } - if st != 0 { - hasError = true - logger.Errorf("get dir stat for inode %d: %v", inode, st) - continue - } - if stat == nil || stat.space < 0 || stat.inodes < 0 { - logger.Warnf("usage stat of %s is missing or broken", path) - statBroken = true - } - - if !repair && statAll { - s, st := m.calcDirStat(ctx, inode) + if n.attr.Typ == TypeFile && dataChecker != nil { + chunkCnt := uint32((n.attr.Length + ChunkSize - 1) / ChunkSize) + for i := uint32(0); i < chunkCnt; i++ { + ss, st := m.en.doRead(ctx, n.inode, i) if st != 0 { + logger.Errorf("path %s read inode %d chunk %d: %s", n.path, n.inode, i, st) hasError = true - logger.Errorf("calc dir stat for inode %d: %v", inode, st) - continue - } - if stat.space != s.space || stat.inodes != s.inodes { - logger.Warnf("usage stat of %s should be %v, but got %v", path, s, stat) - statBroken = true } - } - - if repair { - if statBroken || statAll { - if _, st := m.en.doSyncDirStat(ctx, inode); st == 0 || st == syscall.ENOENT { - logger.Debugf("Stat of path %s (inode %d) is successfully synced", path, inode) - } else { - hasError = true - logger.Errorf("Sync stat of path %s inode %d: %s", path, inode, st) + slices := make([]Slice, 0, len(ss)) + for _, s := range ss { + if s.id > 0 { + slices = append(slices, Slice{Id: s.id, Size: s.size}) } } - } else if statBroken { - logger.Warnf("Stat of path %s (inode %d) should be synced, please re-run with '--path %s --repair --sync-dir-stat' to fix it", path, inode, path) - hasError = true + if err := dataChecker(n.inode, slices); err != nil { + logger.Error(err) + hasError = true + } } } } }() } + wg.Wait() if hasError { return errors.New("some errors occurred, please check the log of fsck") diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go index d389f538915e..c144350fc1d3 100644 --- a/pkg/meta/base_test.go +++ b/pkg/meta/base_test.go @@ -2407,7 +2407,8 @@ func testCheckAndRepair(t *testing.T, m Meta) { dirAttr.Nlink = 0 setAttr(t, m, d4Inode, dirAttr) - if err := m.Check(Background, "/check", false, false, false); err == nil { + metaChecker := m.GetMetaChecker(Background, false, false) + if err := m.Check(Background, "/check", false, 20, metaChecker, nil, nil); err == nil { t.Fatal("check should fail") } if st := m.GetAttr(Background, checkInode, dirAttr); st != 0 { @@ -2417,7 +2418,8 @@ func testCheckAndRepair(t *testing.T, m Meta) { t.Fatalf("checkInode nlink should is 0 now: %d", dirAttr.Nlink) } - if err := m.Check(Background, "/check", true, false, false); err != nil { + metaChecker = m.GetMetaChecker(Background, true, false) + if err := m.Check(Background, "/check", false, 20, metaChecker, nil, nil); err != nil { t.Fatalf("check: %s", err) } if st := m.GetAttr(Background, checkInode, dirAttr); st != 0 { @@ -2427,7 +2429,7 @@ func testCheckAndRepair(t *testing.T, m Meta) { t.Fatalf("checkInode nlink should is 3 now: %d", dirAttr.Nlink) } - if err := m.Check(Background, "/check/d1/d2", true, false, false); err != nil { + if err := m.Check(Background, "/check/d1/d2", false, 20, metaChecker, nil, nil); err != nil { t.Fatalf("check: %s", err) } if st := m.GetAttr(Background, d2Inode, dirAttr); st != 0 { @@ -2444,7 +2446,7 @@ func testCheckAndRepair(t *testing.T, m Meta) { } if m.Name() != "etcd" { - if err := m.Check(Background, "/", true, true, false); err != nil { + if err := m.Check(Background, "/", true, 20, metaChecker, nil, nil); err != nil { t.Fatalf("check: %s", err) } for _, ino := range []Ino{checkInode, d1Inode, d2Inode, d3Inode} { diff --git a/pkg/meta/interface.go b/pkg/meta/interface.go index f5b272cc849c..01ed3229c2f4 100644 --- a/pkg/meta/interface.go +++ b/pkg/meta/interface.go @@ -430,8 +430,11 @@ type Meta interface { Clone(ctx Context, srcIno, dstParentIno Ino, dstName string, cmode uint8, cumask uint16, count, total *uint64) syscall.Errno // GetPaths returns all paths of an inode GetPaths(ctx Context, inode Ino) []string - // Check integrity of an absolute path and repair it if asked - Check(ctx Context, fpath string, repair bool, recursive bool, statAll bool) error + // GetMetaChecker returns a function to check metadata + GetMetaChecker(ctx Context, repair, statAll bool) func(Ino, string, *Attr) error + // Check meta and data integrity of an absolute path + Check(ctx Context, fpath string, recursive bool, threads uint, metaChecker func(Ino, string, *Attr) error, dataChecker func(Ino, []Slice) error, progress *utils.Progress) error + // Change root to a directory specified by subdir Chroot(ctx Context, subdir string) syscall.Errno // chroot set the root directory by inode