cmd/fsck: add repair option to repair broken directories (#2654)

cmd/fsck.go

@@ -43,18 +43,41 @@ It scans all objects in data storage and slices in metadata, comparing them to s
 lost object or broken file.
 
 Examples:
-$ juicefs fsck redis://localhost`,
+$ juicefs fsck redis://localhost
+
+# Repair broken directories
+$ juicefs fsck redis://localhost --path /d1/d2 --repair`,
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "path",
+				Usage: "absolute path within JuiceFS to check",
+			},
+			&cli.BoolFlag{
+				Name:  "repair",
+				Usage: "repair specified path if it's broken",
+			},
+		},
 	}
 }
 
 func fsck(ctx *cli.Context) error {
 	setup(ctx, 1)
+	if ctx.Bool("repair") && ctx.String("path") == "" {
+		logger.Fatalf("Please provide the path to repair with `--path` option")
+	}
 	removePassword(ctx.Args().Get(0))
 	m := meta.NewClient(ctx.Args().Get(0), &meta.Config{Retries: 10, Strict: true})
 	format, err := m.Load(true)
 	if err != nil {
 		logger.Fatalf("load setting: %s", err)
 	}
+	var c = meta.NewContext(0, 0, []uint32{0})
+	if p := ctx.String("path"); p != "" {
+		if !strings.HasPrefix(p, "/") {
+			logger.Fatalf("File path should be the absolute path within JuiceFS")
+		}
+		return m.Check(c, p, ctx.Bool("repair"))
+	}
 
 	chunkConf := chunk.Config{
 		BlockSize:  format.BlockSize * 1024,
@@ -106,7 +129,6 @@ func fsck(ctx *cli.Context) error {
 
 	// List all slices in metadata engine
 	sliceCSpin := progress.AddCountSpinner("Listed slices")
-	var c = meta.NewContext(0, 0, []uint32{0})
 	slices := make(map[meta.Ino][]meta.Slice)
 	r := m.ListSlices(c, slices, false, sliceCSpin.Increment)
 	if r != 0 {

cmd/fsck_test.go

@@ -35,4 +35,7 @@ func TestFsck(t *testing.T) {
 	if err := Main([]string{"", "fsck", testMeta}); err != nil {
 		t.Fatalf("fsck failed: %s", err)
 	}
+	if err := Main([]string{"", "fsck", testMeta, "--path", "/f3.txt"}); err != nil {
+		t.Fatalf("fsck failed: %s", err)
+	}
 }

cmd/main.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"syscall"
 
 	"github.com/erikdubbelboer/gspt"
 	"github.com/google/gops/agent"
@@ -82,7 +83,11 @@ func Main(args []string) error {
 			args = []string{"mount", "--help"}
 		}
 	}
-	return app.Run(reorderOptions(app, args))
+	err := app.Run(reorderOptions(app, args))
+	if errno, ok := err.(syscall.Errno); ok && errno == 0 {
+		err = nil
+	}
+	return err
 }
 
 func calledViaMount(args []string) bool {

pkg/meta/base.go

@@ -19,6 +19,7 @@ package meta
 import (
 	"encoding/json"
 	"fmt"
+	"path"
 	"runtime"
 	"sort"
 	"strings"
@@ -73,6 +74,7 @@ type engine interface {
 	doSetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno
 	doRemoveXattr(ctx Context, inode Ino, name string) syscall.Errno
 	doGetParents(ctx Context, inode Ino) map[Ino]int
+	doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno
 
 	GetSession(sid uint64, detail bool) (*Session, error)
 }
@@ -951,6 +953,52 @@ func (m *baseMeta) GetParents(ctx Context, inode Ino) map[Ino]int {
 	}
 }
 
+func (m *baseMeta) Check(ctx Context, fpath string, repair bool) (st syscall.Errno) {
+	var attr Attr
+	var inode Ino
+	var parent Ino = 1
+	ps := strings.Split(fpath, "/")
+	for i, name := range ps {
+		if len(name) == 0 {
+			continue
+		}
+		if st = m.Lookup(ctx, parent, name, &inode, &attr); st != 0 {
+			logger.Errorf("Lookup parent %d name %s: %s", parent, name, st)
+			return
+		}
+		if attr.Full {
+			parent = inode
+			continue
+		}
+
+		// missing attribute
+		p := "/" + path.Join(ps[:i+1]...)
+		if attr.Typ != TypeDirectory { // TODO: determine file size?
+			logger.Errorf("Path %s (inode %d type %d) cannot be auto-repaired, you have to repair it manually or remove it", p, inode, attr.Typ)
+		} else if !repair {
+			logger.Warnf("Path %s (inode %d) can be repaired, please re-check with 'repair' enabled", p, inode)
+		} else { // repair directory inode
+			now := time.Now().Unix()
+			attr.Mode = 0644
+			attr.Uid = ctx.Uid()
+			attr.Gid = ctx.Gid()
+			attr.Atime = now
+			attr.Mtime = now
+			attr.Ctime = now
+			attr.Length = 4 << 10
+			attr.Parent = parent
+			if st = m.en.doRepair(ctx, inode, &attr); st == 0 {
+				logger.Infof("Path %s (inode %d) is successfully repaired", p, inode)
+			} else {
+				logger.Errorf("Repair path %s inode %d: %s", p, inode, st)
+			}
+		}
+		return // handle one missing inode at a time
+	}
+	logger.Infof("Path %s inode %d is valid", fpath, parent)
+	return
+}
+
 func (m *baseMeta) fileDeleted(opened bool, inode Ino, length uint64) {
 	if opened {
 		m.Lock()

pkg/meta/interface.go

@@ -349,6 +349,8 @@ type Meta interface {
 	ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno
 	// Remove all files and directories recursively.
 	Remove(ctx Context, parent Ino, name string, count *uint64) syscall.Errno
+	// Check integrity of an absolute path and repair it if asked
+	Check(ctx Context, fpath string, repair bool) syscall.Errno
 
 	// OnMsg add a callback for the given message type.
 	OnMsg(mtype uint32, cb MsgCallback)

pkg/meta/redis.go

@@ -2824,6 +2824,23 @@ func (m *redisMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool,
 	return errno(err)
 }
 
+func (m *redisMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno {
+	return errno(m.txn(ctx, func(tx *redis.Tx) error {
+		attr.Nlink = 2
+		vals, err := tx.HGetAll(ctx, m.entryKey(inode)).Result()
+		if err != nil {
+			return err
+		}
+		for _, v := range vals {
+			typ, _ := m.parseEntry([]byte(v))
+			if typ == TypeDirectory {
+				attr.Nlink++
+			}
+		}
+		return tx.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0).Err()
+	}, m.entryKey(inode), m.inodeKey(inode)))
+}
+
 func (m *redisMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno {
 	defer m.timeit(time.Now())
 	inode = m.checkRoot(inode)

pkg/meta/sql.go

@@ -2605,6 +2605,34 @@ func (m *dbMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, sh
 	return errno(err)
 }
 
+func (m *dbMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno {
+	n := &node{
+		Inode:  inode,
+		Type:   attr.Typ,
+		Mode:   attr.Mode,
+		Uid:    attr.Uid,
+		Gid:    attr.Gid,
+		Atime:  attr.Atime * 1e6,
+		Mtime:  attr.Mtime * 1e6,
+		Ctime:  attr.Ctime * 1e6,
+		Length: 4 << 10,
+		Parent: attr.Parent,
+	}
+	return errno(m.txn(func(s *xorm.Session) error {
+		n.Nlink = 2
+		var rows []edge
+		if err := s.Find(&rows, &edge{Parent: inode}); err != nil {
+			return err
+		}
+		for _, row := range rows {
+			if row.Type == TypeDirectory {
+				n.Nlink++
+			}
+		}
+		return mustInsert(s, n)
+	}, inode))
+}
+
 func (m *dbMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno {
 	defer m.timeit(time.Now())
 	inode = m.checkRoot(inode)

pkg/meta/tkv.go

@@ -2221,6 +2221,21 @@ func (m *kvMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, sh
 	return 0
 }
 
+func (m *kvMeta) doRepair(ctx Context, inode Ino, attr *Attr) syscall.Errno {
+	return errno(m.txn(func(tx kvTxn) error {
+		attr.Nlink = 2
+		_ = tx.scanValues(m.entryKey(inode, ""), 0, func(k, v []byte) bool {
+			typ, _ := m.parseEntry(v)
+			if typ == TypeDirectory {
+				attr.Nlink++
+			}
+			return false
+		})
+		tx.set(m.inodeKey(inode), m.marshal(attr))
+		return nil
+	}, inode))
+}
+
 func (m *kvMeta) GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno {
 	defer m.timeit(time.Now())
 	inode = m.checkRoot(inode)