From 71792bf6e0354139cba6288c90663aa2c42e111a Mon Sep 17 00:00:00 2001 From: xixi Date: Mon, 6 Mar 2023 16:28:03 +0800 Subject: [PATCH 1/4] info recursively support fast mode Signed-off-by: xixi --- cmd/info.go | 15 ++++++++++++--- pkg/meta/utils.go | 47 +++++++++++++++++++++++++++++++++++++++++++++ pkg/vfs/internal.go | 11 ++++++++++- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/cmd/info.go b/cmd/info.go index e667b802eee0..41f3f17ba289 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -59,6 +59,11 @@ $ juicefs info -i 100`, Aliases: []string{"r"}, Usage: "get summary of directories recursively (NOTE: it may take a long time for huge trees)", }, + &cli.BoolFlag{ + Name: "fast", + Aliases: []string{"f"}, + Usage: "get summary of directories fastly (NOTE: it may be inaccurate)", + }, &cli.BoolFlag{ Name: "raw", Usage: "show internal raw information", @@ -73,10 +78,13 @@ func info(ctx *cli.Context) error { logger.Infof("Windows is not supported") return nil } - var recursive, raw uint8 + var recursive, fast, raw uint8 if ctx.Bool("recursive") { recursive = 1 } + if ctx.Bool("fast") { + fast = 1 + } if ctx.Bool("raw") { raw = 1 } @@ -108,11 +116,12 @@ func info(ctx *cli.Context) error { continue } - wb := utils.NewBuffer(8 + 10) + wb := utils.NewBuffer(8 + 11) wb.Put32(meta.InfoV2) - wb.Put32(10) + wb.Put32(11) wb.Put64(inode) wb.Put8(recursive) + wb.Put8(fast) wb.Put8(raw) _, err = f.Write(wb.Bytes()) if err != nil { diff --git a/pkg/meta/utils.go b/pkg/meta/utils.go index c486dfe0a280..2a1fd2db7608 100644 --- a/pkg/meta/utils.go +++ b/pkg/meta/utils.go @@ -369,3 +369,50 @@ func GetSummary(r Meta, ctx Context, inode Ino, summary *Summary, recursive bool } return 0 } + +func FastGetSummary(r Meta, ctx Context, inode Ino, summary *Summary, recursive bool) syscall.Errno { + var attr Attr + if st := r.GetAttr(ctx, inode, &attr); st != 0 { + return st + } + if attr.Typ == TypeDirectory { + return fastGetSummary(r, ctx, inode, summary, recursive) + } else { + summary.Files++ + summary.Length += attr.Length + summary.Size += uint64(align4K(attr.Length)) + } + return 0 +} + +func fastGetSummary(r Meta, ctx Context, inode Ino, summary *Summary, recursive bool) syscall.Errno { + space, _, err := r.GetDirStat(ctx, inode) + if err != nil { + return errno(err) + } + summary.Size += space + summary.Length += space + summary.Dirs++ + + var entries []*Entry + if st := r.Readdir(ctx, inode, 0, &entries); st != 0 { + return st + } + for _, e := range entries { + if e.Inode == inode || len(e.Name) == 2 && bytes.Equal(e.Name, []byte("..")) { + continue + } + if e.Attr.Typ == TypeDirectory { + if recursive { + if st := fastGetSummary(r, ctx, e.Inode, summary, recursive); st != 0 { + return st + } + } else { + summary.Dirs++ + } + } else { + summary.Files++ + } + } + return 0 +} diff --git a/pkg/vfs/internal.go b/pkg/vfs/internal.go index 4a274c339d08..15af3cccc1c6 100644 --- a/pkg/vfs/internal.go +++ b/pkg/vfs/internal.go @@ -394,12 +394,21 @@ func (v *VFS) handleInternalMsg(ctx meta.Context, cmd uint32, r *utils.Buffer, o if r.HasMore() { recursive = r.Get8() } + var fast uint8 + if r.HasMore() { + fast = r.Get8() + } var raw bool if r.HasMore() { raw = r.Get8() != 0 } - r := meta.GetSummary(v.Meta, ctx, inode, &info.Summary, recursive != 0) + var r syscall.Errno + if fast == 0 { + r = meta.GetSummary(v.Meta, ctx, inode, &info.Summary, recursive != 0) + } else { + r = meta.FastGetSummary(v.Meta, ctx, inode, &info.Summary, recursive != 0) + } if r != 0 { info.Failed = true info.Reason = r.Error() From 30df2b68a8de219b18a97733044700235ee6e264 Mon Sep 17 00:00:00 2001 From: xixi Date: Tue, 7 Mar 2023 11:41:09 +0800 Subject: [PATCH 2/4] replace fast with strict Signed-off-by: xixi --- cmd/info.go | 15 +++++++-------- pkg/vfs/internal.go | 10 +++++----- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/info.go b/cmd/info.go index 41f3f17ba289..041eb7e211f4 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -57,12 +57,11 @@ $ juicefs info -i 100`, &cli.BoolFlag{ Name: "recursive", Aliases: []string{"r"}, - Usage: "get summary of directories recursively (NOTE: it may take a long time for huge trees)", + Usage: "get summary of directories recursively (NOTE: it may be inaccurate, use --strict to get accurate result)", }, &cli.BoolFlag{ - Name: "fast", - Aliases: []string{"f"}, - Usage: "get summary of directories fastly (NOTE: it may be inaccurate)", + Name: "strict", + Usage: "get accurate summary of directories (NOTE: it may take a long time for huge trees)", }, &cli.BoolFlag{ Name: "raw", @@ -78,12 +77,12 @@ func info(ctx *cli.Context) error { logger.Infof("Windows is not supported") return nil } - var recursive, fast, raw uint8 + var recursive, strict, raw uint8 if ctx.Bool("recursive") { recursive = 1 } - if ctx.Bool("fast") { - fast = 1 + if ctx.Bool("strict") { + strict = 1 } if ctx.Bool("raw") { raw = 1 @@ -121,8 +120,8 @@ func info(ctx *cli.Context) error { wb.Put32(11) wb.Put64(inode) wb.Put8(recursive) - wb.Put8(fast) wb.Put8(raw) + wb.Put8(strict) _, err = f.Write(wb.Bytes()) if err != nil { logger.Fatalf("write message: %s", err) diff --git a/pkg/vfs/internal.go b/pkg/vfs/internal.go index 15af3cccc1c6..7ad440133afb 100644 --- a/pkg/vfs/internal.go +++ b/pkg/vfs/internal.go @@ -394,17 +394,17 @@ func (v *VFS) handleInternalMsg(ctx meta.Context, cmd uint32, r *utils.Buffer, o if r.HasMore() { recursive = r.Get8() } - var fast uint8 - if r.HasMore() { - fast = r.Get8() - } var raw bool if r.HasMore() { raw = r.Get8() != 0 } + var strict bool + if r.HasMore() { + strict = r.Get8() != 0 + } var r syscall.Errno - if fast == 0 { + if strict { r = meta.GetSummary(v.Meta, ctx, inode, &info.Summary, recursive != 0) } else { r = meta.FastGetSummary(v.Meta, ctx, inode, &info.Summary, recursive != 0) From 5064b14ae1cda206cbcb85d71f13997200c4e9bf Mon Sep 17 00:00:00 2001 From: xixi Date: Tue, 7 Mar 2023 11:43:39 +0800 Subject: [PATCH 3/4] add short alias for strict Signed-off-by: xixi --- cmd/info.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/info.go b/cmd/info.go index 041eb7e211f4..25693b584707 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -60,8 +60,9 @@ $ juicefs info -i 100`, Usage: "get summary of directories recursively (NOTE: it may be inaccurate, use --strict to get accurate result)", }, &cli.BoolFlag{ - Name: "strict", - Usage: "get accurate summary of directories (NOTE: it may take a long time for huge trees)", + Name: "strict", + Aliases: []string{"s"}, + Usage: "get accurate summary of directories (NOTE: it may take a long time for huge trees)", }, &cli.BoolFlag{ Name: "raw", From ececbdb59669510640ddac6d6906f3c2b68ccfb4 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Tue, 7 Mar 2023 23:33:15 +0800 Subject: [PATCH 4/4] remove short alias --- cmd/info.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/info.go b/cmd/info.go index 25693b584707..041eb7e211f4 100644 --- a/cmd/info.go +++ b/cmd/info.go @@ -60,9 +60,8 @@ $ juicefs info -i 100`, Usage: "get summary of directories recursively (NOTE: it may be inaccurate, use --strict to get accurate result)", }, &cli.BoolFlag{ - Name: "strict", - Aliases: []string{"s"}, - Usage: "get accurate summary of directories (NOTE: it may take a long time for huge trees)", + Name: "strict", + Usage: "get accurate summary of directories (NOTE: it may take a long time for huge trees)", }, &cli.BoolFlag{ Name: "raw",