Merge pull request #4550 from ndecker/ls-ncdu

Ls ncdu

changelog/unreleased/issue-4549

@@ -0,0 +1,11 @@
+Enhancement: Add `--ncdu` option to `ls` command
+
+NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories.
+It has an option to save a directory tree and analyse it later.
+The `ls` command now supports the `--ncdu` option which outputs information
+about a snapshot in the NCDU format.
+
+You can use it as follows: `restic ls latest --ncdu | ncdu -f -`
+
+https://github.com/restic/restic/issues/4549
+https://github.com/restic/restic/pull/4550

cmd/restic/cmd_find.go

@@ -260,7 +260,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
+	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
@@ -327,7 +327,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		debug.Log("    found match\n")
 		f.out.PrintPattern(nodepath, node)
 		return nil
-	})
+	}})
 }
 
 func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
@@ -338,7 +338,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
+	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
@@ -388,7 +388,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 		}
 
 		return nil
-	})
+	}})
 }
 
 var errAllPacksFound = errors.New("all packs found")

cmd/restic/cmd_ls.go

@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"io"
 	"os"
 	"strings"
 	"time"
@@ -51,6 +53,7 @@ type LsOptions struct {
 	restic.SnapshotFilter
 	Recursive     bool
 	HumanReadable bool
+	Ncdu          bool
 }
 
 var lsOptions LsOptions
@@ -63,16 +66,47 @@ func init() {
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 	flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
+	flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')")
 }
 
-type lsSnapshot struct {
-	*restic.Snapshot
-	ID         *restic.ID `json:"id"`
-	ShortID    string     `json:"short_id"`
-	StructType string     `json:"struct_type"` // "snapshot"
+type lsPrinter interface {
+	Snapshot(sn *restic.Snapshot)
+	Node(path string, node *restic.Node)
+	LeaveDir(path string)
+	Close()
+}
+
+type jsonLsPrinter struct {
+	enc *json.Encoder
+}
+
+func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
+	type lsSnapshot struct {
+		*restic.Snapshot
+		ID         *restic.ID `json:"id"`
+		ShortID    string     `json:"short_id"`
+		StructType string     `json:"struct_type"` // "snapshot"
+	}
+
+	err := p.enc.Encode(lsSnapshot{
+		Snapshot:   sn,
+		ID:         sn.ID(),
+		ShortID:    sn.ID().Str(),
+		StructType: "snapshot",
+	})
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
 }
 
 // Print node in our custom JSON format, followed by a newline.
+func (p *jsonLsPrinter) Node(path string, node *restic.Node) {
+	err := lsNodeJSON(p.enc, path, node)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+}
+
 func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	n := &struct {
 		Name        string      `json:"name"`
@@ -114,10 +148,117 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	return enc.Encode(n)
 }
 
+func (p *jsonLsPrinter) LeaveDir(_ string) {}
+func (p *jsonLsPrinter) Close()            {}
+
+type ncduLsPrinter struct {
+	out   io.Writer
+	depth int
+}
+
+// lsSnapshotNcdu prints a restic snapshot in Ncdu save format.
+// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu.
+// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt
+func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) {
+	const NcduMajorVer = 1
+	const NcduMinorVer = 2
+
+	snapshotBytes, err := json.Marshal(sn)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+	p.depth++
+	fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
+}
+
+func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
+	type NcduNode struct {
+		Name   string `json:"name"`
+		Asize  uint64 `json:"asize"`
+		Dsize  uint64 `json:"dsize"`
+		Dev    uint64 `json:"dev"`
+		Ino    uint64 `json:"ino"`
+		NLink  uint64 `json:"nlink"`
+		NotReg bool   `json:"notreg"`
+		UID    uint32 `json:"uid"`
+		GID    uint32 `json:"gid"`
+		Mode   uint16 `json:"mode"`
+		Mtime  int64  `json:"mtime"`
+	}
+
+	outNode := NcduNode{
+		Name:   node.Name,
+		Asize:  node.Size,
+		Dsize:  node.Size,
+		Dev:    node.DeviceID,
+		Ino:    node.Inode,
+		NLink:  node.Links,
+		NotReg: node.Type != "dir" && node.Type != "file",
+		UID:    node.UID,
+		GID:    node.GID,
+		Mode:   uint16(node.Mode & os.ModePerm),
+		Mtime:  node.ModTime.Unix(),
+	}
+	//  bits according to inode(7) manpage
+	if node.Mode&os.ModeSetuid != 0 {
+		outNode.Mode |= 0o4000
+	}
+	if node.Mode&os.ModeSetgid != 0 {
+		outNode.Mode |= 0o2000
+	}
+	if node.Mode&os.ModeSticky != 0 {
+		outNode.Mode |= 0o1000
+	}
+
+	return json.Marshal(outNode)
+}
+
+func (p *ncduLsPrinter) Node(path string, node *restic.Node) {
+	out, err := lsNcduNode(path, node)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+
+	if node.Type == "dir" {
+		fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat("  ", p.depth), strings.Repeat("  ", p.depth+1), string(out))
+		p.depth++
+	} else {
+		fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat("  ", p.depth), string(out))
+	}
+}
+
+func (p *ncduLsPrinter) LeaveDir(_ string) {
+	p.depth--
+	fmt.Fprintf(p.out, "\n%s]", strings.Repeat("  ", p.depth))
+}
+
+func (p *ncduLsPrinter) Close() {
+	fmt.Fprint(p.out, "\n]\n")
+}
+
+type textLsPrinter struct {
+	dirs          []string
+	ListLong      bool
+	HumanReadable bool
+}
+
+func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
+	Verbosef("%v filtered by %v:\n", sn, p.dirs)
+}
+func (p *textLsPrinter) Node(path string, node *restic.Node) {
+	Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
+}
+
+func (p *textLsPrinter) LeaveDir(_ string) {}
+func (p *textLsPrinter) Close()            {}
+
 func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
 		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
 	}
+	if opts.Ncdu && gopts.JSON {
+		return errors.Fatal("only either '--json' or '--ncdu' can be specified")
+	}
 
 	// extract any specific directories to walk
 	var dirs []string
@@ -179,38 +320,21 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	var (
-		printSnapshot func(sn *restic.Snapshot)
-		printNode     func(path string, node *restic.Node)
-	)
+	var printer lsPrinter
 
 	if gopts.JSON {
-		enc := json.NewEncoder(globalOptions.stdout)
-
-		printSnapshot = func(sn *restic.Snapshot) {
-			err := enc.Encode(lsSnapshot{
-				Snapshot:   sn,
-				ID:         sn.ID(),
-				ShortID:    sn.ID().Str(),
-				StructType: "snapshot",
-			})
-			if err != nil {
-				Warnf("JSON encode failed: %v\n", err)
-			}
+		printer = &jsonLsPrinter{
+			enc: json.NewEncoder(globalOptions.stdout),
 		}
-
-		printNode = func(path string, node *restic.Node) {
-			err := lsNodeJSON(enc, path, node)
-			if err != nil {
-				Warnf("JSON encode failed: %v\n", err)
-			}
+	} else if opts.Ncdu {
+		printer = &ncduLsPrinter{
+			out: globalOptions.stdout,
 		}
 	} else {
-		printSnapshot = func(sn *restic.Snapshot) {
-			Verbosef("%v filtered by %v:\n", sn, dirs)
-		}
-		printNode = func(path string, node *restic.Node) {
-			Printf("%s\n", formatNode(path, node, opts.ListLong, opts.HumanReadable))
+		printer = &textLsPrinter{
+			dirs:          dirs,
+			ListLong:      opts.ListLong,
+			HumanReadable: opts.HumanReadable,
 		}
 	}
 
@@ -228,9 +352,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	printSnapshot(sn)
+	printer.Snapshot(sn)
 
-	err = walker.Walk(ctx, repo, *sn.Tree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
+	processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			return err
 		}
@@ -240,7 +364,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 
 		if withinDir(nodepath) {
 			// if we're within a dir, print the node
-			printNode(nodepath, node)
+			printer.Node(nodepath, node)
 
 			// if recursive listing is requested, signal the walker that it
 			// should continue walking recursively
@@ -261,11 +385,22 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 			return walker.ErrSkipNode
 		}
 		return nil
+	}
+
+	err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{
+		ProcessNode: processNode,
+		LeaveDir: func(path string) {
+			// the root path `/` has no corresponding node and is thus also skipped by processNode
+			if withinDir(path) && path != "/" {
+				printer.LeaveDir(path)
+			}
+		},
 	})
 
 	if err != nil {
 		return err
 	}
 
+	printer.Close()
 	return nil
 }

cmd/restic/cmd_ls_integration_test.go

@@ -2,18 +2,46 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
+func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte {
 	buf, err := withCaptureStdout(func() error {
 		gopts.Quiet = true
-		opts := LsOptions{}
-		return runLs(context.TODO(), opts, gopts, []string{snapshotID})
+		return runLs(context.TODO(), opts, gopts, args)
 	})
 	rtest.OK(t, err)
-	return strings.Split(buf.String(), "\n")
+	return buf.Bytes()
+}
+
+func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
+	out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID})
+	return strings.Split(string(out), "\n")
+}
+
+func assertIsValidJSON(t *testing.T, data []byte) {
+	// Sanity check: output must be valid JSON.
+	var v interface{}
+	err := json.Unmarshal(data, &v)
+	rtest.OK(t, err)
+}
+
+func TestRunLsNcdu(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	opts := BackupOptions{}
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+
+	ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"})
+	assertIsValidJSON(t, ncdu)
+
+	ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"})
+	assertIsValidJSON(t, ncdu)
 }