Add repair command

changelog/unreleased/issue-1759

@@ -0,0 +1,20 @@
+Enhancement: Add `repair index` and `repair snapshots` commands
+
+The `rebuild-index` command has been renamed to `repair index`. The old name
+will still work, but is deprecated.
+
+When a snapshot was damaged, the only option up to now was to completely forget
+the snapshot, even if only some unimportant file was damaged.
+
+We've added a `repair snapshots` command, which can repair snapshots by removing
+damaged directories and missing files contents. Note that using this command
+can lead to data loss! Please see the "Troubleshooting" section in the documentation
+for more details.
+
+https://github.com/restic/restic/issues/1759
+https://github.com/restic/restic/issues/1714
+https://github.com/restic/restic/issues/1798
+https://github.com/restic/restic/issues/2334
+https://github.com/restic/restic/pull/2876
+https://forum.restic.net/t/corrupted-repo-how-to-repair/799
+https://forum.restic.net/t/recovery-options-for-damaged-repositories/1571

cmd/restic/cmd_check.go

@@ -245,7 +245,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	if suggestIndexRebuild {
-		Printf("Duplicate packs/old indexes are non-critical, you can run `restic rebuild-index' to correct this.\n")
+		Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n")
 	}
 	if mixedFound {
 		Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")

cmd/restic/cmd_prune.go

@@ -488,7 +488,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi
 			// Pack size does not fit and pack is needed => error
 			// If the pack is not needed, this is no error, the pack can
 			// and will be simply removed, see below.
-			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.\n",
+			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
 				id.Str(), p.unusedSize+p.usedSize, packSize)
 			return errorSizeNotMatching
 		}

cmd/restic/cmd_repair.go

@@ -0,0 +1,14 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+)
+
+var cmdRepair = &cobra.Command{
+	Use:   "repair",
+	Short: "Repair the repository",
+}
+
+func init() {
+	cmdRoot.AddCommand(cmdRepair)
+}

cmd/restic/cmd_rebuild_index.go → cmd/restic/cmd_repair_index.go

@@ -7,15 +7,15 @@ import (
 	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
-
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
 )
 
-var cmdRebuildIndex = &cobra.Command{
-	Use:   "rebuild-index [flags]",
+var cmdRepairIndex = &cobra.Command{
+	Use:   "index [flags]",
 	Short: "Build a new index",
 	Long: `
-The "rebuild-index" command creates a new index based on the pack files in the
+The "repair index" command creates a new index based on the pack files in the
 repository.
 
 EXIT STATUS
@@ -25,25 +25,37 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions)
+		return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions)
 	},
 }
 
-// RebuildIndexOptions collects all options for the rebuild-index command.
-type RebuildIndexOptions struct {
+var cmdRebuildIndex = &cobra.Command{
+	Use:               "rebuild-index [flags]",
+	Short:             cmdRepairIndex.Short,
+	Long:              cmdRepairIndex.Long,
+	Deprecated:        `Use "repair index" instead`,
+	DisableAutoGenTag: true,
+	RunE:              cmdRepairIndex.RunE,
+}
+
+// RepairIndexOptions collects all options for the repair index command.
+type RepairIndexOptions struct {
 	ReadAllPacks bool
 }
 
-var rebuildIndexOptions RebuildIndexOptions
+var repairIndexOptions RepairIndexOptions
 
 func init() {
+	cmdRepair.AddCommand(cmdRepairIndex)
+	// add alias for old name
 	cmdRoot.AddCommand(cmdRebuildIndex)
-	f := cmdRebuildIndex.Flags()
-	f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
 
+	for _, f := range []*pflag.FlagSet{cmdRepairIndex.Flags(), cmdRebuildIndex.Flags()} {
+		f.BoolVar(&repairIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch")
+	}
 }
 
-func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error {
+func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error {
 	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
@@ -58,7 +70,7 @@ func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts Global
 	return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet())
 }
 
-func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
+func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
 	var obsoleteIndexes restic.IDs
 	packSizeFromList := make(map[restic.ID]int64)
 	packSizeFromIndex := make(map[restic.ID]int64)

cmd/restic/cmd_repair_snapshots.go

@@ -0,0 +1,176 @@
+package main
+
+import (
+	"context"
+
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/walker"
+
+	"github.com/spf13/cobra"
+)
+
+var cmdRepairSnapshots = &cobra.Command{
+	Use:   "snapshots [flags] [snapshot ID] [...]",
+	Short: "Repair snapshots",
+	Long: `
+The "repair snapshots" command repairs broken snapshots. It scans the given
+snapshots and generates new ones with damaged directories and file contents
+removed. If the broken snapshots are deleted, a prune run will be able to
+clean up the repository.
+
+The command depends on a correct index, thus make sure to run "repair index"
+first!
+
+
+WARNING
+=======
+
+Repairing and deleting broken snapshots causes data loss! It will remove broken
+directories and modify broken files in the modified snapshots.
+
+If the contents of directories and files are still available, the better option
+is to run "backup" which in that case is able to heal existing snapshots. Only
+use the "repair snapshots" command if you need to recover an old and broken
+snapshot!
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runRepairSnapshots(cmd.Context(), globalOptions, repairSnapshotOptions, args)
+	},
+}
+
+// RepairOptions collects all options for the repair command.
+type RepairOptions struct {
+	DryRun bool
+	Forget bool
+
+	restic.SnapshotFilter
+}
+
+var repairSnapshotOptions RepairOptions
+
+func init() {
+	cmdRepair.AddCommand(cmdRepairSnapshots)
+	flags := cmdRepairSnapshots.Flags()
+
+	flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
+	flags.BoolVarP(&repairSnapshotOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
+
+	initMultiSnapshotFilter(flags, &repairSnapshotOptions.SnapshotFilter, true)
+}
+
+func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error {
+	repo, err := OpenRepository(ctx, globalOptions)
+	if err != nil {
+		return err
+	}
+
+	if !opts.DryRun {
+		var lock *restic.Lock
+		var err error
+		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	} else {
+		repo.SetDryRun()
+	}
+
+	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
+		return err
+	}
+
+	if err := repo.LoadIndex(ctx); err != nil {
+		return err
+	}
+
+	// Three error cases are checked:
+	// - tree is a nil tree (-> will be replaced by an empty tree)
+	// - trees which cannot be loaded (-> the tree contents will be removed)
+	// - files whose contents are not fully available  (-> file will be modified)
+	rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
+		RewriteNode: func(node *restic.Node, path string) *restic.Node {
+			if node.Type != "file" {
+				return node
+			}
+
+			ok := true
+			var newContent restic.IDs = restic.IDs{}
+			var newSize uint64
+			// check all contents and remove if not available
+			for _, id := range node.Content {
+				if size, found := repo.LookupBlobSize(id, restic.DataBlob); !found {
+					ok = false
+				} else {
+					newContent = append(newContent, id)
+					newSize += uint64(size)
+				}
+			}
+			if !ok {
+				Verbosef("  file %q: removed missing content\n", path)
+			} else if newSize != node.Size {
+				Verbosef("  file %q: fixed incorrect size\n", path)
+			}
+			// no-ops if already correct
+			node.Content = newContent
+			node.Size = newSize
+			return node
+		},
+		RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) {
+			if path == "/" {
+				Verbosef("  dir %q: not readable\n", path)
+				// remove snapshots with invalid root node
+				return restic.ID{}, nil
+			}
+			// If a subtree fails to load, remove it
+			Verbosef("  dir %q: replaced with empty directory\n", path)
+			emptyID, err := restic.SaveTree(ctx, repo, &restic.Tree{})
+			if err != nil {
+				return restic.ID{}, err
+			}
+			return emptyID, nil
+		},
+		AllowUnstableSerialization: true,
+	})
+
+	changedCount := 0
+	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
+		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+		changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
+			func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
+				return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
+			}, opts.DryRun, opts.Forget, "repaired")
+		if err != nil {
+			return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
+		}
+		if changed {
+			changedCount++
+		}
+	}
+
+	Verbosef("\n")
+	if changedCount == 0 {
+		if !opts.DryRun {
+			Verbosef("no snapshots were modified\n")
+		} else {
+			Verbosef("no snapshots would be modified\n")
+		}
+	} else {
+		if !opts.DryRun {
+			Verbosef("modified %v snapshots\n", changedCount)
+		} else {
+			Verbosef("would modify %v snapshots\n", changedCount)
+		}
+	}
+
+	return nil
+}