[pull] master from restic:master

.github/workflows/tests.yml

@@ -9,7 +9,7 @@ on:
   pull_request:
 
 env:
-  latest_go: "1.18.x"
+  latest_go: "1.19.x"
   GO111MODULE: on
 
 jobs:
@@ -19,24 +19,30 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.18.x
+            go: 1.19.x
             os: windows-latest
             install_verb: install
 
           - job_name: macOS
-            go: 1.18.x
+            go: 1.19.x
             os: macOS-latest
             test_fuse: false
             install_verb: install
 
           - job_name: Linux
-            go: 1.18.x
+            go: 1.19.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
             install_verb: install
 
+          - job_name: Linux
+            go: 1.18.x
+            os: ubuntu-latest
+            test_fuse: true
+            install_verb: install
+
           - job_name: Linux
             go: 1.17.x
             os: ubuntu-latest
@@ -260,7 +266,7 @@ jobs:
         uses: golangci/golangci-lint-action@v2
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.45
+          version: v1.48
           # Optional: show only new issues if it's a pull request. The default value is `false`.
           only-new-issues: true
           args: --verbose --timeout 5m

CONTRIBUTING.md

@@ -48,9 +48,8 @@ environment was used and so on. Please tell us at least the following things:
 Remember, the easier it is for us to reproduce the bug, the earlier it will be
 corrected!
 
-In addition, you can compile restic with debug support by running
-`go run build.go -tags debug` and instructing it to create a debug
-log by setting the environment variable `DEBUG_LOG` to a file, e.g. like this:
+In addition, you can instruct restic to create a debug log by setting the
+environment variable `DEBUG_LOG` to a file, e.g. like this:
 
     $ export DEBUG_LOG=/tmp/restic-debug.log
     $ restic backup ~/work

changelog/unreleased/issue-1842

@@ -0,0 +1,8 @@
+Change: Support debug log creation in release builds
+
+Creating a debug log was only possible in debug builds which required users to
+manually build restic. We changed the release builds to allow creating debug
+logs by setting the environment variable `DEBUG_LOG=logname.log`.
+
+https://github.com/restic/restic/issues/1842
+https://github.com/restic/restic/pull/3826

changelog/unreleased/issue-2291

@@ -0,0 +1,12 @@
+Enhancement: Allow pack size customization
+
+Restic now uses a target pack size of 16 MiB by default. It can be customized
+using the `--pack-size size` option. Supported pack sizes range between 4 and
+128 MiB.
+
+It is possible to migrate an existing repository to _larger_ pack files using
+`prune --repack-small`. This will rewrite every pack file which is
+significantly smaller than the target size.
+
+https://github.com/restic/restic/issues/2291
+https://github.com/restic/restic/pull/3731

changelog/unreleased/issue-2907

@@ -0,0 +1,10 @@
+Enhancement: Make snapshot directory structure of mount command custimizable
+
+We've added the possibility to customize the snapshot directory structure of
+the mount command using templates passed to `--snapshot-template`. The
+formatting of the time for a snapshot is now controlled using `--time-template`
+and supports subdirectories to for example group snapshots by year. Please
+refer to the help output of the `mount` command for further details.
+
+https://github.com/restic/restic/issues/2907
+https://github.com/restic/restic/pull/2913

cmd/restic/cmd_init.go

@@ -86,7 +86,13 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
 		return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
 	}
 
-	s := repository.New(be, repository.Options{Compression: gopts.Compression})
+	s, err := repository.New(be, repository.Options{
+		Compression: gopts.Compression,
+		PackSize:    gopts.PackSize * 1024 * 1024,
+	})
+	if err != nil {
+		return err
+	}
 
 	err = s.Init(gopts.ctx, version, gopts.password, chunkerPolynomial)
 	if err != nil {

cmd/restic/cmd_mount.go

@@ -31,10 +31,13 @@ read-only mount.
 Snapshot Directories
 ====================
 
-If you need a different template for all directories that contain snapshots,
-you can pass a template via --snapshot-template. Example without colons:
+If you need a different template for directories that contain snapshots,
+you can pass a time template via --time-template and path templates via
+--path-template.
 
-    --snapshot-template "2006-01-02_15-04-05"
+Example time template without colons:
+
+    --time-template "2006-01-02_15-04-05"
 
 You need to specify a sample format for exactly the following timestamp:
 
@@ -43,6 +46,20 @@ You need to specify a sample format for exactly the following timestamp:
 For details please see the documentation for time.Format() at:
   https://godoc.org/time#Time.Format
 
+For path templates, you can use the following patterns which will be replaced:
+    %i by short snapshot ID
+    %I by long snapshot ID
+    %u by username
+    %h by hostname
+    %t by tags
+    %T by timestamp as specified by --time-template
+
+The default path templates are:
+    "ids/%i"
+    "snapshots/%T"
+    "hosts/%h/%T"
+    "tags/%t/%T"
+
 EXIT STATUS
 ===========
 
@@ -62,7 +79,8 @@ type MountOptions struct {
 	Hosts                []string
 	Tags                 restic.TagLists
 	Paths                []string
-	SnapshotTemplate     string
+	TimeTemplate         string
+	PathTemplates        []string
 }
 
 var mountOptions MountOptions
@@ -79,16 +97,21 @@ func init() {
 	mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
 	mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
 
-	mountFlags.StringVar(&mountOptions.SnapshotTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
+	mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)")
+	mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
+	mountFlags.StringVar(&mountOptions.TimeTemplate, "time-template", time.RFC3339, "set `template` to use for times")
+	_ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template")
 }
 
 func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
-	if opts.SnapshotTemplate == "" {
-		return errors.Fatal("snapshot template string cannot be empty")
+	if opts.TimeTemplate == "" {
+		return errors.Fatal("time template string cannot be empty")
 	}
-	if strings.ContainsAny(opts.SnapshotTemplate, `\/`) {
-		return errors.Fatal("snapshot template string contains a slash (/) or backslash (\\) character")
+
+	if strings.HasPrefix(opts.TimeTemplate, "/") || strings.HasSuffix(opts.TimeTemplate, "/") {
+		return errors.Fatal("time template string cannot start or end with '/'")
 	}
+
 	if len(args) == 0 {
 		return errors.Fatal("wrong number of parameters")
 	}
@@ -154,11 +177,12 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
 	}
 
 	cfg := fuse.Config{
-		OwnerIsRoot:      opts.OwnerRoot,
-		Hosts:            opts.Hosts,
-		Tags:             opts.Tags,
-		Paths:            opts.Paths,
-		SnapshotTemplate: opts.SnapshotTemplate,
+		OwnerIsRoot:   opts.OwnerRoot,
+		Hosts:         opts.Hosts,
+		Tags:          opts.Tags,
+		Paths:         opts.Paths,
+		TimeTemplate:  opts.TimeTemplate,
+		PathTemplates: opts.PathTemplates,
 	}
 	root := fuse.NewRoot(repo, cfg)
 

cmd/restic/cmd_prune.go

@@ -52,6 +52,7 @@ type PruneOptions struct {
 	MaxRepackBytes uint64
 
 	RepackCachableOnly bool
+	RepackSmall        bool
 	RepackUncompressed bool
 }
 
@@ -70,6 +71,7 @@ func addPruneOptions(c *cobra.Command) {
 	f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
 	f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
 	f.BoolVar(&pruneOptions.RepackCachableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable")
+	f.BoolVar(&pruneOptions.RepackSmall, "repack-small", false, "repack pack files below 80%% of target pack size")
 	f.BoolVar(&pruneOptions.RepackUncompressed, "repack-uncompressed", false, "repack all uncompressed data")
 }
 
@@ -422,7 +424,14 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 	repackPacks := restic.NewIDSet()
 
 	var repackCandidates []packInfoWithID
+	var repackSmallCandidates []packInfoWithID
 	repoVersion := repo.Config().Version
+	// only repack very small files by default
+	targetPackSize := repo.PackSize() / 25
+	if opts.RepackSmall {
+		// consider files with at least 80% of the target size as large enough
+		targetPackSize = repo.PackSize() / 5 * 4
+	}
 
 	// loop over all packs and decide what to do
 	bar := newProgressMax(!gopts.Quiet, uint64(len(indexPack)), "packs processed")
@@ -477,8 +486,12 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 			stats.packs.keep++
 
 		case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress:
-			// All blobs in pack are used and not mixed => keep pack!
-			stats.packs.keep++
+			if packSize >= int64(targetPackSize) {
+				// All blobs in pack are used and not mixed => keep pack!
+				stats.packs.keep++
+			} else {
+				repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p})
+			}
 
 		default:
 			// all other packs are candidates for repacking
@@ -521,11 +534,19 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 		}
 	}
 
+	if len(repackSmallCandidates) < 10 {
+		// too few small files to be worth the trouble, this also prevents endlessly repacking
+		// if there is just a single pack file below the target size
+		stats.packs.keep += uint(len(repackSmallCandidates))
+	} else {
+		repackCandidates = append(repackCandidates, repackSmallCandidates...)
+	}
+
 	// Sort repackCandidates such that packs with highest ratio unused/used space are picked first.
 	// This is equivalent to sorting by unused / total space.
 	// Instead of unused[i] / used[i] > unused[j] / used[j] we use
 	// unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64
-	// Morover packs containing trees are sorted to the beginning
+	// Moreover packs containing trees and too small packs are sorted to the beginning
 	sort.Slice(repackCandidates, func(i, j int) bool {
 		pi := repackCandidates[i].packInfo
 		pj := repackCandidates[j].packInfo
@@ -534,6 +555,10 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 			return true
 		case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
 			return false
+		case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
+			return true
+		case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
+			return false
 		}
 		return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
 	})
@@ -552,6 +577,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 	for _, p := range repackCandidates {
 		reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
 		reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
+		packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize)
 
 		switch {
 		case reachedRepackSize:
@@ -561,7 +587,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
 			// repacking non-data packs / uncompressed-trees is only limited by repackSize
 			repack(p.ID, p.packInfo)
 
-		case reachedUnusedSizeAfter:
+		case reachedUnusedSizeAfter && packIsLargeEnough:
 			// for all other packs stop repacking if tolerated unused size is reached.
 			stats.packs.keep++
 

cmd/restic/global.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -62,6 +63,7 @@ type GlobalOptions struct {
 	NoCache         bool
 	CleanupCache    bool
 	Compression     repository.CompressionMode
+	PackSize        uint
 
 	backend.TransportOptions
 	limiter.Limits
@@ -102,6 +104,9 @@ func init() {
 		return nil
 	})
 
+	// parse target pack size from env, on error the default value will be used
+	targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
+
 	f := cmdRoot.PersistentFlags()
 	f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
 	f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", os.Getenv("RESTIC_REPOSITORY_FILE"), "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)")
@@ -121,6 +126,7 @@ func init() {
 	f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)")
 	f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
 	f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
+	f.UintVar(&globalOptions.PackSize, "pack-size", uint(targetPackSize), "set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
 	f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
 	// Use our "generate" command instead of the cobra provided "completion" command
 	cmdRoot.CompletionOptions.DisableDefaultCmd = true
@@ -440,7 +446,13 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 		}
 	}
 
-	s := repository.New(be, repository.Options{Compression: opts.Compression})
+	s, err := repository.New(be, repository.Options{
+		Compression: opts.Compression,
+		PackSize:    opts.PackSize * 1024 * 1024,
+	})
+	if err != nil {
+		return nil, err
+	}
 
 	passwordTriesLeft := 1
 	if stdinIsTerminal() && opts.password == "" {

cmd/restic/integration_fuse_test.go

@@ -55,7 +55,7 @@ func waitForMount(t testing.TB, dir string) {
 
 func testRunMount(t testing.TB, gopts GlobalOptions, dir string) {
 	opts := MountOptions{
-		SnapshotTemplate: time.RFC3339,
+		TimeTemplate: time.RFC3339,
 	}
 	rtest.OK(t, runMount(opts, gopts, []string{dir}))
 }

cmd/restic/integration_test.go

@@ -1611,6 +1611,11 @@ func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) {
 		checkOpts := CheckOptions{ReadData: true}
 		testPrune(t, opts, checkOpts)
 	})
+	t.Run("Small", func(t *testing.T) {
+		opts := PruneOptions{MaxUnused: "unlimited", RepackSmall: true}
+		checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
+		testPrune(t, opts, checkOpts)
+	})
 }
 
 func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {

doc/040_backup.rst

@@ -554,6 +554,7 @@ environment variables. The following lists these environment variables:
     RESTIC_CACHE_DIR                    Location of the cache directory
     RESTIC_COMPRESSION                  Compression mode (only available for repository format version 2)
     RESTIC_PROGRESS_FPS                 Frames per second by which the progress bar is updated
+    RESTIC_PACK_SIZE                    Target size for pack files
 
     TMPDIR                              Location for temporary files