Merge branch 'master' into prune-repack-threshold

.gitignore

@@ -1,3 +1,2 @@
 /restic
 /.vagrant
-/doc/_build

CHANGELOG.md

@@ -1,3 +1,161 @@
+Changelog for restic 0.9.3 (2018-10-13)
+=======================================
+
+The following sections list the changes in restic 0.9.3 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #1935: Remove truncated files from cache
+ * Fix #1978: Do not return an error when the scanner is faster than backup
+ * Enh #1766: Restore: suppress lchown errors when not running as root
+ * Enh #1909: Reject files/dirs by name first
+ * Enh #1940: Add directory filter to ls command
+ * Enh #1967: Use `--host` everywhere
+ * Enh #2028: Display size of cache directories
+ * Enh #1777: Improve the `find` command
+ * Enh #1876: Display reason why forget keeps snapshots
+ * Enh #1891: Accept glob in paths loaded via --files-from
+ * Enh #1920: Vendor dependencies with Go 1.11 Modules
+ * Enh #1949: Add new command `self-update`
+ * Enh #1953: Ls: Add JSON output support for restic ls cmd
+
+Details
+-------
+
+ * Bugfix #1935: Remove truncated files from cache
+
+   When a file in the local cache is truncated, and restic tries to access data beyond the end of the
+   (cached) file, it used to return an error "EOF". This is now fixed, such truncated files are
+   removed and the data is fetched directly from the backend.
+
+   https://github.com/restic/restic/issues/1935
+
+ * Bugfix #1978: Do not return an error when the scanner is faster than backup
+
+   When restic makes a backup, there's a background task called "scanner" which collects
+   information on how many files and directories are to be saved, in order to display progress
+   information to the user. When the backup finishes faster than the scanner, it is aborted
+   because the result is not needed any more. This logic contained a bug, where quitting the
+   scanner process was treated as an error, and caused restic to print an unhelpful error message
+   ("context canceled").
+
+   https://github.com/restic/restic/issues/1978
+   https://github.com/restic/restic/pull/1991
+
+ * Enhancement #1766: Restore: suppress lchown errors when not running as root
+
+   Like "cp" and "rsync" do, restic now only reports errors for changing the ownership of files
+   during restore if it is run as root, on non-Windows operating systems. On Windows, the error
+   is reported as usual.
+
+   https://github.com/restic/restic/issues/1766
+
+ * Enhancement #1909: Reject files/dirs by name first
+
+   The current scanner/archiver code had an architectural limitation: it always ran the
+   `lstat()` system call on all files and directories before a decision to include/exclude the
+   file/dir was made. This lead to a lot of unnecessary system calls for items that could have been
+   rejected by their name or path only.
+
+   We've changed the archiver/scanner implementation so that it now first rejects by name/path,
+   and only runs the system call on the remaining items. This reduces the number of `lstat()`
+   system calls a lot (depending on the exclude settings).
+
+   https://github.com/restic/restic/issues/1909
+   https://github.com/restic/restic/pull/1912
+
+ * Enhancement #1940: Add directory filter to ls command
+
+   The ls command can now be filtered by directories, so that only files in the given directories
+   will be shown. If the --recursive flag is specified, then ls will traverse subfolders and list
+   their files as well.
+
+   It used to be possible to specify multiple snapshots, but that has been replaced by only one
+   snapshot and the possibility of specifying multiple directories.
+
+   Specifying directories constrains the walk, which can significantly speed up the listing.
+
+   https://github.com/restic/restic/issues/1940
+   https://github.com/restic/restic/pull/1941
+
+ * Enhancement #1967: Use `--host` everywhere
+
+   We now use the flag `--host` for all commands which need a host name, using `--hostname` (e.g.
+   for `restic backup`) still works, but will print a deprecation warning. Also, add the short
+   option `-H` where possible.
+
+   https://github.com/restic/restic/issues/1967
+
+ * Enhancement #2028: Display size of cache directories
+
+   The `cache` command now by default shows the size of the individual cache directories. It can be
+   disabled with `--no-size`.
+
+   https://github.com/restic/restic/issues/2028
+   https://github.com/restic/restic/pull/2033
+
+ * Enhancement #1777: Improve the `find` command
+
+   We've updated the `find` command to support multiple patterns.
+
+   `restic find` is now able to list the snapshots containing a specific tree or blob, or even the
+   snapshots that contain blobs belonging to a given pack. A list of IDs can be given, as long as they
+   all have the same type.
+
+   The command `find` can also display the pack IDs the blobs belong to, if the `--show-pack-id`
+   flag is provided.
+
+   https://github.com/restic/restic/issues/1777
+   https://github.com/restic/restic/pull/1780
+
+ * Enhancement #1876: Display reason why forget keeps snapshots
+
+   We've added a column to the list of snapshots `forget` keeps which details the reasons to keep a
+   particuliar snapshot. This makes debugging policies for forget much easier. Please remember
+   to always try things out with `--dry-run`!
+
+   https://github.com/restic/restic/pull/1876
+
+ * Enhancement #1891: Accept glob in paths loaded via --files-from
+
+   Before that, behaviour was different if paths were appended to command line or from a file,
+   because wild card characters were expanded by shell if appended to command line, but not
+   expanded if loaded from file.
+
+   https://github.com/restic/restic/issues/1891
+
+ * Enhancement #1920: Vendor dependencies with Go 1.11 Modules
+
+   Until now, we've used `dep` for managing dependencies, we've now switch to using Go modules.
+   For users this does not change much, only if you want to compile restic without downloading
+   anything with Go 1.11, then you need to run: `go build -mod=vendor build.go`
+
+   https://github.com/restic/restic/pull/1920
+
+ * Enhancement #1949: Add new command `self-update`
+
+   We have added a new command called `self-update` which downloads the latest released version
+   of restic from GitHub and replaces the current binary with it. It does not rely on any external
+   program (so it'll work everywhere), but still verifies the GPG signature using the embedded
+   GPG public key.
+
+   By default, the `self-update` command is hidden behind the `selfupdate` built tag, which is
+   only set when restic is built using `build.go` (including official releases). The reason for
+   this is that downstream distributions will then not include the command by default, so users
+   are encouraged to use the platform-specific distribution mechanism.
+
+   https://github.com/restic/restic/pull/1949
+
+ * Enhancement #1953: Ls: Add JSON output support for restic ls cmd
+
+   We've implemented listing files in the repository with JSON as output, just pass `--json` as an
+   option to `restic ls`. This makes the output of the command machine readable.
+
+   https://github.com/restic/restic/pull/1953
+
+
 Changelog for restic 0.9.2 (2018-08-06)
 =======================================
 

CONTRIBUTING.md

@@ -174,7 +174,7 @@ locally as a pre-commit hook checks formatting before committing automatically,
 just copy this script to `.git/hooks/pre-commit`.
 
 For each pull request, several different systems run the integration tests on
-Linux, OS X and Windows. We won't merge any code that does not pass all tests
+Linux, macOS and Windows. We won't merge any code that does not pass all tests
 for all systems, so when a tests fails, try to find out what's wrong and fix
 it. If you need help on this, please leave a comment in the pull request, and
 we'll be glad to assist. Having a PR with failing integration tests is nothing

VERSION

@@ -1 +1 @@
-0.9.2
+0.9.3

build.go

@@ -525,11 +525,6 @@ func main() {
 
 	verbosePrintf("detected Go version %v\n", goVersion)
 
-	if len(buildTags) == 0 {
-		verbosePrintf("adding build-tag release\n")
-		buildTags = []string{"release"}
-	}
-
 	for i := range buildTags {
 		buildTags[i] = strings.TrimSpace(buildTags[i])
 	}

changelog/0.9.3_2018-10-13/issue-1967

@@ -0,0 +1,7 @@
+Enhancement: Use `--host` everywhere
+
+We now use the flag `--host` for all commands which need a host name, using
+`--hostname` (e.g. for `restic backup`) still works, but will print a
+deprecation warning. Also, add the short option `-H` where possible.
+
+https://github.com/restic/restic/issues/1967

changelog/0.9.3_2018-10-13/issue-2028

@@ -0,0 +1,7 @@
+Enhancement: Display size of cache directories
+
+The `cache` command now by default shows the size of the individual cache
+directories. It can be disabled with `--no-size`.
+
+https://github.com/restic/restic/issues/2028
+https://github.com/restic/restic/pull/2033

changelog/0.9.3_2018-10-13/pull-1891

@@ -0,0 +1,7 @@
+Enhancement: Accept glob in paths loaded via --files-from
+
+Before that, behaviour was different if paths were appended to command line or
+from a file, because wild card characters were expanded by shell if appended to
+command line, but not expanded if loaded from file.
+
+https://github.com/restic/restic/issues/1891

changelog/0.9.3_2018-10-13/pull-1962

@@ -0,0 +1,13 @@
+Enhancement: Stream JSON output for ls command
+
+The `ls` command now supports JSON output with the global `--json`
+flag, and this change streams out JSON messages one object at a time
+rather than en entire array buffered in memory before encoding. The
+advantage is it allows large listings to be handled efficiently.
+
+Two message types are printed: snapshots and nodes. A snapshot
+object will precede node objects which belong to that snapshot.
+The `struct_type` field can be used to determine which kind of
+message an object is.
+
+https://github.com/restic/restic/pull/1962

changelog/unreleased/issue-1605

@@ -0,0 +1,11 @@
+Enhancement: Concurrent restore
+
+This change significantly improves restore performance, especially
+when using high-latency remote repositories like B2.
+
+The implementation now uses several concurrent threads to download and process
+multiple remote files concurrently. To further reduce restore time, each remote
+file is downloaded using a single repository request.
+
+https://github.com/restic/restic/issues/1605
+https://github.com/restic/restic/pull/1719

cmd/restic/cmd_backup.go

@@ -4,8 +4,10 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -32,13 +34,13 @@ The "backup" command creates a new snapshot and saves the files and directories
 given as the arguments.
 `,
 	PreRun: func(cmd *cobra.Command, args []string) {
-		if backupOptions.Hostname == "" {
+		if backupOptions.Host == "" {
 			hostname, err := os.Hostname()
 			if err != nil {
 				debug.Log("os.Hostname() returned err: %v", err)
 				return
 			}
-			backupOptions.Hostname = hostname
+			backupOptions.Host = hostname
 		}
 	},
 	DisableAutoGenTag: true,
@@ -72,7 +74,7 @@ type BackupOptions struct {
 	Stdin            bool
 	StdinFilename    string
 	Tags             []string
-	Hostname         string
+	Host             string
 	FilesFrom        string
 	TimeStamp        string
 	WithAtime        bool
@@ -94,7 +96,11 @@ func init() {
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
 	f.StringArrayVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
-	f.StringVar(&backupOptions.Hostname, "hostname", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
+
+	f.StringVar(&backupOptions.Host, "host", "H", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
+	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
+	f.MarkDeprecated("hostname", "use --host")
+
 	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
 	f.StringVar(&backupOptions.TimeStamp, "time", "", "time of the backup (ex. '2012-11-01 22:08:41') (default: now)")
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
@@ -301,10 +307,21 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 		return nil, err
 	}
 
+	// expand wildcards
+	var lines []string
+	for _, line := range fromfile {
+		var expanded []string
+		expanded, err := filepath.Glob(line)
+		if err != nil {
+			return nil, errors.WithMessage(err, fmt.Sprintf("pattern: %s", line))
+		}
+		lines = append(lines, expanded...)
+	}
+
 	// merge files from files-from into normal args so we can reuse the normal
 	// args checks and have the ability to use both files-from and args at the
 	// same time
-	args = append(args, fromfile...)
+	args = append(args, lines...)
 	if len(args) == 0 && !opts.Stdin {
 		return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
 	}
@@ -333,7 +350,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, opts.Hostname)
+		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, opts.Host)
 		if err == nil {
 			parentID = &id
 		} else if err != restic.ErrNoSnapshotFound {
@@ -482,7 +499,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		Excludes:       opts.Excludes,
 		Tags:           opts.Tags,
 		Time:           timeStamp,
-		Hostname:       opts.Hostname,
+		Hostname:       opts.Host,
 		ParentSnapshot: *parentSnapshotID,
 	}