Merge pull request restic#2581 from aawsome/multiple-hostnames

Allow multiple hostnames tags

changelog/unreleased/issue-1570

@@ -0,0 +1,9 @@
+Enhancement: Support specifying multiple host flags for various commands
+
+Previously commands didn't take more than one `--host` or `-H` argument into account, which could be limiting with e.g.
+the `forget` command.
+
+The `dump`, `find`, `forget`, `ls`, `mount`, `restore`, `snapshots`, `stats` and `tag` commands will now take into account
+multiple `--host` and `-H` flags.
+
+https://github.com/restic/restic/issues/1570

cmd/restic/cmd_backup.go

@@ -382,7 +382,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 
 	// Find last snapshot to set it as parent, if not already set
 	if !opts.Force && parentID == nil {
-		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, opts.Host)
+		id, err := restic.FindLatestSnapshot(ctx, repo, targets, []restic.TagList{}, []string{opts.Host})
 		if err == nil {
 			parentID = &id
 		} else if err != restic.ErrNoSnapshotFound {

cmd/restic/cmd_dump.go

@@ -41,7 +41,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // DumpOptions collects all options for the dump command.
 type DumpOptions struct {
-	Host  string
+	Hosts []string
 	Paths []string
 	Tags  restic.TagLists
 }
@@ -52,7 +52,7 @@ func init() {
 	cmdRoot.AddCommand(cmdDump)
 
 	flags := cmdDump.Flags()
-	flags.StringVarP(&dumpOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
+	flags.StringArrayVarP(&dumpOptions.Hosts, "host", "H", nil, `only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)`)
 	flags.Var(&dumpOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
 	flags.StringArrayVar(&dumpOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
 }
@@ -141,9 +141,9 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Host)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
 		if err != nil {
-			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, opts.Paths, opts.Host)
+			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
 	} else {
 		id, err = restic.FindSnapshot(repo, snapshotIDString)

cmd/restic/cmd_find.go

@@ -51,7 +51,7 @@ type FindOptions struct {
 	PackID, ShowPackID bool
 	CaseInsensitive    bool
 	ListLong           bool
-	Host               string
+	Hosts              []string
 	Paths              []string
 	Tags               restic.TagLists
 }
@@ -72,7 +72,7 @@ func init() {
 	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
 	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 
-	f.StringVarP(&findOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	f.StringArrayVarP(&findOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
 	f.Var(&findOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
 	f.StringArrayVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
 }
@@ -567,7 +567,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
 		f.packsToBlobs(ctx, []string{f.pat.pattern[0]}) // TODO: support multiple packs
 	}
 
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) {
 		if f.blobIDs != nil || f.treeIDs != nil {
 			if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" {
 				return err

cmd/restic/cmd_forget.go

@@ -40,7 +40,7 @@ type ForgetOptions struct {
 	Within   restic.Duration
 	KeepTags restic.TagLists
 
-	Host    string
+	Hosts   []string
 	Tags    restic.TagLists
 	Paths   []string
 	Compact bool
@@ -66,8 +66,8 @@ func init() {
 	f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
 
 	f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
-	f.StringVar(&forgetOptions.Host, "host", "", "only consider snapshots with the given `host`")
-	f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname`")
+	f.StringArrayVar(&forgetOptions.Hosts, "host", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
+	f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
 	f.MarkDeprecated("hostname", "use --host")
 
 	f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
@@ -101,7 +101,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 
 	var snapshots restic.Snapshots
 
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		snapshots = append(snapshots, sn)
 	}
 

cmd/restic/cmd_ls.go

@@ -48,7 +48,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 // LsOptions collects all options for the ls command.
 type LsOptions struct {
 	ListLong  bool
-	Host      string
+	Hosts     []string
 	Tags      restic.TagLists
 	Paths     []string
 	Recursive bool
@@ -61,7 +61,7 @@ func init() {
 
 	flags := cmdLs.Flags()
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
-	flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
 	flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
 	flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
@@ -89,7 +89,7 @@ type lsNode struct {
 }
 
 func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
-	if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
+	if len(args) == 0 && len(opts.Hosts) == 0 && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
 		return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
 	}
 
@@ -191,7 +191,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args[:1]) {
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args[:1]) {
 		printSnapshot(sn)
 
 		err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {

cmd/restic/cmd_mount.go

@@ -62,7 +62,7 @@ type MountOptions struct {
 	AllowRoot            bool
 	AllowOther           bool
 	NoDefaultPermissions bool
-	Host                 string
+	Hosts                []string
 	Tags                 restic.TagLists
 	Paths                []string
 	SnapshotTemplate     string
@@ -79,7 +79,7 @@ func init() {
 	mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
 	mountFlags.BoolVar(&mountOptions.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files")
 
-	mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`)
+	mountFlags.StringArrayVarP(&mountOptions.Hosts, "host", "H", nil, `only consider snapshots for this host (can be specified multiple times)`)
 	mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
 	mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
 
@@ -143,7 +143,7 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 
 	cfg := fuse.Config{
 		OwnerIsRoot:      opts.OwnerRoot,
-		Host:             opts.Host,
+		Hosts:            opts.Hosts,
 		Tags:             opts.Tags,
 		Paths:            opts.Paths,
 		SnapshotTemplate: opts.SnapshotTemplate,

cmd/restic/cmd_restore.go

@@ -1,12 +1,13 @@
 package main
 
 import (
+	"strings"
+
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/restorer"
-	"strings"
 
 	"github.com/spf13/cobra"
 )
@@ -39,7 +40,7 @@ type RestoreOptions struct {
 	Include            []string
 	InsensitiveInclude []string
 	Target             string
-	Host               string
+	Hosts              []string
 	Paths              []string
 	Tags               restic.TagLists
 	Verify             bool
@@ -57,7 +58,7 @@ func init() {
 	flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames")
 	flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
 
-	flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
+	flags.StringArrayVarP(&restoreOptions.Hosts, "host", "H", nil, `only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)`)
 	flags.Var(&restoreOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
 	flags.StringArrayVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
 	flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
@@ -116,9 +117,9 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
 	var id restic.ID
 
 	if snapshotIDString == "latest" {
-		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Host)
+		id, err = restic.FindLatestSnapshot(ctx, repo, opts.Paths, opts.Tags, opts.Hosts)
 		if err != nil {
-			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, opts.Paths, opts.Host)
+			Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
 		}
 	} else {
 		id, err = restic.FindSnapshot(repo, snapshotIDString)

cmd/restic/cmd_snapshots.go

@@ -32,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // SnapshotOptions bundles all options for the snapshots command.
 type SnapshotOptions struct {
-	Host    string
+	Hosts   []string
 	Tags    restic.TagLists
 	Paths   []string
 	Compact bool
@@ -46,7 +46,7 @@ func init() {
 	cmdRoot.AddCommand(cmdSnapshots)
 
 	f := cmdSnapshots.Flags()
-	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
+	f.StringArrayVarP(&snapshotOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host` (can be specified multiple times)")
 	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` (can be specified multiple times)")
 	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
 	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
@@ -72,7 +72,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	defer cancel()
 
 	var snapshots restic.Snapshots
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		snapshots = append(snapshots, sn)
 	}
 	snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)

cmd/restic/cmd_stats.go

@@ -54,7 +54,7 @@ func init() {
 	cmdRoot.AddCommand(cmdStats)
 	f := cmdStats.Flags()
 	f.StringVar(&countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file, or raw-data")
-	f.StringVarP(&snapshotByHost, "host", "H", "", "filter latest snapshot by this hostname")
+	f.StringArrayVarP(&snapshotByHosts, "host", "H", nil, "filter latest snapshot by this hostname (can be specified multiple times)")
 }
 
 func runStats(gopts GlobalOptions, args []string) error {
@@ -101,7 +101,7 @@ func runStats(gopts GlobalOptions, args []string) error {
 
 		var sID restic.ID
 		if snapshotIDString == "latest" {
-			sID, err = restic.FindLatestSnapshot(ctx, repo, []string{}, []restic.TagList{}, snapshotByHost)
+			sID, err = restic.FindLatestSnapshot(ctx, repo, []string{}, []restic.TagList{}, snapshotByHosts)
 			if err != nil {
 				return errors.Fatalf("latest snapshot for criteria not found: %v", err)
 			}
@@ -335,7 +335,7 @@ var (
 
 	// snapshotByHost is the host to filter latest
 	// snapshot by, if given by user
-	snapshotByHost string
+	snapshotByHosts []string
 )
 
 const (

cmd/restic/cmd_tag.go

@@ -35,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // TagOptions bundles all options for the 'tag' command.
 type TagOptions struct {
-	Host       string
+	Hosts      []string
 	Paths      []string
 	Tags       restic.TagLists
 	SetTags    []string
@@ -53,7 +53,7 @@ func init() {
 	tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)")
 	tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
 
-	tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+	tagFlags.StringArrayVarP(&tagOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
 	tagFlags.Var(&tagOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
 	tagFlags.StringArrayVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
 }
@@ -129,7 +129,7 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
 	changeCnt := 0
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		changed, err := changeTags(ctx, repo, sn, opts.SetTags, opts.AddTags, opts.RemoveTags)
 		if err != nil {
 			Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)

cmd/restic/find.go

@@ -8,7 +8,7 @@ import (
 )
 
 // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
-func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
+func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hosts []string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
 	out := make(chan *restic.Snapshot)
 	go func() {
 		defer close(out)
@@ -22,9 +22,9 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			// Process all snapshot IDs given as arguments.
 			for _, s := range snapshotIDs {
 				if s == "latest" {
-					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, host)
+					id, err = restic.FindLatestSnapshot(ctx, repo, paths, tags, hosts)
 					if err != nil {
-						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Host:%v)\n", s, paths, tags, host)
+						Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)\n", s, paths, tags, hosts)
 						usedFilter = true
 						continue
 					}
@@ -39,7 +39,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			}
 
 			// Give the user some indication their filters are not used.
-			if !usedFilter && (host != "" || len(tags) != 0 || len(paths) != 0) {
+			if !usedFilter && (len(hosts) != 0 || len(tags) != 0 || len(paths) != 0) {
 				Warnf("Ignoring filters as there are explicit snapshot ids given\n")
 			}
 
@@ -58,7 +58,7 @@ func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, hos
 			return
 		}
 
-		snapshots, err := restic.FindFilteredSnapshots(ctx, repo, host, tags, paths)
+		snapshots, err := restic.FindFilteredSnapshots(ctx, repo, hosts, tags, paths)
 		if err != nil {
 			Warnf("could not load snapshots: %v\n", err)
 			return

cmd/restic/integration_test.go

@@ -94,10 +94,10 @@ func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID res
 	testRunRestoreExcludes(t, opts, dir, snapshotID, nil)
 }
 
-func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, host string) {
+func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) {
 	opts := RestoreOptions{
 		Target: dir,
-		Host:   host,
+		Hosts:  hosts,
 		Paths:  paths,
 	}
 
@@ -765,7 +765,7 @@ func TestRestore(t *testing.T) {
 
 	// Restore latest without any filters
 	restoredir := filepath.Join(env.base, "restore")
-	testRunRestoreLatest(t, env.gopts, restoredir, nil, "")
+	testRunRestoreLatest(t, env.gopts, restoredir, nil, nil)
 
 	rtest.Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))),
 		"directories are not equal")
@@ -802,7 +802,7 @@ func TestRestoreLatest(t *testing.T) {
 	testRunCheck(t, env.gopts)
 
 	// Restore latest without any filters
-	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore0"), nil, "")
+	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore0"), nil, nil)
 	rtest.OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", "testfile.c"), int64(101)))
 
 	// Setup test files in different directories backed up in different snapshots
@@ -823,14 +823,14 @@ func TestRestoreLatest(t *testing.T) {
 	p1rAbs := filepath.Join(env.base, "restore1", "p1/testfile.c")
 	p2rAbs := filepath.Join(env.base, "restore2", "p2/testfile.c")
 
-	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore1"), []string{filepath.Dir(p1)}, "")
+	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore1"), []string{filepath.Dir(p1)}, nil)
 	rtest.OK(t, testFileSize(p1rAbs, int64(102)))
 	if _, err := os.Stat(p2rAbs); os.IsNotExist(errors.Cause(err)) {
 		rtest.Assert(t, os.IsNotExist(errors.Cause(err)),
 			"expected %v to not exist in restore, but it exists, err %v", p2rAbs, err)
 	}
 
-	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore2"), []string{filepath.Dir(p2)}, "")
+	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore2"), []string{filepath.Dir(p2)}, nil)
 	rtest.OK(t, testFileSize(p2rAbs, int64(103)))
 	if _, err := os.Stat(p1rAbs); os.IsNotExist(errors.Cause(err)) {
 		rtest.Assert(t, os.IsNotExist(errors.Cause(err)),

cmd/restic/local_layout_test.go

@@ -33,7 +33,7 @@ func TestRestoreLocalLayout(t *testing.T) {
 
 		// restore latest snapshot
 		target := filepath.Join(env.base, "restore")
-		testRunRestoreLatest(t, env.gopts, target, nil, "")
+		testRunRestoreLatest(t, env.gopts, target, nil, nil)
 
 		rtest.RemoveAll(t, filepath.Join(env.base, "repo"))
 		rtest.RemoveAll(t, target)