Backup from stdin and dump to stdout (#694)

Author: Dipta Das

pkg/restic/backup.go

@@ -22,17 +22,28 @@ func (w *ResticWrapper) RunBackup(backupOption BackupOptions) (*BackupOutput, er
 		},
 	}
 
-	// Backup all target directories
-	for _, dir := range backupOption.BackupDirs {
-		out, err := w.backup(dir, backupOption.Host, nil)
+	if backupOption.StdinPipeCommand.Name != "" { // Backup from stdin
+		out, err := w.backupFromStdin(backupOption)
 		if err != nil {
 			return nil, err
 		}
 		// Extract information from the output of backup command
-		err = backupOutput.extractBackupInfo(out, dir, backupOption.Host)
+		err = backupOutput.extractBackupInfo(out, backupOption.StdinFileName, backupOption.Host)
 		if err != nil {
 			return nil, err
 		}
+	} else { // Backup all target directories
+		for _, dir := range backupOption.BackupDirs {
+			out, err := w.backup(dir, backupOption.Host, nil)
+			if err != nil {
+				return nil, err
+			}
+			// Extract information from the output of backup command
+			err = backupOutput.extractBackupInfo(out, dir, backupOption.Host)
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	// Check repository integrity

pkg/restic/commands.go

@@ -1,18 +1,22 @@
 package restic
 
 import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/appscode/go/log"
 	"github.com/appscode/stash/apis/stash/v1alpha1"
-	"github.com/pkg/errors"
+	"github.com/armon/circbuf"
 )
 
 const (
-	Exe = "/bin/restic_0.9.4"
+	ResticCMD = "/bin/restic_0.9.4"
 )
 
 type Snapshot struct {
@@ -34,8 +38,11 @@ func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error)
 	for _, id := range snapshotIDs {
 		args = append(args, id)
 	}
-
-	err := w.sh.Command(Exe, args...).UnmarshalJSON(&result)
+	out, err := w.run(Command{Name: ResticCMD, Args: args})
+	if err != nil {
+		return nil, err
+	}
+	err = json.Unmarshal(out, &result)
 	return result, err
 }
 
@@ -46,18 +53,18 @@ func (w *ResticWrapper) deleteSnapshots(snapshotIDs []string) ([]byte, error) {
 		args = append(args, id)
 	}
 
-	return w.run(Exe, args)
+	return w.run(Command{Name: ResticCMD, Args: args})
 }
 
 func (w *ResticWrapper) initRepositoryIfAbsent() ([]byte, error) {
 	log.Infoln("Ensuring restic repository in the backend")
 	args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json"})
 	args = w.appendCaCertFlag(args)
-	if _, err := w.run(Exe, args); err != nil {
+	if _, err := w.run(Command{Name: ResticCMD, Args: args}); err != nil {
 		args = w.appendCacheDirFlag([]interface{}{"init"})
 		args = w.appendCaCertFlag(args)
 
-		return w.run(Exe, args)
+		return w.run(Command{Name: ResticCMD, Args: args})
 	}
 	return nil, nil
 }
@@ -77,7 +84,32 @@ func (w *ResticWrapper) backup(path, host string, tags []string) ([]byte, error)
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCaCertFlag(args)
 
-	return w.run(Exe, args)
+	return w.run(Command{Name: ResticCMD, Args: args})
+}
+
+func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) {
+	log.Infoln("Backing up stdin data")
+
+	// first add StdinPipeCommand, then add restic command
+	var commands []Command
+	if options.StdinPipeCommand.Name != "" {
+		commands = append(commands, options.StdinPipeCommand)
+	}
+
+	args := []interface{}{"backup", "--stdin"}
+	if options.StdinFileName != "" {
+		args = append(args, "--stdin-filename")
+		args = append(args, options.StdinFileName)
+	}
+	if options.Host != "" {
+		args = append(args, "--host")
+		args = append(args, options.Host)
+	}
+	args = w.appendCacheDirFlag(args)
+	args = w.appendCaCertFlag(args)
+
+	commands = append(commands, Command{Name: ResticCMD, Args: args})
+	return w.run(commands...)
 }
 
 func (w *ResticWrapper) cleanup(retentionPolicy v1alpha1.RetentionPolicy) ([]byte, error) {
@@ -124,41 +156,78 @@ func (w *ResticWrapper) cleanup(retentionPolicy v1alpha1.RetentionPolicy) ([]byt
 		args = w.appendCacheDirFlag(args)
 		args = w.appendCaCertFlag(args)
 
-		return w.run(Exe, args)
+		return w.run(Command{Name: ResticCMD, Args: args})
 	}
 	return nil, nil
 }
 
 func (w *ResticWrapper) restore(path, host, snapshotID string) ([]byte, error) {
 	log.Infoln("Restoring backed up data")
+
 	args := []interface{}{"restore"}
 	if snapshotID != "" {
 		args = append(args, snapshotID)
 	} else {
 		args = append(args, "latest")
 	}
-	args = append(args, "--path")
-	args = append(args, path) // source-path specified in restic fileGroup
-	args = append(args, "--host")
-	args = append(args, host)
+	if path != "" {
+		args = append(args, "--path")
+		args = append(args, path) // source-path specified in restic fileGroup
+	}
+	if host != "" {
+		args = append(args, "--host")
+		args = append(args, host)
+	}
+
+	args = append(args, "--target", "/") // restore in absolute path
+	args = w.appendCacheDirFlag(args)
+	args = w.appendCaCertFlag(args)
+
+	return w.run(Command{Name: ResticCMD, Args: args})
+}
 
-	// Remove last part from the path.
-	// https://github.com/appscode/stash/issues/392
-	args = append(args, "--target", "/")
-	// args = append(args, filepath.Dir(path))
+func (w *ResticWrapper) dump(dumpOptions DumpOptions) ([]byte, error) {
+	log.Infoln("Dumping backed up data")
+
+	args := []interface{}{"dump"}
+	if dumpOptions.Snapshot != "" {
+		args = append(args, dumpOptions.Snapshot)
+	} else {
+		args = append(args, "latest")
+	}
+	if dumpOptions.FileName != "" {
+		args = append(args, dumpOptions.FileName)
+	} else {
+		args = append(args, "stdin")
+	}
+	if dumpOptions.Host != "" {
+		args = append(args, "--host")
+		args = append(args, dumpOptions.Host)
+	}
+	if dumpOptions.Path != "" {
+		args = append(args, "--path")
+		args = append(args, dumpOptions.Path)
+	}
 
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCaCertFlag(args)
 
-	return w.run(Exe, args)
+	// first add restic command, then add StdoutPipeCommand
+	commands := []Command{
+		{Name: ResticCMD, Args: args},
+	}
+	if dumpOptions.StdoutPipeCommand.Name != "" {
+		commands = append(commands, dumpOptions.StdoutPipeCommand)
+	}
+	return w.run(commands...)
 }
 
 func (w *ResticWrapper) check() ([]byte, error) {
 	log.Infoln("Checking integrity of repository")
 	args := w.appendCacheDirFlag([]interface{}{"check"})
 	args = w.appendCaCertFlag(args)
 
-	return w.run(Exe, args)
+	return w.run(Command{Name: ResticCMD, Args: args})
 }
 
 func (w *ResticWrapper) stats() ([]byte, error) {
@@ -167,7 +236,7 @@ func (w *ResticWrapper) stats() ([]byte, error) {
 	args = append(args, "--mode=raw-data", "--quiet")
 	args = w.appendCaCertFlag(args)
 
-	return w.run(Exe, args)
+	return w.run(Command{Name: ResticCMD, Args: args})
 }
 
 func (w *ResticWrapper) appendCacheDirFlag(args []interface{}) []interface{} {
@@ -185,15 +254,30 @@ func (w *ResticWrapper) appendCaCertFlag(args []interface{}) []interface{} {
 	return args
 }
 
-func (w *ResticWrapper) run(cmd string, args []interface{}) ([]byte, error) {
-	out, err := w.sh.Command(cmd, args...).Output()
+func (w *ResticWrapper) run(commands ...Command) ([]byte, error) {
+	// write std errors into os.Stderr and buffer
+	errBuff, err := circbuf.NewBuffer(256)
 	if err != nil {
-		log.Errorf("Error running command '%s %s' output:\n%s", cmd, args, string(out))
-		parts := strings.Split(strings.TrimSuffix(string(out), "\n"), "\n")
-		if len(parts) > 1 {
-			parts = parts[len(parts)-1:]
-			return nil, errors.New(parts[0])
-		}
-	}
-	return out, err
+		return nil, err
+	}
+	w.sh.Stderr = io.MultiWriter(os.Stderr, errBuff)
+
+	for _, cmd := range commands {
+		w.sh.Command(cmd.Name, cmd.Args...)
+	}
+	out, err := w.sh.Output()
+	if err != nil {
+		return nil, formatError(err, errBuff.String())
+	}
+	log.Infoln("sh-output:", string(out))
+	return out, nil
+}
+
+// return last line of std error as error reason
+func formatError(err error, stdErr string) error {
+	parts := strings.Split(strings.TrimSuffix(stdErr, "\n"), "\n")
+	if len(parts) > 1 {
+		return fmt.Errorf("%s, reason: %s", err, parts[len(parts)-1:][0])
+	}
+	return err
 }

pkg/restic/config.go

@@ -15,10 +15,18 @@ type ResticWrapper struct {
 	config SetupOptions
 }
 
+type Command struct {
+	Name string
+	Args []interface{}
+}
+
+// if StdinPipeCommand is specified, BackupDirs will not be used
 type BackupOptions struct {
-	Host            string
-	BackupDirs      []string
-	RetentionPolicy v1alpha1.RetentionPolicy
+	Host             string
+	BackupDirs       []string
+	StdinPipeCommand Command
+	StdinFileName    string // default "stdin"
+	RetentionPolicy  v1alpha1.RetentionPolicy
 }
 
 type RestoreOptions struct {
@@ -28,6 +36,14 @@ type RestoreOptions struct {
 	Snapshots   []string // when Snapshots are specified SourceHost and RestoreDirs will not be used
 }
 
+type DumpOptions struct {
+	Host              string
+	Snapshot          string // default "latest"
+	Path              string
+	FileName          string // default "stdin"
+	StdoutPipeCommand Command
+}
+
 type SetupOptions struct {
 	Provider    string
 	Bucket      string