Make pipe commands a array + fix output parsing of cleanup command (#87)

Signed-off-by: Emruz Hossain <emruz@appscode.com>

pkg/restic/backup.go

@@ -128,7 +128,7 @@ func (w *ResticWrapper) runBackup(backupOption BackupOptions) (api_v1beta1.HostB
 
 	//fmt.Println("shell: ",w)
 	// Backup from stdin
-	if backupOption.StdinPipeCommand.Name != "" {
+	if len(backupOption.StdinPipeCommands) != 0 {
 		out, err := w.backupFromStdin(backupOption)
 		if err != nil {
 			return hostStats, err

pkg/restic/commands.go

@@ -142,10 +142,12 @@ func (w *ResticWrapper) backup(params backupParams) ([]byte, error) {
 func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) {
 	log.Infoln("Backing up stdin data")
 
-	// first add StdinPipeCommand, then add restic command
+	// first add StdinPipeCommands, then add restic command
 	var commands []Command
-	if options.StdinPipeCommand.Name != "" {
-		commands = append(commands, options.StdinPipeCommand)
+	if len(options.StdinPipeCommands) != 0 {
+		for i := range options.StdinPipeCommands {
+			commands = append(commands, options.StdinPipeCommands[i])
+		}
 	}
 
 	args := []interface{}{"backup", "--stdin", "--quiet", "--json"}
@@ -288,12 +290,14 @@ func (w *ResticWrapper) dump(dumpOptions DumpOptions) ([]byte, error) {
 	args = w.appendCaCertFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
-	// first add restic command, then add StdoutPipeCommand
+	// first add restic command, then add StdoutPipeCommands
 	commands := []Command{
 		{Name: ResticCMD, Args: args},
 	}
-	if dumpOptions.StdoutPipeCommand.Name != "" {
-		commands = append(commands, dumpOptions.StdoutPipeCommand)
+	if len(dumpOptions.StdoutPipeCommands) != 0 {
+		for i := range dumpOptions.StdoutPipeCommands {
+			commands = append(commands, dumpOptions.StdoutPipeCommands[i])
+		}
 	}
 	return w.run(commands...)
 }

pkg/restic/config.go

@@ -46,14 +46,14 @@ type Command struct {
 }
 
 // BackupOptions specifies backup information
-// if StdinPipeCommand is specified, BackupPaths will not be used
+// if StdinPipeCommands is specified, BackupPaths will not be used
 type BackupOptions struct {
-	Host             string
-	BackupPaths      []string
-	StdinPipeCommand Command
-	StdinFileName    string // default "stdin"
-	RetentionPolicy  v1alpha1.RetentionPolicy
-	Exclude          []string
+	Host              string
+	BackupPaths       []string
+	StdinPipeCommands []Command
+	StdinFileName     string // default "stdin"
+	RetentionPolicy   v1alpha1.RetentionPolicy
+	Exclude           []string
 }
 
 // RestoreOptions specifies restore information
@@ -68,12 +68,12 @@ type RestoreOptions struct {
 }
 
 type DumpOptions struct {
-	Host              string
-	SourceHost        string
-	Snapshot          string // default "latest"
-	Path              string
-	FileName          string // default "stdin"
-	StdoutPipeCommand Command
+	Host               string
+	SourceHost         string
+	Snapshot           string // default "latest"
+	Path               string
+	FileName           string // default "stdin"
+	StdoutPipeCommands []Command
 }
 
 type SetupOptions struct {

pkg/restic/output.go

@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1"
@@ -190,7 +191,12 @@ func extractCheckInfo(out []byte) bool {
 // save valuable information into backupOutput
 func extractCleanupInfo(out []byte) (int64, int64, error) {
 	var fg []ForgetGroup
-	err := json.Unmarshal(out, &fg)
+	// The output can have some warning message along with a array of json. Here, we are going to extract the json part.
+	// The json part start with "[{" and ends with "}]". We are going to use use regular expression to take the first section
+	// that start with "[{" and end with "}]".
+	regex := regexp.MustCompile(`\[{.*}]`)
+	jsonPart := regex.Find(out)
+	err := json.Unmarshal(jsonPart, &fg)
 	if err != nil {
 		return 0, 0, err
 	}

pkg/restic/restic_test.go

@@ -223,8 +223,8 @@ func TestBackupRestoreStdin(t *testing.T) {
 	}
 
 	backupOpt := BackupOptions{
-		StdinPipeCommand: stdinPipeCommand,
-		StdinFileName:    fileName,
+		StdinPipeCommands: []Command{stdinPipeCommand},
+		StdinFileName:     fileName,
 		RetentionPolicy: api_v1alpha1.RetentionPolicy{
 			Name:     "keep-last-1",
 			KeepLast: 1,
@@ -239,8 +239,8 @@ func TestBackupRestoreStdin(t *testing.T) {
 	fmt.Println("backup output:", backupOut)
 
 	dumpOpt := DumpOptions{
-		FileName:          fileName,
-		StdoutPipeCommand: stdoutPipeCommand,
+		FileName:           fileName,
+		StdoutPipeCommands: []Command{stdoutPipeCommand},
 	}
 	dumpOut, err := w.Dump(dumpOpt, testTargetRef)
 	if err != nil {
@@ -338,8 +338,8 @@ func TestBackupRestoreStdinWithScheduling(t *testing.T) {
 	}
 
 	backupOpt := BackupOptions{
-		StdinPipeCommand: stdinPipeCommand,
-		StdinFileName:    fileName,
+		StdinPipeCommands: []Command{stdinPipeCommand},
+		StdinFileName:     fileName,
 		RetentionPolicy: api_v1alpha1.RetentionPolicy{
 			Name:     "keep-last-1",
 			KeepLast: 1,
@@ -354,8 +354,8 @@ func TestBackupRestoreStdinWithScheduling(t *testing.T) {
 	fmt.Println("backup output:", backupOut)
 
 	dumpOpt := DumpOptions{
-		FileName:          fileName,
-		StdoutPipeCommand: stdoutPipeCommand,
+		FileName:           fileName,
+		StdoutPipeCommands: []Command{stdoutPipeCommand},
 	}
 	dumpOut, err := w.Dump(dumpOpt, testTargetRef)
 	if err != nil {
@@ -825,19 +825,19 @@ func newParallelDumpOptions() []DumpOptions {
 
 	return []DumpOptions{
 		{
-			Host:              "host-0",
-			FileName:          filepath.Join(targetPath, fileName),
-			StdoutPipeCommand: stdoutPipeCommand,
+			Host:               "host-0",
+			FileName:           filepath.Join(targetPath, fileName),
+			StdoutPipeCommands: []Command{stdoutPipeCommand},
 		},
 		{
-			Host:              "host-1",
-			FileName:          filepath.Join(targetPath, fileName),
-			StdoutPipeCommand: stdoutPipeCommand,
+			Host:               "host-1",
+			FileName:           filepath.Join(targetPath, fileName),
+			StdoutPipeCommands: []Command{stdoutPipeCommand},
 		},
 		{
-			Host:              "host-2",
-			FileName:          filepath.Join(targetPath, fileName),
-			StdoutPipeCommand: stdoutPipeCommand,
+			Host:               "host-2",
+			FileName:           filepath.Join(targetPath, fileName),
+			StdoutPipeCommands: []Command{stdoutPipeCommand},
 		},
 	}
 }