Scrub from bundle [ch2747]

Makefile

@@ -23,4 +23,10 @@ build:
 	mkdir -p bin
 	go build -o ./bin/support-bundle .
 
+githooks:
+	echo 'make integration-test' > .git/hooks/pre-push
+	chmod +x .git/hooks/pre-push
+	echo 'go fmt ./...' > .git/hooks/pre-commit
+	chmod +x .git/hooks/pre-commit
+
 all: build test

cmd/generate.go

@@ -59,6 +59,10 @@ func init() {
 }
 
 func generate(cmd *cobra.Command, args []string) error {
+	return Generate(cfgFile, bundlePath, skipDefault, timeoutSeconds)
+}
+
+func Generate(cfgFile string, bundlePath string, skipDefault bool, timeoutSeconds int) error {
 	jww.SetStdoutThreshold(jww.LevelTrace)
 
 	jww.FEEDBACK.Println("Generating a new support bundle")

pkg/plans/byte-source.go

@@ -12,8 +12,12 @@ import (
 type ByteSource struct {
 	// Producer provides the seed data for this task
 	Producer func(context.Context) ([]byte, error)
+	// RawScrubber, if defined, rewrites the raw data to to remove sensitive data
+	RawScrubber func([]byte) []byte
 	// Parser, if defined, structures the raw data for json and human sinks
 	Parser func([]byte) (interface{}, error)
+	// StructuredScrubber, if defined, rewrites the structured data to remove sensitive data
+	//StructedScrubber func(interface{}) (interface{}, error)
 	// Template, if defined, renders structured data in a human-readable format
 	Template string
 	// If RawPath is defined it will get a copy of the raw data []byte
@@ -35,6 +39,7 @@ type ByteSource struct {
 
 func (task *ByteSource) Exec(ctx context.Context, rootDir string) []*types.Result {
 	parser := task.Parser != nil
+	rawScrubber := task.RawScrubber != nil
 
 	raw := task.RawPath != ""
 
@@ -76,6 +81,10 @@ func (task *ByteSource) Exec(ctx context.Context, rootDir string) []*types.Resul
 		return resultsWithErr(err, results)
 	}
 
+	if rawScrubber {
+		data = task.RawScrubber(data)
+	}
+
 	if raw {
 		write(rootDir, task.RawPath, data, rawResult)
 	}

pkg/plugins/core/planners/read-file.go

@@ -1,9 +1,10 @@
 package planners
 
 import (
-	"errors"
+	"regexp"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/replicatedcom/support-bundle/pkg/plans"
 	"github.com/replicatedcom/support-bundle/pkg/plugins/core/producers"
 	"github.com/replicatedcom/support-bundle/pkg/types"
@@ -17,11 +18,20 @@ func ReadFile(spec types.Spec) []types.Task {
 		return []types.Task{task}
 	}
 
+	scrubber, err := rawScrubber(spec.Config.Scrub)
+	if err != nil {
+		err := errors.New("spec for core.read-file has invalid scrubber spec")
+		task := plans.PreparedError(err, spec)
+
+		return []types.Task{task}
+	}
+
 	task := &plans.ByteSource{
-		Producer:  producers.ReadFile(spec.Config.FilePath),
-		RawPath:   spec.Raw,
-		JSONPath:  spec.JSON,
-		HumanPath: spec.Human,
+		Producer:    producers.ReadFile(spec.Config.FilePath),
+		RawScrubber: scrubber,
+		RawPath:     spec.Raw,
+		JSONPath:    spec.JSON,
+		HumanPath:   spec.Human,
 	}
 
 	if spec.TimeoutSeconds != 0 {
@@ -30,3 +40,19 @@ func ReadFile(spec types.Spec) []types.Task {
 
 	return []types.Task{task}
 }
+
+func rawScrubber(scrubSpec types.Scrub) (types.BytesScrubber, error) {
+	if scrubSpec.Regex == "" {
+		return nil, nil
+	}
+
+	regex, err := regexp.Compile(scrubSpec.Regex)
+	if err != nil {
+		return nil, errors.Wrapf(err, "parse regex %s", scrubSpec.Regex)
+	}
+
+	return func(in []byte) []byte {
+		return regex.ReplaceAll(in, []byte(scrubSpec.Replace))
+	}, nil
+
+}

pkg/plugins/docker/planners/daemon.go

@@ -4,8 +4,8 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/replicatedcom/support-bundle/pkg/types"
 	"github.com/replicatedcom/support-bundle/pkg/plans"
+	"github.com/replicatedcom/support-bundle/pkg/types"
 )
 
 // path returns "" if dir is empty, otherwise returns the joined pathnme

pkg/types/plugin.go

@@ -15,3 +15,5 @@ type StreamsProducer func(context.Context) (io.Reader, io.Reader, error)
 type Planner func(Spec) []Task
 
 type Plugin map[string]Planner
+
+type BytesScrubber func([]byte) []byte

pkg/types/spec.go

@@ -16,5 +16,11 @@ type Spec struct {
 		ContainerID   string   `yaml:"container_id"`
 		ContainerName string   `yaml:"container_name"`
 		Command       string   `yaml:"command"`
+		Scrub         Scrub    `yaml:"scrub"`
 	}
 }
+
+type Scrub struct {
+	Regex   string `yaml:"regex"`
+	Replace string `yaml:"replace"`
+}

tests/ginkgo/helpers.go

@@ -0,0 +1,78 @@
+package ginkgo
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	. "github.com/onsi/gomega"
+	jww "github.com/spf13/jwalterweatherman"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+)
+
+var tmpdir string
+var cwd string
+var err error
+
+func EnterNewTempDir() {
+	cwd, err = os.Getwd()
+	Expect(err).NotTo(HaveOccurred())
+	tmpdir, err = ioutil.TempDir("", "support-bundle")
+	Expect(err).NotTo(HaveOccurred())
+	err = os.Chdir(tmpdir)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func CleanupDir() {
+	err = os.Chdir(cwd)
+	Expect(err).NotTo(HaveOccurred())
+	err = os.RemoveAll(tmpdir)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func WriteFile(path string, contents string) {
+	err := ioutil.WriteFile(path, []byte(contents), 0666)
+	Expect(err).NotTo(HaveOccurred())
+}
+
+func ReadFile(filename string) []byte {
+	data, err := ioutil.ReadFile(filename)
+	Expect(err).NotTo(HaveOccurred())
+	return data
+}
+
+func ReadFileFromBundle(archivePath, targetFile string) string {
+	file, err := os.Open(archivePath)
+	defer CloseLogErr(file)
+	Expect(err).NotTo(HaveOccurred())
+
+	gzr, err := gzip.NewReader(file)
+	defer CloseLogErr(gzr)
+	Expect(err).NotTo(HaveOccurred())
+
+	tr := tar.NewReader(gzr)
+
+	for {
+		header, err := tr.Next()
+		Expect(err).NotTo(HaveOccurred())
+		if header == nil {
+			continue
+		}
+
+		filePath := strings.TrimLeft(header.Name, "0123456789")
+		jww.DEBUG.Printf("reading tar entry %s looking for %s", filePath, targetFile)
+
+		if filePath == targetFile && header.Typeflag == tar.TypeReg {
+			contents, err := ioutil.ReadAll(tr)
+			Expect(err).NotTo(HaveOccurred())
+			return string(contents)
+		}
+	}
+}
+
+func CloseLogErr(c io.Closer) {
+	if err := c.Close(); err != nil {
+		jww.ERROR.Print(err)
+	}
+}

tests/ginkgo/scrub_test.go

@@ -3,15 +3,49 @@ package ginkgo
 import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"log"
+	"github.com/replicatedcom/support-bundle/cmd"
+	"path"
 )
 
 var _ = Describe("Scrubbing secrets from file", func() {
-	BeforeEach(func() {
-		log.Println("lol")
-	})
-	Specify("Then Replicated should redirect to the new hostname", func() {
-		Expect("foo").Should(Equal("foo"))
 
+	BeforeEach(EnterNewTempDir)
+	AfterEach(CleanupDir)
+
+	It("Scrubs any instances of PGPASSWORD=.*", func() {
+
+		WriteFile("pg.env", `
+PGDATABASE=mydata
+PGPASSWORD=mypass`)
+
+		WriteFile("config.yml", `
+specs:
+  - builtin: core.read-file
+    raw: /pg/pg.env
+    config:
+      file_path: `+path.Join(tmpdir, "pg.env")+`
+      scrub:
+        regex: (PGPASSWORD)=(.*)
+        replace: $1=REDACTED
+      `)
+
+		err := cmd.Generate(
+			path.Join(tmpdir, "config.yml"),
+			path.Join(tmpdir, "bundle.tar.gz"),
+			true,
+			60,
+		)
+
+		Expect(err).To(BeNil())
+
+		contents := ReadFileFromBundle(
+			path.Join("bundle.tar.gz"),
+			"/pg/pg.env",
+		)
+
+		Expect(contents).To(ContainSubstring("PGDATABASE=mydata"))
+		Expect(contents).NotTo(ContainSubstring("PGPASSWORD=mypass"))
+		Expect(contents).To(ContainSubstring("PGPASSWORD=REDACTED"))
 	})
-});
+
+})