Added scrubber support to stream and streams types

.circleci/config.yml

@@ -11,4 +11,7 @@ jobs:
     steps:
       - checkout
       - setup_remote_docker
+      - run: go get github.com/onsi/ginkgo/ginkgo
       - run: make test
+      - run: make integration-test
+      - run: make integration-test-docker

Makefile

@@ -14,10 +14,14 @@ generate:
 	./bin/support-bundle generate
 
 test:
-	go test -v `go list ./... | grep -v /vendor/`
+	go test -v `go list ./... | grep -v '/vendor/\|/ginkgo'`
 
 integration-test:
-	ginkgo -v -r -p tests/ginkgo
+	ginkgo -v -r -p --skip="docker container" tests/ginkgo
+
+integration-test-docker:
+	docker pull ubuntu:latest
+	ginkgo -v -r -p --focus="docker container" tests/ginkgo
 
 build:
 	mkdir -p bin
@@ -29,4 +33,4 @@ githooks:
 	echo 'go fmt ./...' > .git/hooks/pre-commit
 	chmod +x .git/hooks/pre-commit
 
-all: build test
+all: build test integration-test integration-test-docker

pkg/plans/stream-source.go

@@ -14,6 +14,8 @@ import (
 type StreamSource struct {
 	// Producer provides the seed data for this task as an io.Reader
 	Producer func(context.Context) (io.Reader, error)
+	// RawScrubber, if defined, rewrites the raw data to to remove sensitive data
+	RawScrubber func([]byte) []byte
 	// Template, if defined, renders structured data in a human-readable format
 	Template string
 	// If RawPath is defined it will get a copy of the data
@@ -66,6 +68,12 @@ func (task *StreamSource) Exec(ctx context.Context, rootDir string) []*types.Res
 		defer closeLogErr(closer)
 	}
 
+	if task.RawScrubber != nil {
+		scrubbedReader, scrubbedWriter := io.Pipe()
+		go filterStreams(data, scrubbedWriter, task.RawScrubber)
+		data = scrubbedReader
+	}
+
 	rawResult := types.Result{}
 	jsonResult := types.Result{}
 	humanResult := types.Result{}

pkg/plans/streams-source.go

@@ -16,6 +16,8 @@ type StreamsSource struct {
 	// Producer provides the seed data for this task as an io.Reader array
 	// Names of the sources are provided as a string array
 	Producer func(context.Context) (map[string]io.Reader, error)
+	// RawScrubber, if defined, rewrites the raw data to to remove sensitive data
+	RawScrubber func([]byte) []byte
 	// Template, if defined, renders structured data in a human-readable format
 	Template string
 	// If RawPath is defined it will get a copy of the data
@@ -76,6 +78,12 @@ func (task *StreamsSource) Exec(ctx context.Context, rootDir string) []*types.Re
 				defer closeLogErr(closer)
 			}
 
+			if task.RawScrubber != nil {
+				scrubbedReader, scrubbedWriter := io.Pipe()
+				go filterStreams(reader, scrubbedWriter, task.RawScrubber)
+				reader = scrubbedReader
+			}
+
 			rawResult := types.Result{}
 			jsonResult := types.Result{}
 			humanResult := types.Result{}

pkg/plans/util.go

@@ -1,7 +1,11 @@
 package plans
 
 import (
+	"bufio"
 	"io"
+	"regexp"
+
+	"github.com/pkg/errors"
 
 	"github.com/replicatedcom/support-bundle/pkg/types"
 	jww "github.com/spf13/jwalterweatherman"
@@ -25,3 +29,33 @@ func closeLogErr(c io.Closer) {
 		jww.ERROR.Print(err)
 	}
 }
+
+// RawScrubber creates a scrubber function from a scrubSpec
+func RawScrubber(scrubSpec types.Scrub) (types.BytesScrubber, error) {
+	if scrubSpec.Regex == "" {
+		return nil, nil
+	}
+
+	regex, err := regexp.Compile(scrubSpec.Regex)
+	if err != nil {
+		return nil, errors.Wrapf(err, "parse regex %s", scrubSpec.Regex)
+	}
+
+	return func(in []byte) []byte {
+		return regex.ReplaceAll(in, []byte(scrubSpec.Replace))
+	}, nil
+}
+
+func filterStreams(readFrom io.Reader, writeTo *io.PipeWriter, scrubber func([]byte) []byte) {
+	lineScanner := bufio.NewScanner(readFrom)
+	for lineScanner.Scan() {
+		line := lineScanner.Bytes()
+		line = scrubber(line)
+
+		_, err := writeTo.Write(line)
+		if err != nil {
+			writeTo.CloseWithError(err)
+		}
+	}
+	writeTo.Close()
+}

pkg/plugins/core/planners/read-command.go

@@ -1,9 +1,10 @@
 package planners
 
 import (
-	"errors"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/replicatedcom/support-bundle/pkg/plans"
 	"github.com/replicatedcom/support-bundle/pkg/plugins/core/producers"
 	"github.com/replicatedcom/support-bundle/pkg/types"
@@ -17,11 +18,20 @@ func ReadCommand(spec types.Spec) []types.Task {
 		return []types.Task{task}
 	}
 
+	scrubber, err := plans.RawScrubber(spec.Config.Scrub)
+	if err != nil {
+		err = errors.Wrap(err, "create scrubber for docker.read-file")
+		task := plans.PreparedError(err, spec)
+
+		return []types.Task{task}
+	}
+
 	task := &plans.ByteSource{
-		Producer:  producers.ReadCommand(spec.Config.Command, spec.Config.Args...),
-		RawPath:   spec.Raw,
-		JSONPath:  spec.JSON,
-		HumanPath: spec.Human,
+		Producer:    producers.ReadCommand(spec.Config.Command, spec.Config.Args...),
+		RawScrubber: scrubber,
+		RawPath:     spec.Raw,
+		JSONPath:    spec.JSON,
+		HumanPath:   spec.Human,
 	}
 
 	if spec.TimeoutSeconds != 0 {

pkg/plugins/core/planners/read-file.go

@@ -1,7 +1,6 @@
 package planners
 
 import (
-	"regexp"
 	"time"
 
 	"github.com/pkg/errors"
@@ -18,9 +17,9 @@ func ReadFile(spec types.Spec) []types.Task {
 		return []types.Task{task}
 	}
 
-	scrubber, err := rawScrubber(spec.Config.Scrub)
+	scrubber, err := plans.RawScrubber(spec.Config.Scrub)
 	if err != nil {
-		err := errors.New("spec for core.read-file has invalid scrubber spec")
+		err = errors.Wrap(err, "create scrubber for core.read-file")
 		task := plans.PreparedError(err, spec)
 
 		return []types.Task{task}
@@ -40,19 +39,3 @@ func ReadFile(spec types.Spec) []types.Task {
 
 	return []types.Task{task}
 }
-
-func rawScrubber(scrubSpec types.Scrub) (types.BytesScrubber, error) {
-	if scrubSpec.Regex == "" {
-		return nil, nil
-	}
-
-	regex, err := regexp.Compile(scrubSpec.Regex)
-	if err != nil {
-		return nil, errors.Wrapf(err, "parse regex %s", scrubSpec.Regex)
-	}
-
-	return func(in []byte) []byte {
-		return regex.ReplaceAll(in, []byte(scrubSpec.Replace))
-	}, nil
-
-}

pkg/plugins/docker/planners/exec-command.go

@@ -1,9 +1,10 @@
 package planners
 
 import (
-	"errors"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/replicatedcom/support-bundle/pkg/plans"
 	"github.com/replicatedcom/support-bundle/pkg/types"
 )
@@ -21,13 +22,22 @@ func (d *Docker) ExecCommand(spec types.Spec) []types.Task {
 		return []types.Task{task}
 	}
 
+	scrubber, err := plans.RawScrubber(spec.Config.Scrub)
+	if err != nil {
+		err = errors.Wrap(err, "create scrubber for docker.exec-command")
+		task := plans.PreparedError(err, spec)
+
+		return []types.Task{task}
+	}
+
 	fullCommand := append([]string{spec.Config.Command}, spec.Config.Args...)
 
 	task := &plans.StreamsSource{
-		Producer:  d.producers.ExecCommand(spec.Config.ContainerID, fullCommand),
-		RawPath:   spec.Raw,
-		JSONPath:  spec.JSON,
-		HumanPath: spec.Human,
+		Producer:    d.producers.ExecCommand(spec.Config.ContainerID, fullCommand),
+		RawScrubber: scrubber,
+		RawPath:     spec.Raw,
+		JSONPath:    spec.JSON,
+		HumanPath:   spec.Human,
 	}
 
 	if spec.Config.ContainerName != "" {

pkg/plugins/docker/planners/logs.go

@@ -1,9 +1,10 @@
 package planners
 
 import (
-	"errors"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/replicatedcom/support-bundle/pkg/plans"
 	"github.com/replicatedcom/support-bundle/pkg/types"
 )
@@ -16,17 +17,26 @@ func (d *Docker) Logs(spec types.Spec) []types.Task {
 		return []types.Task{task}
 	}
 
+	scrubber, err := plans.RawScrubber(spec.Config.Scrub)
+	if err != nil {
+		err = errors.Wrap(err, "create scrubber for docker.logs")
+		task := plans.PreparedError(err, spec)
+
+		return []types.Task{task}
+	}
+
 	producer := d.producers.Logs(spec.Config.ContainerID)
 
 	if spec.Config.ContainerName != "" {
 		producer = d.producers.LogsName(spec.Config.ContainerName)
 	}
 
 	task := &plans.StreamSource{
-		Producer:  producer,
-		RawPath:   spec.Raw,
-		JSONPath:  spec.JSON,
-		HumanPath: spec.Human,
+		Producer:    producer,
+		RawScrubber: scrubber,
+		RawPath:     spec.Raw,
+		JSONPath:    spec.JSON,
+		HumanPath:   spec.Human,
 	}
 
 	if spec.TimeoutSeconds != 0 {

pkg/plugins/docker/planners/read-file.go

@@ -1,9 +1,10 @@
 package planners
 
 import (
-	"errors"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/replicatedcom/support-bundle/pkg/plans"
 	"github.com/replicatedcom/support-bundle/pkg/types"
 )
@@ -21,17 +22,26 @@ func (d *Docker) ReadFile(spec types.Spec) []types.Task {
 		return []types.Task{task}
 	}
 
+	scrubber, err := plans.RawScrubber(spec.Config.Scrub)
+	if err != nil {
+		err = errors.Wrap(err, "create scrubber for docker.read-file")
+		task := plans.PreparedError(err, spec)
+
+		return []types.Task{task}
+	}
+
 	producer := d.producers.ReadFile(spec.Config.ContainerID, spec.Config.FilePath)
 
 	if spec.Config.ContainerName != "" {
 		producer = d.producers.ReadFileByName(spec.Config.ContainerName, spec.Config.FilePath)
 	}
 
 	task := &plans.StreamSource{
-		Producer:  producer,
-		RawPath:   spec.Raw,
-		JSONPath:  spec.JSON,
-		HumanPath: spec.Human,
+		Producer:    producer,
+		RawScrubber: scrubber,
+		RawPath:     spec.Raw,
+		JSONPath:    spec.JSON,
+		HumanPath:   spec.Human,
 	}
 
 	if spec.TimeoutSeconds != 0 {