diff --git a/Makefile b/Makefile index bbd152dc1..88eb2c2ee 100644 --- a/Makefile +++ b/Makefile @@ -23,4 +23,10 @@ build: mkdir -p bin go build -o ./bin/support-bundle . +githooks: + echo 'make integration-test' > .git/hooks/pre-push + chmod +x .git/hooks/pre-push + echo 'go fmt ./...' > .git/hooks/pre-commit + chmod +x .git/hooks/pre-commit + all: build test diff --git a/cmd/generate.go b/cmd/generate.go index 00293e23c..b7328a19c 100644 --- a/cmd/generate.go +++ b/cmd/generate.go @@ -59,6 +59,10 @@ func init() { } func generate(cmd *cobra.Command, args []string) error { + return Generate(cfgFile, bundlePath, skipDefault, timeoutSeconds) +} + +func Generate(cfgFile string, bundlePath string, skipDefault bool, timeoutSeconds int) error { jww.SetStdoutThreshold(jww.LevelTrace) jww.FEEDBACK.Println("Generating a new support bundle") diff --git a/pkg/plans/byte-source.go b/pkg/plans/byte-source.go index 52e2c4acf..37aa82279 100644 --- a/pkg/plans/byte-source.go +++ b/pkg/plans/byte-source.go @@ -12,8 +12,12 @@ import ( type ByteSource struct { // Producer provides the seed data for this task Producer func(context.Context) ([]byte, error) + // RawScrubber, if defined, rewrites the raw data to to remove sensitive data + RawScrubber func([]byte) []byte // Parser, if defined, structures the raw data for json and human sinks Parser func([]byte) (interface{}, error) + // StructuredScrubber, if defined, rewrites the structured data to remove sensitive data + //StructedScrubber func(interface{}) (interface{}, error) // Template, if defined, renders structured data in a human-readable format Template string // If RawPath is defined it will get a copy of the raw data []byte @@ -35,6 +39,7 @@ type ByteSource struct { func (task *ByteSource) Exec(ctx context.Context, rootDir string) []*types.Result { parser := task.Parser != nil + rawScrubber := task.RawScrubber != nil raw := task.RawPath != "" @@ -76,6 +81,10 @@ func (task *ByteSource) Exec(ctx context.Context, rootDir string) []*types.Resul return resultsWithErr(err, results) } + if rawScrubber { + data = task.RawScrubber(data) + } + if raw { write(rootDir, task.RawPath, data, rawResult) } diff --git a/pkg/plugins/core/planners/read-file.go b/pkg/plugins/core/planners/read-file.go index 93ad40c98..8bc69d4a4 100644 --- a/pkg/plugins/core/planners/read-file.go +++ b/pkg/plugins/core/planners/read-file.go @@ -1,9 +1,10 @@ package planners import ( - "errors" + "regexp" "time" + "github.com/pkg/errors" "github.com/replicatedcom/support-bundle/pkg/plans" "github.com/replicatedcom/support-bundle/pkg/plugins/core/producers" "github.com/replicatedcom/support-bundle/pkg/types" @@ -17,11 +18,20 @@ func ReadFile(spec types.Spec) []types.Task { return []types.Task{task} } + scrubber, err := rawScrubber(spec.Config.Scrub) + if err != nil { + err := errors.New("spec for core.read-file has invalid scrubber spec") + task := plans.PreparedError(err, spec) + + return []types.Task{task} + } + task := &plans.ByteSource{ - Producer: producers.ReadFile(spec.Config.FilePath), - RawPath: spec.Raw, - JSONPath: spec.JSON, - HumanPath: spec.Human, + Producer: producers.ReadFile(spec.Config.FilePath), + RawScrubber: scrubber, + RawPath: spec.Raw, + JSONPath: spec.JSON, + HumanPath: spec.Human, } if spec.TimeoutSeconds != 0 { @@ -30,3 +40,19 @@ func ReadFile(spec types.Spec) []types.Task { return []types.Task{task} } + +func rawScrubber(scrubSpec types.Scrub) (types.BytesScrubber, error) { + if scrubSpec.Regex == "" { + return nil, nil + } + + regex, err := regexp.Compile(scrubSpec.Regex) + if err != nil { + return nil, errors.Wrapf(err, "parse regex %s", scrubSpec.Regex) + } + + return func(in []byte) []byte { + return regex.ReplaceAll(in, []byte(scrubSpec.Replace)) + }, nil + +} diff --git a/pkg/plugins/docker/planners/daemon.go b/pkg/plugins/docker/planners/daemon.go index 6373ac351..8c98e7857 100644 --- a/pkg/plugins/docker/planners/daemon.go +++ b/pkg/plugins/docker/planners/daemon.go @@ -4,8 +4,8 @@ import ( "path/filepath" "time" - "github.com/replicatedcom/support-bundle/pkg/types" "github.com/replicatedcom/support-bundle/pkg/plans" + "github.com/replicatedcom/support-bundle/pkg/types" ) // path returns "" if dir is empty, otherwise returns the joined pathnme diff --git a/pkg/types/plugin.go b/pkg/types/plugin.go index cf4d5b22a..ff913b3c6 100644 --- a/pkg/types/plugin.go +++ b/pkg/types/plugin.go @@ -15,3 +15,5 @@ type StreamsProducer func(context.Context) (io.Reader, io.Reader, error) type Planner func(Spec) []Task type Plugin map[string]Planner + +type BytesScrubber func([]byte) []byte diff --git a/pkg/types/spec.go b/pkg/types/spec.go index 17ee7d094..5e4736c22 100644 --- a/pkg/types/spec.go +++ b/pkg/types/spec.go @@ -16,5 +16,11 @@ type Spec struct { ContainerID string `yaml:"container_id"` ContainerName string `yaml:"container_name"` Command string `yaml:"command"` + Scrub Scrub `yaml:"scrub"` } } + +type Scrub struct { + Regex string `yaml:"regex"` + Replace string `yaml:"replace"` +} diff --git a/tests/ginkgo/helpers.go b/tests/ginkgo/helpers.go new file mode 100644 index 000000000..17507cc57 --- /dev/null +++ b/tests/ginkgo/helpers.go @@ -0,0 +1,78 @@ +package ginkgo + +import ( + "archive/tar" + "compress/gzip" + . "github.com/onsi/gomega" + jww "github.com/spf13/jwalterweatherman" + "io" + "io/ioutil" + "os" + "strings" +) + +var tmpdir string +var cwd string +var err error + +func EnterNewTempDir() { + cwd, err = os.Getwd() + Expect(err).NotTo(HaveOccurred()) + tmpdir, err = ioutil.TempDir("", "support-bundle") + Expect(err).NotTo(HaveOccurred()) + err = os.Chdir(tmpdir) + Expect(err).NotTo(HaveOccurred()) +} + +func CleanupDir() { + err = os.Chdir(cwd) + Expect(err).NotTo(HaveOccurred()) + err = os.RemoveAll(tmpdir) + Expect(err).NotTo(HaveOccurred()) +} + +func WriteFile(path string, contents string) { + err := ioutil.WriteFile(path, []byte(contents), 0666) + Expect(err).NotTo(HaveOccurred()) +} + +func ReadFile(filename string) []byte { + data, err := ioutil.ReadFile(filename) + Expect(err).NotTo(HaveOccurred()) + return data +} + +func ReadFileFromBundle(archivePath, targetFile string) string { + file, err := os.Open(archivePath) + defer CloseLogErr(file) + Expect(err).NotTo(HaveOccurred()) + + gzr, err := gzip.NewReader(file) + defer CloseLogErr(gzr) + Expect(err).NotTo(HaveOccurred()) + + tr := tar.NewReader(gzr) + + for { + header, err := tr.Next() + Expect(err).NotTo(HaveOccurred()) + if header == nil { + continue + } + + filePath := strings.TrimLeft(header.Name, "0123456789") + jww.DEBUG.Printf("reading tar entry %s looking for %s", filePath, targetFile) + + if filePath == targetFile && header.Typeflag == tar.TypeReg { + contents, err := ioutil.ReadAll(tr) + Expect(err).NotTo(HaveOccurred()) + return string(contents) + } + } +} + +func CloseLogErr(c io.Closer) { + if err := c.Close(); err != nil { + jww.ERROR.Print(err) + } +} diff --git a/tests/ginkgo/scrub_test.go b/tests/ginkgo/scrub_test.go index 0fc850bae..16b302e1a 100644 --- a/tests/ginkgo/scrub_test.go +++ b/tests/ginkgo/scrub_test.go @@ -3,15 +3,49 @@ package ginkgo import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "log" + "github.com/replicatedcom/support-bundle/cmd" + "path" ) var _ = Describe("Scrubbing secrets from file", func() { - BeforeEach(func() { - log.Println("lol") - }) - Specify("Then Replicated should redirect to the new hostname", func() { - Expect("foo").Should(Equal("foo")) + BeforeEach(EnterNewTempDir) + AfterEach(CleanupDir) + + It("Scrubs any instances of PGPASSWORD=.*", func() { + + WriteFile("pg.env", ` +PGDATABASE=mydata +PGPASSWORD=mypass`) + + WriteFile("config.yml", ` +specs: + - builtin: core.read-file + raw: /pg/pg.env + config: + file_path: `+path.Join(tmpdir, "pg.env")+` + scrub: + regex: (PGPASSWORD)=(.*) + replace: $1=REDACTED + `) + + err := cmd.Generate( + path.Join(tmpdir, "config.yml"), + path.Join(tmpdir, "bundle.tar.gz"), + true, + 60, + ) + + Expect(err).To(BeNil()) + + contents := ReadFileFromBundle( + path.Join("bundle.tar.gz"), + "/pg/pg.env", + ) + + Expect(contents).To(ContainSubstring("PGDATABASE=mydata")) + Expect(contents).NotTo(ContainSubstring("PGPASSWORD=mypass")) + Expect(contents).To(ContainSubstring("PGPASSWORD=REDACTED")) }) -}); + +})