diff --git a/CHANGELOG b/CHANGELOG index 9f35ea5e8..634434311 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -238,3 +238,6 @@ 0.28.2/2020-02-25 * patch CVE-2019-5436, CVE-2019-5481 and CVE-2019-5482 in curl-7.52.1-5+deb9u9 and libcurl3-7.52.1-5+deb9u9 + +0.29.0/2020-04-22 + * Added support for global file redaction `meta.redact.files` diff --git a/Makefile b/Makefile index 12d05087e..1c243f3bd 100644 --- a/Makefile +++ b/Makefile @@ -156,7 +156,7 @@ e2e-supportbundle-core: -v /var/run/docker.sock:/var/run/docker.sock \ -w /go/src/$(PKG) \ -l com.replicated.support-bundle=true \ - golang:1.10 \ + golang:1.13 \ /bin/sh -c " \ ./e2e/collect/e2e.sh \ " @@ -171,7 +171,7 @@ e2e-supportbundle-docker: -w /go/src/$(PKG) \ -l com.replicated.support-bundle=true \ -e DOCKER=1 \ - golang:1.10 \ + golang:1.13 \ /bin/sh -c " \ ./e2e/collect/e2e.sh \ " @@ -185,7 +185,7 @@ e2e-supportbundle-swarm: -w /go/src/$(PKG) \ -l com.replicated.support-bundle=true \ -e SWARM=1 \ - golang:1.10 \ + golang:1.13 \ /bin/sh -c " \ ./e2e/collect/e2e.sh \ " diff --git a/e2e/collect/meta/meta_file_redaction_test.go b/e2e/collect/meta/meta_file_redaction_test.go new file mode 100644 index 000000000..d6d24da95 --- /dev/null +++ b/e2e/collect/meta/meta_file_redaction_test.go @@ -0,0 +1,55 @@ +package meta + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/replicatedcom/support-bundle/e2e/collect/ginkgo" +) + +var _ = Describe("meta.files_redaction", func() { + + BeforeEach(EnterNewTempDir) + AfterEach(LogResultsFromBundle) + AfterEach(CleanupDir) + + Context("When a meta.redact.files spec is included", func() { + + It("should output the correct files in the bundle", func() { + + WriteBundleConfig(` +specs: + - os.run-command: + name: sh + args: [-c, echo $HI] + env: [HI=hello!] + output_dir: /os/run-command/echohi/ + - os.run-command: + name: sh + args: [-c, echo $BYE] + env: [BYE=goodbye!] + output_dir: /os/run-command/echobye/ + - meta.redact: + output_dir: /redact/ + files: ["**/echohi/*"] +`) + + GenerateBundle() + + hiResult := GetResultFromBundle("os/run-command/echohi/stdout") + Expect(hiResult.Redacted).To(Equal(true)) + ExpectFileNotInBundle("os/run-command/echohi/stdout") + + byeResult := GetResultFromBundle("os/run-command/echobye/stdout") + Expect(byeResult.Redacted).To(Equal(false)) + contents := GetFileFromBundle("os/run-command/echobye/stdout") + Expect(contents).To(Equal("goodbye!\n")) + + _ = GetResultFromBundle("redact/file_redactions.json") + redactions := GetFileFromBundle("redact/file_redactions.json") + Expect(redactions).To(Equal(`[ + "**/echohi/*" +] +`)) + }) + }) +}) diff --git a/go.mod b/go.mod index f3ad2bcb1..a4db85716 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( github.com/go-kit/kit v0.8.0 github.com/go-logfmt/logfmt v0.3.0 // indirect github.com/go-stack/stack v1.8.0 + github.com/gobwas/glob v0.2.3 github.com/golang/mock v1.2.0 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/googleapis/gnostic v0.2.0 // indirect @@ -47,8 +48,8 @@ require ( github.com/mitchellh/go-homedir v1.0.0 github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675 // indirect github.com/nwaples/rardecode v0.0.0-20171029023500-e06696f847ae // indirect - github.com/onsi/ginkgo v1.10.1 - github.com/onsi/gomega v1.7.0 + github.com/onsi/ginkgo v1.12.0 + github.com/onsi/gomega v1.7.1 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/pelletier/go-toml v1.2.0 // indirect @@ -68,6 +69,7 @@ require ( golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 google.golang.org/grpc v1.20.0 // indirect + gopkg.in/square/go-jose.v2 v2.5.0 gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.17.2 k8s.io/apimachinery v0.17.2 diff --git a/go.sum b/go.sum index b03a87b82..d94e2f98b 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,8 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= @@ -252,9 +254,13 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= @@ -397,6 +403,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To= @@ -455,6 +463,8 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/square/go-jose.v2 v2.5.0 h1:OZ4sdq+Y+SHfYB7vfthi1Ei8b0vkP8ZPQgUfUwdUSqo= +gopkg.in/square/go-jose.v2 v2.5.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= diff --git a/hack/docs/mutations.json b/hack/docs/mutations.json index b943f62a0..215b399d1 100644 --- a/hack/docs/mutations.json +++ b/hack/docs/mutations.json @@ -4270,7 +4270,7 @@ "required": ["scrubs", "output_dir"] }, "merge": { - "description": "A list of scrubbers to apply to all assets. If multiple 'meta.redact' specs are specified, all lists will be used. If a spec specifies a scrubber, it will be run in addition to the global scrubbers.", + "description": "Redaction to apply to all assets. If multiple 'meta.redact' specs are specified, all lists will be used. If a spec specifies a scrubber, it will be run in addition to the global scrubbers.", "examples": [ { "scrubs": [ @@ -4279,6 +4279,9 @@ "replace": "xyz" } ], + "files": [ + "**/secret.json" + ], "output_dir": "redact/" } ], @@ -4286,6 +4289,10 @@ { "path": "scrubs.json", "description": "A list of scrubs that were added globally by this item" + }, + { + "path": "file_redactions.json", + "description": "A list of file redaction patterns that were added globally by this item" } ] } @@ -4296,6 +4303,12 @@ "description": "the list of scrubbers to apply to all outputs" } }, + { + "path": "properties.collect.properties.v1.items.properties[\"meta.redact\"].properties.files", + "merge": { + "description": "the list of file patterns to redact" + } + }, { "path": "properties.collect.properties.v1.items.properties[\"os.hostname\"]", "merge": { diff --git a/hack/docs/schema.json b/hack/docs/schema.json index a7e4e4449..9407c559e 100644 --- a/hack/docs/schema.json +++ b/hack/docs/schema.json @@ -5399,6 +5399,9 @@ "description": "The Kubernetes pod query options (used when querying for a label selector)", "$schema": "http://json-schema.org/draft-04/schema#", "properties": { + "allowWatchBookmarks": { + "type": "boolean" + }, "apiVersion": { "type": "string" }, @@ -5408,9 +5411,6 @@ "fieldSelector": { "type": "string" }, - "includeUninitialized": { - "type": "boolean" - }, "kind": { "type": "string" }, @@ -5543,6 +5543,9 @@ "description": "The Kubernetes pod query options (used when querying for a label selector)", "$schema": "http://json-schema.org/draft-04/schema#", "properties": { + "allowWatchBookmarks": { + "type": "boolean" + }, "apiVersion": { "type": "string" }, @@ -5552,9 +5555,6 @@ "fieldSelector": { "type": "string" }, - "includeUninitialized": { - "type": "boolean" - }, "kind": { "type": "string" }, @@ -5645,6 +5645,9 @@ "follow": { "type": "boolean" }, + "insecureSkipTLSVerifyBackend": { + "type": "boolean" + }, "kind": { "type": "string" }, @@ -5797,6 +5800,9 @@ "description": "An instance of metav1.ListOptions", "$schema": "http://json-schema.org/draft-04/schema#", "properties": { + "allowWatchBookmarks": { + "type": "boolean" + }, "apiVersion": { "type": "string" }, @@ -5806,9 +5812,6 @@ "fieldSelector": { "type": "string" }, - "includeUninitialized": { - "type": "boolean" - }, "kind": { "type": "string" }, @@ -6078,7 +6081,7 @@ "type": "object" }, "meta.redact": { - "description": "A list of scrubbers to apply to all assets. If multiple 'meta.redact' specs are specified, all lists will be used. If a spec specifies a scrubber, it will be run in addition to the global scrubbers.", + "description": "Redaction to apply to all assets. If multiple 'meta.redact' specs are specified, all lists will be used. If a spec specifies a scrubber, it will be run in addition to the global scrubbers.", "examples": [ { "scrubs": [ @@ -6087,6 +6090,9 @@ "replace": "xyz" } ], + "files": [ + "**/secret.json" + ], "output_dir": "redact/" } ], @@ -6094,6 +6100,10 @@ { "path": "scrubs.json", "description": "A list of scrubs that were added globally by this item" + }, + { + "path": "file_redactions.json", + "description": "A list of file redaction patterns that were added globally by this item" } ], "$schema": "http://json-schema.org/draft-04/schema#", @@ -6104,6 +6114,13 @@ "description": { "type": "string" }, + "files": { + "description": "the list of file patterns to redact", + "items": { + "type": "string" + }, + "type": "array" + }, "include_empty": { "type": "boolean" }, diff --git a/pkg/collect/bundle/defaultspec/asset.go b/pkg/collect/bundle/defaultspec/asset.go index eec61f66a..cbb3b37da 100644 --- a/pkg/collect/bundle/defaultspec/asset.go +++ b/pkg/collect/bundle/defaultspec/asset.go @@ -71,7 +71,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var _assetsCoreYml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x96\x3b\x8f\xdb\x30\x10\x84\x7b\xff\x8a\x05\x02\xa4\x3a\x9d\x8a\x00\x29\x54\x1f\x52\xa6\xb9\x32\x08\x0c\x5a\x5c\xc9\xc4\x51\x5c\x86\xbb\xb4\xf3\x40\xfe\x7b\x40\xbd\x2e\x77\xe7\x07\x6d\x0b\x48\x1a\xc3\xb6\x66\xbe\x19\xd1\x4b\x5a\xec\xb1\xe6\x6a\xb5\x02\x78\x07\x1c\xbd\xa7\x20\xc5\x26\x3a\x6d\x71\x05\x50\x40\x4b\x81\xa2\x18\x87\x5c\xc1\xaf\xdf\x2b\x00\x00\x8d\x5c\x07\xe3\xc5\x90\xab\xe0\x71\xb0\xc0\x60\x79\x96\x83\x8e\x9d\xef\xe5\x14\xc5\x47\x59\x6b\x13\x2a\xd0\xd8\xa8\x68\xa5\x7c\x13\x94\xa8\x0d\x86\x0a\x24\xc4\x21\xd8\x52\x9b\x17\x99\x84\xd7\x06\x0d\x77\xfd\x83\x05\xbb\x3e\x94\xf8\x3e\x44\x57\xd4\xd4\x75\xca\xe9\xaa\xd7\x03\x38\xd5\x61\x05\x5a\x09\x8e\x5f\x1c\x4a\x1a\x3d\x5c\x8e\xba\x33\xb0\x66\xfc\xa8\x42\xba\xcd\x2f\x85\xb2\x5f\xb3\xe0\xcd\x35\x68\x93\xc9\x5e\x1b\x47\x1a\xf9\x7c\x44\x87\xdc\x66\x21\x47\xe1\x69\x5c\x13\x10\x5f\x75\xee\xb2\x1a\x8f\xc6\xd3\x74\xe3\x5f\xb1\xe9\x0e\x94\xd6\xe1\x0e\x78\x4b\xfb\xac\x1c\xe3\xd7\xc9\xb1\x4e\x86\xab\xf2\xac\x71\x4f\x97\xe5\x25\xc7\xf5\x79\x69\x13\xe2\x65\x81\xbd\x25\x2f\xd1\xf3\xcb\xc4\x46\xc5\xef\xfb\x7d\x5e\x92\x9f\xa7\xcb\x92\xd2\x6a\xd7\x56\x39\xae\x51\x3b\x59\xb7\xc4\xd2\x57\xc9\xf1\x4e\xe2\xc9\x1c\xbd\x98\x4c\xeb\x20\x9d\xd7\x03\x95\x2e\x1a\x63\x67\x6f\x7a\xef\x95\x6c\x2b\x28\x51\xea\xb2\x61\x51\x9b\x13\x58\x94\x3a\x17\xf5\x57\xe7\x85\x68\xbc\x0c\x8a\xb8\x08\x68\x51\xf1\x42\xd5\x86\xb3\x77\x59\x66\x8d\x4e\x96\xee\x99\xe6\xe0\x27\xb9\x05\x68\x3e\x50\x5d\xd6\x3e\x1a\xd7\xd0\x09\x5a\x92\x65\xe3\x3a\xec\x16\xc5\x51\x74\x27\x07\xe6\x22\xda\xbc\x89\x16\xa1\xed\x30\xb0\x21\xb7\x18\xae\x63\x51\x72\x8e\xd6\x3f\x26\x38\x94\x3d\x85\x27\xe3\x5a\x78\x0f\x8d\x09\xb8\x57\xd6\x5e\x30\xe7\x35\xb9\xc6\xb4\xa5\xf1\xa2\x36\x16\xa7\xe5\x7d\xf1\x64\x33\x5d\x83\x59\x0e\x8c\x22\xc6\xb5\x67\xb6\xef\x33\xff\xfa\x42\xc5\x0c\xf8\x5f\x7a\x7d\xfc\xe7\x2b\x15\x39\x94\xd6\x6c\xca\x4f\xe3\xcf\xfd\x70\xa8\xcb\x34\x0b\x3a\xa7\xc2\x48\xcc\xfe\x4f\x99\xd8\x37\x07\xa7\xe3\xa9\x9f\x63\xed\x38\x37\x3d\x20\x93\xdd\xdd\xa7\xf5\x3a\x94\xff\xf0\xf9\x11\xde\x4a\x6e\x3c\x69\xb5\xe3\x4e\xf1\xb7\xa3\xa1\x07\xae\xdf\x9a\xb8\xad\xfd\x87\xf4\x6a\x0d\x3a\x39\x1e\x9c\x64\x70\x48\x76\x6c\xd4\x7a\xc7\xea\x4f\x00\x00\x00\xff\xff\x41\x4e\x03\x7e\x61\x0d\x00\x00") +var _assetsCoreYml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x96\x4b\x6f\xdb\x30\x10\x84\xef\xfe\x15\x0b\x14\xe8\x29\x8a\x0e\x05\x7a\xd0\x39\xe8\xb1\x97\x1c\x8b\xc2\xa0\xc5\x95\x4c\x84\xe2\xb2\xdc\x65\xdc\x07\xfa\xdf\x0b\xea\xe5\x26\xf1\x83\xb6\x05\xb4\x97\x20\xb6\x66\xbe\x19\x51\x4b\x5a\xec\xb1\xe6\x6a\xb5\x02\x78\x07\x1c\xbd\xa7\x20\xc5\x26\x3a\x6d\x71\x05\x50\x40\x4b\x81\xa2\x18\x87\x5c\xc1\xaf\xdf\x2b\x00\x00\x8d\x5c\x07\xe3\xc5\x90\xab\xe0\x71\xb0\xc0\x60\xd9\xcb\x41\xc7\xce\xf7\x72\x8a\xe2\xa3\xac\xb5\x09\x15\x68\x6c\x54\xb4\x52\xbe\x09\x4a\xd4\x06\x43\x05\x12\xe2\x10\x6c\xa9\xcd\x8b\x4c\xc2\x6b\x83\x86\xbb\xfe\xc1\x82\x5d\x1f\x4a\x7c\x1f\xa2\x2b\x6a\xea\x3a\xe5\x74\xd5\xeb\x01\x9c\xea\xb0\x02\xad\x04\xc7\x2f\x0e\x25\x8d\x1e\x2e\x47\xdd\x19\x58\x33\x7e\x54\x21\xdd\xe6\x97\x42\xd9\xaf\x59\xf0\xe6\x1a\xb4\xc9\x64\xaf\x8d\x23\x8d\x7c\x3e\xa2\x43\x6e\xb3\x90\xa3\xf0\x34\xae\x09\x88\xaf\x3a\x77\x59\x8d\x47\xe3\x69\xba\xf1\xaf\xd8\x74\x07\x4a\xeb\x70\x07\xbc\xa5\x5d\x56\x8e\xf1\xeb\xe4\x58\x27\xc3\x55\x79\xd6\xb8\xa7\xcb\xf2\x92\xe3\xfa\xbc\xb4\x09\xf1\xb2\xc0\xde\x92\x97\xe8\xf9\x65\x62\xa3\xe2\xf7\xdd\x2e\x2f\xc9\xcf\xd3\x65\x49\x69\xf5\xdc\x56\x39\xae\x51\x3b\x59\xb7\xc4\xd2\x57\xc9\xf1\x4e\xe2\xc9\x1c\xbd\x98\x4c\xeb\x20\x9d\xd7\x03\x95\x2e\x1a\x63\x67\x6f\xfa\xdf\x2b\xd9\x56\x50\xa2\xd4\x65\xc3\xa2\x36\x27\xb0\x28\x75\x2e\xea\xaf\xce\x0b\xd1\x78\x19\x14\x71\x11\xd0\xa2\xe2\x85\xaa\x0d\x67\xef\xb2\xcc\x1a\x9d\x2c\xdd\x33\xcd\xc1\x4f\x72\x0b\xd0\x7c\xa0\xba\xac\x7d\x34\xae\xa1\x13\xb4\x24\xcb\xc6\x75\xd8\x2d\x8a\xa3\xe8\x4e\x0e\xcc\x45\xb4\x79\x13\x2d\x42\xdb\x9f\x04\x8b\xe0\x9e\x31\xb0\x21\xb7\x18\xae\x63\x51\x72\x8e\xd6\xbf\x75\x38\x94\x1d\x85\x27\xe3\x5a\x78\x0f\x8d\x09\xb8\x53\xd6\x5e\xb0\x6d\x6a\x72\x8d\x69\x4b\xe3\x45\x6d\x2c\x4e\x4f\xeb\xc5\x8b\xd2\x74\x0d\x66\x39\x30\x8a\x18\xd7\x9e\x39\x0d\xf6\xfc\xeb\x0b\x15\x33\xe0\x7f\xe9\xf5\xf1\x9f\xaf\x54\xe4\x50\x5a\xb3\x29\x3f\x8d\x8f\xfb\xe1\x50\x97\x69\x16\x74\x4e\x85\x91\x98\xfd\x13\x35\xb1\x6f\x0e\x4e\xa7\x5d\x3f\xc7\xda\x71\x6e\x7a\x40\x26\xfb\x7c\x9f\xd6\xeb\x50\xfe\xc3\xe7\x47\x78\x2b\xb9\xf1\xe0\xd6\x8e\x3b\xc5\xdf\x8e\x86\x1e\xb8\x7e\x6b\xe2\xb6\xf6\x1f\xd2\x5f\x6b\xd0\xc9\xf1\xe0\x24\x83\x43\xb2\x63\xa3\xd6\x3b\x56\x7f\x02\x00\x00\xff\xff\xa9\x83\xf5\xdb\xb0\x0d\x00\x00") func assetsCoreYmlBytes() ([]byte, error) { return bindataRead( @@ -86,7 +86,7 @@ func assetsCoreYml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "assets/core.yml", size: 3425, mode: os.FileMode(420), modTime: time.Unix(1555096708, 0)} + info := bindataFileInfo{name: "assets/core.yml", size: 3504, mode: os.FileMode(420), modTime: time.Unix(1587562930, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -106,7 +106,7 @@ func assetsDockerYml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "assets/docker.yml", size: 915, mode: os.FileMode(420), modTime: time.Unix(1556821593, 0)} + info := bindataFileInfo{name: "assets/docker.yml", size: 915, mode: os.FileMode(420), modTime: time.Unix(1548452059, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -126,7 +126,7 @@ func assetsKubernetesYml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "assets/kubernetes.yml", size: 2218, mode: os.FileMode(420), modTime: time.Unix(1548727427, 0)} + info := bindataFileInfo{name: "assets/kubernetes.yml", size: 2218, mode: os.FileMode(420), modTime: time.Unix(1548452059, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -146,7 +146,7 @@ func assetsReplicatedYml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "assets/replicated.yml", size: 1287, mode: os.FileMode(420), modTime: time.Unix(1574458345, 0)} + info := bindataFileInfo{name: "assets/replicated.yml", size: 1287, mode: os.FileMode(420), modTime: time.Unix(1569526063, 0)} a := &asset{bytes: bytes, info: info} return a, nil } @@ -203,8 +203,8 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "assets/core.yml": assetsCoreYml, - "assets/docker.yml": assetsDockerYml, + "assets/core.yml": assetsCoreYml, + "assets/docker.yml": assetsDockerYml, "assets/kubernetes.yml": assetsKubernetesYml, "assets/replicated.yml": assetsReplicatedYml, } @@ -248,11 +248,10 @@ type bintree struct { Func func() (*asset, error) Children map[string]*bintree } - var _bintree = &bintree{nil, map[string]*bintree{ "assets": &bintree{nil, map[string]*bintree{ - "core.yml": &bintree{assetsCoreYml, map[string]*bintree{}}, - "docker.yml": &bintree{assetsDockerYml, map[string]*bintree{}}, + "core.yml": &bintree{assetsCoreYml, map[string]*bintree{}}, + "docker.yml": &bintree{assetsDockerYml, map[string]*bintree{}}, "kubernetes.yml": &bintree{assetsKubernetesYml, map[string]*bintree{}}, "replicated.yml": &bintree{assetsReplicatedYml, map[string]*bintree{}}, }}, @@ -304,3 +303,4 @@ func _filePath(dir, name string) string { cannonicalName := strings.Replace(name, "\\", "/", -1) return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) } + diff --git a/pkg/collect/plans/streams_source.go b/pkg/collect/plans/streams_source.go index b0d599d58..36becbbb2 100644 --- a/pkg/collect/plans/streams_source.go +++ b/pkg/collect/plans/streams_source.go @@ -117,8 +117,9 @@ func (task *StreamsSource) Exec(ctx context.Context, rootDir string) []*types.Re readerGroup.Wait() if !task.Spec.Shared().IncludeEmpty { - task.cleanupResults(ctx, rootDir, results) + task.excludeEmptyResults(ctx, rootDir, results) } + task.redactFiles(ctx, rootDir, results) return results } @@ -236,7 +237,7 @@ func (task *StreamsSource) execStream(ctx context.Context, rootDir string, fileP return results } } else { - writeResultJSON(ctx, rootDir, task.JSONPath, jsonResult, structured) + writeResultJSON(ctx, rootDir, jsonPath, jsonResult, structured) } } @@ -260,11 +261,11 @@ func (task *StreamsSource) execStream(ctx context.Context, rootDir string, fileP } if humanTemplated { - writeResultTemplate(ctx, rootDir, task.HumanPath, humanResult, task.Template, structured) + writeResultTemplate(ctx, rootDir, humanPath, humanResult, task.Template, structured) } if humanYAML { - writeResultYAML(ctx, rootDir, task.HumanPath, humanResult, structured) + writeResultYAML(ctx, rootDir, humanPath, humanResult, structured) } if raw && humanRaw { @@ -314,12 +315,30 @@ func (task *StreamsSource) resultsWithErr(err error, filePath string) []*types.R return resultsWithErr(err, results) } -func (task *StreamsSource) cleanupResults(ctx context.Context, rootDir string, results []*types.Result) { +func (task *StreamsSource) excludeEmptyResults(ctx context.Context, rootDir string, results []*types.Result) { for _, result := range results { if result.Size == 0 { err := os.Remove(path.Join(rootDir, result.Path)) if err != nil { - jww.DEBUG.Printf("Unable to remove empty file %s within %s because of %s", result.Path, rootDir, err.Error()) + jww.DEBUG.Printf("Unable to remove empty file %s within %s because of %v", result.Path, rootDir, err) + } + } + } +} + +func (task *StreamsSource) redactFiles(ctx context.Context, rootDir string, results []*types.Result) { + if len(GlobalFileRedactors) == 0 { + return + } + for _, result := range results { + for _, fileRedactor := range GlobalFileRedactors { + if fileRedactor(result.Path) { + err := os.Remove(path.Join(rootDir, result.Path)) + if err != nil { + jww.DEBUG.Printf("Unable to remove redacted file %s within %s because of %v", result.Path, rootDir, err) + } else { + result.Redacted = true + } } } } diff --git a/pkg/collect/plans/util.go b/pkg/collect/plans/util.go index 332c4980d..227406ae6 100644 --- a/pkg/collect/plans/util.go +++ b/pkg/collect/plans/util.go @@ -7,14 +7,15 @@ import ( "sync" "time" + "github.com/gobwas/glob" "github.com/pkg/errors" - "github.com/replicatedcom/support-bundle/pkg/collect/types" jww "github.com/spf13/jwalterweatherman" ) var GlobalScrubbers []types.BytesScrubber -var GlobalScrubbersLock sync.Mutex +var GlobalFileRedactors []types.FileRedactor +var GlobalLock sync.Mutex func SetCommonFieldsStreamsSource(task StreamsSource, spec types.Spec) (StreamsSource, error) { task.Spec = spec @@ -99,6 +100,35 @@ func RawScrubber(scrubSpec *types.Scrub) (types.BytesScrubber, error) { }, nil } +// FileRedactor creates a file redactor function from a list of files +func FileRedactor(patterns []string) (types.FileRedactor, error) { + type globPattern struct { + glob glob.Glob + pattern string + } + globs := []globPattern{} + + for _, pattern := range patterns { + g, err := glob.Compile(pattern) + if err != nil { + return nil, errors.Wrapf(err, "compile glob %s", pattern) + } + globs = append(globs, globPattern{ + glob: g, + pattern: pattern, + }) + } + return func(name string) bool { + for _, g := range globs { + if g.glob.Match(name) { + jww.DEBUG.Printf("FileRedactor pattern %s found match for path %s", g.pattern, name) + return true + } + } + return false + }, nil +} + func filterStreams(readFrom io.Reader, writeTo io.Writer, scrubber func([]byte) []byte) error { // preserve newline at end var readCounter bytesCounter diff --git a/pkg/collect/plans/util_test.go b/pkg/collect/plans/util_test.go index 49e53f320..5a07b3a14 100644 --- a/pkg/collect/plans/util_test.go +++ b/pkg/collect/plans/util_test.go @@ -3,11 +3,11 @@ package plans import ( "bytes" "math/rand" + "reflect" "regexp" "testing" "github.com/replicatedcom/support-bundle/pkg/collect/types" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -138,3 +138,61 @@ func randStringBytes(n int, letterBytes string) []byte { } return b } + +func TestFileRedactor(t *testing.T) { + type args struct { + files []string + patterns []string + } + tests := []struct { + name string + args args + want []string + wantErr bool + }{ + { + name: "basic", + args: args{ + files: []string{ + "a/b/c.txt", + "a/b/c.log", + "a/b/c.zip", + "a/b/c.tar", + "/d/e.txt", + "/d/e.log", + "/d/e.zip", + "/d/e.tar", + }, + patterns: []string{ + "**/*.zip", + "/d/e.txt", + "*/e.tar", + }, + }, + want: []string{ + "a/b/c.txt", + "a/b/c.log", + "a/b/c.tar", + "/d/e.log", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fileRedactor, err := FileRedactor(tt.args.patterns) + if (err != nil) != tt.wantErr { + t.Errorf("FileRedactor() error = %v, wantErr %v", err, tt.wantErr) + return + } + got := []string{} + for _, name := range tt.args.files { + if !fileRedactor(name) { + got = append(got, name) + } + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("FileRedactor() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/collect/plugins/supportbundle/planners/global_redaction.go b/pkg/collect/plugins/supportbundle/planners/global_redaction.go index 3db498e3c..bda661c1f 100644 --- a/pkg/collect/plugins/supportbundle/planners/global_redaction.go +++ b/pkg/collect/plugins/supportbundle/planners/global_redaction.go @@ -8,27 +8,45 @@ import ( "github.com/replicatedcom/support-bundle/pkg/collect/types" ) -func (s *SupportBundle) GlobalRedaction(spec types.Spec) []types.Task { +func (s *SupportBundle) GlobalRedaction(spec types.Spec) (tasks []types.Task) { + files := spec.GlobalRedaction.Files + + if len(files) > 0 { + fileRedactor, err := plans.FileRedactor(files) + if err != nil { + tasks = append(tasks, plans.PreparedError(err, spec)) + return + } + plans.GlobalLock.Lock() + plans.GlobalFileRedactors = append(plans.GlobalFileRedactors, fileRedactor) + plans.GlobalLock.Unlock() + } + + tasks = append(tasks, &plans.StructuredSource{ + JSONPath: filepath.Join(spec.Shared().OutputDir, "file_redactions.json"), + Producer: func(ctx context.Context) (interface{}, error) { return files, nil }, + }) + scrubs := spec.GlobalRedaction.Scrubs for _, scrub := range scrubs { scrubber, err := plans.RawScrubber(&types.Scrub{Regex: scrub.Regex, Replace: scrub.Replace}) if err != nil { - task := plans.PreparedError(err, spec) - return []types.Task{task} + tasks = append(tasks, plans.PreparedError(err, spec)) + return } // GlobalScrubbers is used within StreamsSource to scrub all files // any task-specific scrubbers will be called first, followed by the global scrubbers in order of insertion - plans.GlobalScrubbersLock.Lock() + plans.GlobalLock.Lock() plans.GlobalScrubbers = append(plans.GlobalScrubbers, scrubber) - plans.GlobalScrubbersLock.Unlock() + plans.GlobalLock.Unlock() } - scubbersTask := plans.StructuredSource{ + tasks = append(tasks, &plans.StructuredSource{ JSONPath: filepath.Join(spec.Shared().OutputDir, "scrubs.json"), Producer: func(ctx context.Context) (interface{}, error) { return scrubs, nil }, - } + }) - return []types.Task{&scubbersTask} + return } diff --git a/pkg/collect/types/plugin.go b/pkg/collect/types/plugin.go index 3c9a5f40a..569e148c7 100644 --- a/pkg/collect/types/plugin.go +++ b/pkg/collect/types/plugin.go @@ -19,3 +19,5 @@ type StreamProducer func(context.Context) (io.Reader, error) type StreamsProducer func(context.Context) (map[string]io.Reader, error) type BytesScrubber func([]byte) []byte + +type FileRedactor func(string) bool diff --git a/pkg/collect/types/result.go b/pkg/collect/types/result.go index ded1b73b7..275beb743 100644 --- a/pkg/collect/types/result.go +++ b/pkg/collect/types/result.go @@ -10,28 +10,31 @@ import ( // both a Pathname and an Error if the file written was corrupted or incomplete. type Result struct { // The subpath within the bundle - Path string `json:"path"` - Format string `json:"format"` - Size int64 `json:"size"` - Spec Spec `json:"spec"` - Error error `json:"error,omitempty"` + Path string `json:"path"` + Format string `json:"format"` + Size int64 `json:"size"` + Spec Spec `json:"spec"` + Redacted bool `json:"redacted"` + Error error `json:"error,omitempty"` } type resultIntermediate struct { - Path string `json:"path"` - Format string `json:"format"` - Size int64 `json:"size"` - Spec Spec `json:"spec"` - Error string `json:"error,omitempty"` + Path string `json:"path"` + Format string `json:"format"` + Size int64 `json:"size"` + Spec Spec `json:"spec"` + Redacted bool `json:"redacted"` + Error string `json:"error,omitempty"` } // MarshalJSON .Error will be {} if it has no exported fields, so replace it with a string. func (r *Result) MarshalJSON() ([]byte, error) { intermediate := resultIntermediate{ - Path: r.Path, - Format: r.Format, - Spec: r.Spec, - Size: r.Size, + Path: r.Path, + Format: r.Format, + Spec: r.Spec, + Size: r.Size, + Redacted: r.Redacted, } if r.Error != nil { intermediate.Error = r.Error.Error() @@ -50,6 +53,7 @@ func (r *Result) UnmarshalJSON(raw []byte) error { r.Format = intermediate.Format r.Spec = intermediate.Spec r.Size = intermediate.Size + r.Redacted = intermediate.Redacted if intermediate.Error != "" { r.Error = errors.New(intermediate.Error) } diff --git a/pkg/collect/types/spec.go b/pkg/collect/types/spec.go index 63d412894..1db770f2d 100644 --- a/pkg/collect/types/spec.go +++ b/pkg/collect/types/spec.go @@ -151,7 +151,8 @@ type CustomerMetaOptions struct { // meta.redact type GlobalRedactionOptions struct { SpecShared `json:",inline,omitempty"` - Scrubs []Scrub `json:"scrubs,omitempty"` + Scrubs []Scrub `json:"scrubs,omitempty"` + Files []string `json:"files,omitempty"` } // meta.channel