From 980b683f797f3664127018c913e9e8af8db06e18 Mon Sep 17 00:00:00 2001 From: Paul Jacobse Date: Tue, 31 Oct 2023 04:13:23 +0100 Subject: [PATCH] feat: added binaryFiles option (#211) * feat: added binaryFiles option * test: added legacy test * docs: updated README's with binaryFiles example * chore: remove modification from legacy README * style: add sections for different file types * chore: remove modification from legacy README --------- Co-authored-by: Devin Stein Co-authored-by: Devin Stein --- README.md | 32 +++++++++++++++++-- ksops.go | 25 ++++++++++++--- ksops_test.go | 8 +++++ test/krm/binaryfile/generate-resources.yaml | 17 ++++++++++ test/krm/binaryfile/kustomization.yaml | 2 ++ test/krm/binaryfile/secret.enc.yaml | 28 ++++++++++++++++ test/krm/binaryfile/want.yaml | 7 ++++ .../legacy/binaryfile/generate-resources.yaml | 10 ++++++ test/legacy/binaryfile/kustomization.yaml | 2 ++ test/legacy/binaryfile/secret.enc.yaml | 28 ++++++++++++++++ test/legacy/binaryfile/want.yaml | 7 ++++ 11 files changed, 159 insertions(+), 7 deletions(-) create mode 100644 test/krm/binaryfile/generate-resources.yaml create mode 100644 test/krm/binaryfile/kustomization.yaml create mode 100644 test/krm/binaryfile/secret.enc.yaml create mode 100644 test/krm/binaryfile/want.yaml create mode 100644 test/legacy/binaryfile/generate-resources.yaml create mode 100644 test/legacy/binaryfile/kustomization.yaml create mode 100644 test/legacy/binaryfile/secret.enc.yaml create mode 100644 test/legacy/binaryfile/want.yaml diff --git a/README.md b/README.md index 1d5b4a5..fab02aa 100644 --- a/README.md +++ b/README.md @@ -199,8 +199,9 @@ https://github.com/viaduct-ai/kustomize-sops/issues `KSOPS` can also generate a Kubernetes Secret directly from encrypted files or dotenv files. +#### Create a Kubernetes Secret from encrypted text-based file + ```bash -# Create a Kubernetes Secret from encrypted file cat < secret-generator.yaml apiVersion: viaduct.ai/v1 kind: ksops @@ -224,8 +225,35 @@ secretFrom: EOF ``` +#### Create a Kubernetes Secret from encrypted binary file + +```bash +cat < secret-generator.yaml +apiVersion: viaduct.ai/v1 +kind: ksops +metadata: + name: example-secret-generator + annotations: + config.kubernetes.io/function: | + exec: + path: ksops +secretFrom: +- metadata: + name: secret-name + labels: + app: foo + annotations: + kustomize.config.k8s.io/needs-hash: "false" + type: Opaque + binaryFiles: + - ./secret.enc + - secret=./secret.enc +EOF +``` + +#### Create a Kubernetes Secret from an encrypted dotenv file + ```bash -# Create a Kubernetes Secret from encrypted dotenv file cat < secret-generator.yaml apiVersion: viaduct.ai/v1 kind: ksops diff --git a/ksops.go b/ksops.go index af54ece..067d9dd 100644 --- a/ksops.go +++ b/ksops.go @@ -10,6 +10,7 @@ package main import ( "bytes" + "encoding/base64" "fmt" "io/ioutil" "os" @@ -29,14 +30,16 @@ type kubernetesSecret struct { Kind string `json:"kind" yaml:"kind"` Metadata types.ObjectMeta `json:"metadata" yaml:"metadata"` Type string `json:"type,omitempty" yaml:"type,omitempty"` - StringData map[string]string `json:"stringData" yaml:"stringData"` + StringData map[string]string `json:"stringData,omitempty" yaml:"stringData,omitempty"` + Data map[string]string `json:"data,omitempty" yaml:"data,omitempty"` } type secretFrom struct { - Files []string `json:"files,omitempty" yaml:"files,omitempty"` - Envs []string `json:"envs,omitempty" yaml:"envs,omitempty"` - Metadata types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` - Type string `json:"type,omitempty" yaml:"type,omitempty"` + Files []string `json:"files,omitempty" yaml:"files,omitempty"` + BinaryFiles []string `json:"binaryFiles,omitempty" yaml:"binaryFiles,omitempty"` + Envs []string `json:"envs,omitempty" yaml:"envs,omitempty"` + Metadata types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty"` + Type string `json:"type,omitempty" yaml:"type,omitempty"` } type ksops struct { @@ -157,6 +160,7 @@ func generate(raw []byte) (string, error) { for i, secretFrom := range manifest.SecretFrom { stringData := make(map[string]string) + binaryData := make(map[string]string) for _, file := range secretFrom.Files { key, path := fileKeyPath(file) @@ -168,6 +172,16 @@ func generate(raw []byte) (string, error) { stringData[key] = string(data) } + for _, file := range secretFrom.BinaryFiles { + key, path := fileKeyPath(file) + data, err := decryptFile(path) + if err != nil { + return "", fmt.Errorf("error decrypting file %q from secretFrom.Files: %w", path, err) + } + + binaryData[key] = base64.StdEncoding.EncodeToString(data) + } + for _, file := range secretFrom.Envs { data, err := decryptFile(file) if err != nil { @@ -189,6 +203,7 @@ func generate(raw []byte) (string, error) { Metadata: secretFrom.Metadata, Type: secretFrom.Type, StringData: stringData, + Data: binaryData, } d, err := yaml.Marshal(&s) if err != nil { diff --git a/ksops_test.go b/ksops_test.go index b6d99ee..7a4ab80 100644 --- a/ksops_test.go +++ b/ksops_test.go @@ -63,6 +63,10 @@ func TestKSOPSPluginInstallation(t *testing.T) { name: "Legacy From File", dir: "test/legacy/file", }, + { + name: "Legacy From Binary File", + dir: "test/legacy/binaryfile", + }, { name: "Legacy From Envs", dir: "test/legacy/envs", @@ -95,6 +99,10 @@ func TestKSOPSPluginInstallation(t *testing.T) { name: "KRM From File", dir: "test/krm/file", }, + { + name: "KRM From Binary File", + dir: "test/krm/binaryfile", + }, { name: "KRM From Envs", dir: "test/krm/envs", diff --git a/test/krm/binaryfile/generate-resources.yaml b/test/krm/binaryfile/generate-resources.yaml new file mode 100644 index 0000000..99cd363 --- /dev/null +++ b/test/krm/binaryfile/generate-resources.yaml @@ -0,0 +1,17 @@ +apiVersion: viaduct.ai/v1 +kind: ksops +metadata: + name: ksops-secret-from-generator + annotations: + config.kubernetes.io/function: | + exec: + # if the binary is your PATH, you can do + path: ksops + # otherwise, path should be relative to manifest files, like + # path: ../../../ksops +secretFrom: +- metadata: + name: mysecret + type: Opaque + binaryFiles: + - ./secret.enc.yaml diff --git a/test/krm/binaryfile/kustomization.yaml b/test/krm/binaryfile/kustomization.yaml new file mode 100644 index 0000000..052a7d7 --- /dev/null +++ b/test/krm/binaryfile/kustomization.yaml @@ -0,0 +1,2 @@ +generators: + - ./generate-resources.yaml diff --git a/test/krm/binaryfile/secret.enc.yaml b/test/krm/binaryfile/secret.enc.yaml new file mode 100644 index 0000000..244f070 --- /dev/null +++ b/test/krm/binaryfile/secret.enc.yaml @@ -0,0 +1,28 @@ +username: ENC[AES256_GCM,data:wm8jYTk=,iv:uODPx9IQmb2gQi2oZ6gpmIGddMN7n2ogKdweMjIZIHw=,tag:xnfYKwnirnOV3tV7CFvlvA==,type:str] +password: ENC[AES256_GCM,data:Lpu0v6Qjt6wLmYZv,iv:P7kfBIuJ7M86Oh43q9FpUDgUCK4REColGTTFGFdhjD0=,tag:wSNv3kOLhdNxBK0r3MNLpw==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2022-12-14T18:15:56Z" + mac: ENC[AES256_GCM,data:mJUZQ0L26QeAkip3WnGxE/p3JAfgLbF5F2MXElcQRLvKberT0cF6MiOXXXCXFyc1/G/HMYVFGMevEuqFEMwAzZCp4oa70KclZTnE3r8/7lmtvNsF+YbLvRRy5Ib5pwpVYBSFp3RNeRHXmw+lUjz2q61DBK40tyHbRrbXvq0ITy4=,iv:shT/5S+8sjj3gdVSYTNX1r0y0db3rNiQ3YIjIP/0BUM=,tag:VPXX7gTNpsnQDe2U++hexA==,type:str] + pgp: + - created_at: "2022-12-14T18:15:50Z" + enc: | + -----BEGIN PGP MESSAGE----- + + hQEMAyUpShfNkFB/AQf/eWYqsnyMtmf1DpllbYFgViTaxTfZNbS+qF9BJeGoL/Q4 + etjpXqqkl61RBzuLxj7KlM00fFpXCwTLnhaUzGIPaAvojHhk3xIT3lBBci3tshlP + qZ8PVzKIhVfBT8zoZ5hhR5mH7iT0SwZaD6RLepe1ygwXcS33WSf7srvuIydeAaTE + nxOcszcw+1/gZ88rcqmNQ8EnD4gZeswOLQiLOCliHAmZBaW3yybwp3GWbbqiJLYl + RsKCOkIfX0gQVI+BXb8wB1FukXl3ZiefO0zDoTZzV/hP8tq2MEhYyMMb/w/5qJo5 + O49kCAo0nItiTFzMI1jKH3GdO9HSdAEvm4RZPGQSEdJeAWyJxw8Hy9HaecQchwCh + PU5nil4v656adnP+gAcmnUvjvq3WQESNkSYqGttXsxCN+0REP0mR2OuaK1LQJkY4 + zr6Ky43znNJ33mgVYF1NKKJN00Ip8GF4l/Km24FnvQ== + =4non + -----END PGP MESSAGE----- + fp: FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4 + unencrypted_regex: ^(apiVersion|metadata|kind|type)$ + version: 3.7.2 diff --git a/test/krm/binaryfile/want.yaml b/test/krm/binaryfile/want.yaml new file mode 100644 index 0000000..e7d1b0d --- /dev/null +++ b/test/krm/binaryfile/want.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +data: + secret.enc.yaml: dXNlcm5hbWU6IGFkbWluCnBhc3N3b3JkOiAxZjJkMWUyZTY3ZGYK +kind: Secret +metadata: + name: mysecret +type: Opaque diff --git a/test/legacy/binaryfile/generate-resources.yaml b/test/legacy/binaryfile/generate-resources.yaml new file mode 100644 index 0000000..79a780a --- /dev/null +++ b/test/legacy/binaryfile/generate-resources.yaml @@ -0,0 +1,10 @@ +apiVersion: viaduct.ai/v1 +kind: ksops +metadata: + name: ksops-secret-from-generator +secretFrom: +- metadata: + name: mysecret + type: Opaque + binaryFiles: + - ./secret.enc.yaml diff --git a/test/legacy/binaryfile/kustomization.yaml b/test/legacy/binaryfile/kustomization.yaml new file mode 100644 index 0000000..052a7d7 --- /dev/null +++ b/test/legacy/binaryfile/kustomization.yaml @@ -0,0 +1,2 @@ +generators: + - ./generate-resources.yaml diff --git a/test/legacy/binaryfile/secret.enc.yaml b/test/legacy/binaryfile/secret.enc.yaml new file mode 100644 index 0000000..244f070 --- /dev/null +++ b/test/legacy/binaryfile/secret.enc.yaml @@ -0,0 +1,28 @@ +username: ENC[AES256_GCM,data:wm8jYTk=,iv:uODPx9IQmb2gQi2oZ6gpmIGddMN7n2ogKdweMjIZIHw=,tag:xnfYKwnirnOV3tV7CFvlvA==,type:str] +password: ENC[AES256_GCM,data:Lpu0v6Qjt6wLmYZv,iv:P7kfBIuJ7M86Oh43q9FpUDgUCK4REColGTTFGFdhjD0=,tag:wSNv3kOLhdNxBK0r3MNLpw==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2022-12-14T18:15:56Z" + mac: ENC[AES256_GCM,data:mJUZQ0L26QeAkip3WnGxE/p3JAfgLbF5F2MXElcQRLvKberT0cF6MiOXXXCXFyc1/G/HMYVFGMevEuqFEMwAzZCp4oa70KclZTnE3r8/7lmtvNsF+YbLvRRy5Ib5pwpVYBSFp3RNeRHXmw+lUjz2q61DBK40tyHbRrbXvq0ITy4=,iv:shT/5S+8sjj3gdVSYTNX1r0y0db3rNiQ3YIjIP/0BUM=,tag:VPXX7gTNpsnQDe2U++hexA==,type:str] + pgp: + - created_at: "2022-12-14T18:15:50Z" + enc: | + -----BEGIN PGP MESSAGE----- + + hQEMAyUpShfNkFB/AQf/eWYqsnyMtmf1DpllbYFgViTaxTfZNbS+qF9BJeGoL/Q4 + etjpXqqkl61RBzuLxj7KlM00fFpXCwTLnhaUzGIPaAvojHhk3xIT3lBBci3tshlP + qZ8PVzKIhVfBT8zoZ5hhR5mH7iT0SwZaD6RLepe1ygwXcS33WSf7srvuIydeAaTE + nxOcszcw+1/gZ88rcqmNQ8EnD4gZeswOLQiLOCliHAmZBaW3yybwp3GWbbqiJLYl + RsKCOkIfX0gQVI+BXb8wB1FukXl3ZiefO0zDoTZzV/hP8tq2MEhYyMMb/w/5qJo5 + O49kCAo0nItiTFzMI1jKH3GdO9HSdAEvm4RZPGQSEdJeAWyJxw8Hy9HaecQchwCh + PU5nil4v656adnP+gAcmnUvjvq3WQESNkSYqGttXsxCN+0REP0mR2OuaK1LQJkY4 + zr6Ky43znNJ33mgVYF1NKKJN00Ip8GF4l/Km24FnvQ== + =4non + -----END PGP MESSAGE----- + fp: FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4 + unencrypted_regex: ^(apiVersion|metadata|kind|type)$ + version: 3.7.2 diff --git a/test/legacy/binaryfile/want.yaml b/test/legacy/binaryfile/want.yaml new file mode 100644 index 0000000..e7d1b0d --- /dev/null +++ b/test/legacy/binaryfile/want.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +data: + secret.enc.yaml: dXNlcm5hbWU6IGFkbWluCnBhc3N3b3JkOiAxZjJkMWUyZTY3ZGYK +kind: Secret +metadata: + name: mysecret +type: Opaque