Generate manifests for producation usage (#130)

* Generate manifests for producation usage * Multiple configuration options for Vault * Fix generator test * Fix generator test * Fix generator test * Fix generator test
ondat · Jul 7, 2022 · c5f4641 · c5f4641
1 parent 74a2fd7
commit c5f4641
Show file tree

Hide file tree

Showing 52 changed files with 788 additions and 116 deletions.
diff --git a/.github/workflows/_e2e-test.yml b/.github/workflows/_e2e-test.yml
@@ -25,8 +25,6 @@ jobs:
       - uses: actions/checkout@v2
       - name: install Taskfile
         run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task
-      - name: fetch dependencies
-        run: ./bin/task fetch:kuttl fetch:kind
       - name: e2e test
         run: PATH=./bin:$PATH ./bin/task go:e2e-tests:${{ inputs.provider }} KIND_CLUSTER_VERSION=${{ inputs.kubever }}
 
diff --git a/.github/workflows/generator-test-on-pr.yml b/.github/workflows/generator-test-on-pr.yml
@@ -0,0 +1,81 @@
+name: Generator test on pr
+
+on:
+  pull_request:
+    branches: [ main, v2* ]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: read
+  actions: read
+  security-events: write
+  packages: write
+
+concurrency:
+  group: ci-generator-${{ github.ref }}-1
+  cancel-in-progress: true
+
+jobs:
+  docker-compose:
+    name: docker compose
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: install Taskfile
+        run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task
+      - uses: "finnp/create-file-action@master"
+        env:
+          FILE_NAME: "trousseau-env"
+          FILE_DATA: |
+            TR_VERBOSE_LEVEL=3
+            TR_ENABLED_PROVIDERS="--enabled-providers=awskms --enabled-providers=vault"
+            TR_SOCKET_LOCATION=${PWD}/bin/run
+            TR_PROXY_IMAGE=ondat/trousseau:proxy-develop
+            TR_AWSKMS_IMAGE=ondat/trousseau:awskms-develop
+            TR_VAULT_IMAGE=ondat/trousseau:vault-develop
+            TR_TROUSSEAU_IMAGE=ondat/trousseau:trousseau-develop
+            TR_AWSKMS_CREDENTIALS=${HOME}/.aws/credentials
+            TR_AWSKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/awskms.yaml
+            TR_VAULT_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/vault.yaml
+      - name: generate services
+        run: PATH=./bin:$PATH ./bin/task prod:generate:docker-compose ENV_LOCATION=trousseau-env
+      - name: validate compose files
+        run: cd generated_manifests/docker-compose ; docker compose -f docker-compose.yaml -f docker-compose.override.awskms.yaml -f docker-compose.override.vault.yaml config
+  kustomize:
+    name: kustomize
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: install Taskfile
+        run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task
+      - uses: "finnp/create-file-action@master"
+        env:
+          FILE_NAME: "awskms.yaml"
+          FILE_DATA: |
+            profile: default
+      - uses: "finnp/create-file-action@master"
+        env:
+          FILE_NAME: "trousseau-env"
+          FILE_DATA: |
+            TR_VERBOSE_LEVEL=3
+            TR_ENABLED_PROVIDERS="--enabled-providers=awskms --enabled-providers=vault"
+            TR_SOCKET_LOCATION=${PWD}/bin/run
+            TR_PROXY_IMAGE=ondat/trousseau:proxy-develop
+            TR_AWSKMS_IMAGE=ondat/trousseau:awskms-develop
+            TR_VAULT_IMAGE=ondat/trousseau:vault-develop
+            TR_TROUSSEAU_IMAGE=ondat/trousseau:trousseau-develop
+            TR_AWSKMS_CREDENTIALS=${HOME}/.aws/credentials
+            TR_AWSKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/awskms.yaml
+            TR_VAULT_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/vault.yaml
+            TR_VAULT_ADDRESS=http://127.0.0.1:8200
+      - name: generate services
+        run: PATH=./bin:$PATH ./bin/task prod:generate:kustomize ENV_LOCATION=trousseau-env
+      - uses: karancode/kustomize-github-action@master
+        with:
+          kustomize_version: '4.5.5'
+          kustomize_build_dir: 'generated_manifests/kustomize'
+          kustomize_output_file: "manifests.yaml"
+      - uses: makocchi-git/actions-k8s-manifests-validate-kubeval@master
+        with:
+          files: manifests.yaml
diff --git a/.gitignore b/.gitignore
@@ -2,4 +2,5 @@ bin
 *.socket
 cover.out
 kind-logs-*/
-kubeconfig
+kubeconfig
+generated_manifests/
diff --git a/.task/cluster.yml b/.task/cluster.yml
@@ -4,9 +4,12 @@ tasks:
     desc: create kind cluster
     deps:
       - delete
+      - :fetch:kind
     cmds:
       - ./bin/kind create cluster --retain --name "{{.KIND_CLUSTER_NAME}}" --wait 2m --config ./tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/kind.yaml
   delete:
     desc: destroy kind cluster
+    deps:
+      - :fetch:kind
     cmds:
       - ./bin/kind delete cluster --name "{{.KIND_CLUSTER_NAME}}"
diff --git a/.task/docker.yml b/.task/docker.yml
@@ -71,40 +71,40 @@ tasks:
       - task: run:trousseau
   run:proxy:
     deps:
-      - :bin-dir:init
+      - :run-dir:init
     cmds:
       - docker rm -f trousseau-proxy || true
-      - docker run -d --name trousseau-proxy --rm -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:proxy-$IMAGE_VERSION
+      - docker run -d --name trousseau-proxy --rm -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:proxy-$IMAGE_VERSION
   run:debug:
     deps:
-      - :bin-dir:init
+      - :run-dir:init
     cmds:
       - docker rm -f trousseau-debug || true
-      - docker run -d --name trousseau-debug --rm -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:debug-$IMAGE_VERSION
+      - docker run -d --name trousseau-debug --rm -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:debug-$IMAGE_VERSION
   run:vault:
     deps:
-      - :bin-dir:init
+      - :run-dir:init
     cmds:
       - docker rm -f trousseau-local-vault || true
       - docker run -d --name=trousseau-local-vault --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=vault-kms-demo' vault
       - sleep 5
       - docker exec -e VAULT_ADDR=http://127.0.0.1:8200 trousseau-local-vault vault login vault-kms-demo
       - docker exec -e VAULT_ADDR=http://127.0.0.1:8200 trousseau-local-vault vault secrets enable transit
       - docker rm -f trousseau-vault || true
-      - docker run -d --name trousseau-vault --rm --network=container:trousseau-local-vault -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/vault.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:vault-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3
+      - docker run -d --name trousseau-vault --rm --network=container:trousseau-local-vault -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/vault.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:vault-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3
   run:awskms:
     deps:
-      - :bin-dir:init
+      - :run-dir:init
     cmds:
       - docker rm -f trousseau-local-aws || true
       - docker run --name trousseau-local-aws --rm --hostname localhost.localstack.cloud -d -e SERVICES=kms -e HOSTNAME=localhost.localstack.cloud -e HOSTNAME_EXTERNAL=localhost.localstack.cloud -e DEFAULT_REGION=eu-west-1 -e KMS_PROVIDER=kms-local -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:0.14.4
       - sleep 5
       - 'printf %"s\n" "endpoint: https://localhost.localstack.cloud:4566" "profile: trousseau-local-aws" "keyArn: $(docker exec trousseau-local-aws awslocal kms create-key | grep Arn | cut -d''"'' -f4)" > tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/awskms.yaml'
       - docker rm -f trousseau-awskms || true
-      - docker run -d --name trousseau-awskms --rm --network=container:trousseau-local-aws -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/aws-credentials.ini:/.aws/credentials -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/awskms.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:awskms-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3
+      - docker run -d --name trousseau-awskms --rm --network=container:trousseau-local-aws -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/aws-credentials.ini:/.aws/credentials -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/awskms.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:awskms-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3
   run:trousseau:
     deps:
-      - :bin-dir:init
+      - :run-dir:init
     cmds:
       - docker rm -f trousseau-core || true
-      - docker run -d --name trousseau-core --rm -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:trousseau-$IMAGE_VERSION {{.ENABLED_PROVIDERS}} -v=3
+      - docker run -d --name trousseau-core --rm -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:trousseau-$IMAGE_VERSION {{.ENABLED_PROVIDERS}} -v=3
diff --git a/.task/fetch.yml b/.task/fetch.yml
@@ -7,14 +7,11 @@ vars:
   VAULT_VERSION: 1.8.1
   KUBECTL_VERSION: 1.21.1
   KUTTL_VERSION: 0.12.1
+  ENVSUBST_VERSION: 1.2.0
   HUSKY_VERSION: 0.2.8
+  CAP_ARCH:
+    sh: uname
 tasks:
-  all:
-    desc: fetch all tools
-    cmds:
-      - task: kubectl
-      - task: kind
-      - task: kuttl
   golangci:
     deps:
       - :bin-dir:init
@@ -83,6 +80,15 @@ tasks:
       - cd bin ; curl -L https://github.com/kudobuilder/kuttl/releases/download/v{{.KUTTL_VERSION}}/kuttl_{{.KUTTL_VERSION}}_{{OS}}_x86_64.tar.gz | tar -xz kubectl-kuttl
     status:
       - test -f ./bin/kuttl
+  envsubst:
+    deps:
+      - :bin-dir:init
+    desc: install envsubst
+    cmds:
+      - cd bin ; curl -o envsubst -L https://github.com/a8m/envsubst/releases/download/v{{.ENVSUBST_VERSION}}/envsubst-{{.CAP_ARCH}}-x86_64
+      - chmod +x ./bin/envsubst
+    status:
+      - test -f ./bin/envsubst
   husky:
     deps:
       - :bin-dir:init
@@ -91,8 +97,5 @@ tasks:
       - cd bin ; curl -L https://github.com/automation-co/husky/releases/download/v{{.HUSKY_VERSION}}/husky_{{.HUSKY_VERSION}}_{{.CAP_ARCH}}_x86_64.tar.gz | tar -xz husky
       - chmod +x ./bin/husky
       - ./bin/husky install
-    vars:
-      CAP_ARCH:
-        sh: uname
     status:
       - test -f ./bin/husky
diff --git a/.task/go.yml b/.task/go.yml
@@ -95,22 +95,32 @@ tasks:
       - task: gosec:trousseau
   gosec:proxy:
     dir: proxy
+    deps:
+      - :fetch:gosec
     cmds:
       - gosec ./...
   gosec:debug:
     dir: providers/debug
+    deps:
+      - :fetch:gosec
     cmds:
       - gosec ./...
   gosec:vault:
     dir: providers/vault
+    deps:
+      - :fetch:gosec
     cmds:
       - gosec ./...
   gosec:awskms:
     dir: providers/awskms
+    deps:
+      - :fetch:gosec
     cmds:
       - gosec ./...
   gosec:trousseau:
     dir: trousseau
+    deps:
+      - :fetch:gosec
     cmds:
       - gosec ./...
   golangci:
@@ -123,14 +133,20 @@ tasks:
       - task: golangci:trousseau
   golangci:proxy:
     dir: proxy
+    deps:
+      - :fetch:golangci
     cmds:
       - golangci-lint run -c ../.golangci.yaml
   golangci:debug:
     dir: providers/debug
+    deps:
+      - :fetch:golangci
     cmds:
       - golangci-lint run -c ../../.golangci.yaml
   golangci:vault:
     dir: providers/vault
+    deps:
+      - :fetch:golangci
     cmds:
       - golangci-lint run -c ../../.golangci.yaml
   golangci:awskms:
@@ -139,6 +155,8 @@ tasks:
       - golangci-lint run -c ../../.golangci.yaml
   golangci:trousseau:
     dir: trousseau
+    deps:
+      - :fetch:golangci
     cmds:
       - golangci-lint run -c ../.golangci.yaml
   unit-tests:
@@ -172,35 +190,35 @@ tasks:
   run:proxy:
     dir: proxy
     deps:
-      - :bin-dir:init
+      - :run-dir:init
       - tidy:proxy
     cmds:
       - go run main.go --listen-addr unix://../bin/run/proxy.socket --trousseau-addr ../bin/run/trousseau.socket
   run:debug:
     dir: providers/debug
     deps:
-      - :bin-dir:init
+      - :run-dir:init
       - tidy:debug
     cmds:
       - go run main.go --listen-addr unix://../../bin/run/debug/debug.socket
   run:vault:
     dir: providers/vault
     deps:
-      - :bin-dir:init
+      - :run-dir:init
       - tidy:vault
     cmds:
       - go run -ldflags '-X github.com/ondat/trousseau/pkg/utils.SecretLogDivider=1' main.go --config-file-path ../../tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/vault.yaml --listen-addr unix://../../bin/run/vault/vault.socket --zap-encoder=console --v=5
   run:awskms:
     dir: providers/awskms
     deps:
-      - :bin-dir:init
+      - :run-dir:init
       - tidy:awskms
     cmds:
       - go run -ldflags '-X github.com/ondat/trousseau/pkg/utils.SecretLogDivider=1' main.go --config-file-path ../../tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/awskms.yaml --listen-addr unix://../../bin/run/awskms/awskms.socket --zap-encoder=console --v=5
   run:trousseau:
     dir: trousseau
     deps:
-      - :bin-dir:init
+      - :run-dir:init
       - tidy:trousseau
     cmds:
       - go run -ldflags '-X github.com/ondat/trousseau/pkg/utils.SecretLogDivider=1' main.go {{.ENABLED_PROVIDERS}} --socket-location ../bin/run --listen-addr unix://../bin/run/trousseau.socket --zap-encoder=console --v=5
@@ -212,30 +230,36 @@ tasks:
       - task: e2e-tests:awskms
   e2e-tests:vault:
     deps:
+      - :fetch:kuttl
+      - :fetch:kind
       - :docker:build:proxy
       - :docker:build:vault
       - :docker:build:trousseau
     cmds:
       - task: :docker:run:proxy
       - task: :docker:run:vault
-      - ENABLED_PROVIDERS="--enabled-providers vault" task docker:run:trousseau
+      - ENABLED_PROVIDERS="--enabled-providers=vault" task docker:run:trousseau
       - task: :cluster:create
       - ./bin/kubectl-kuttl test --config tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/kuttl.yaml
       - task: :cluster:delete
   e2e-tests:awskms:
     deps:
+      - :fetch:kuttl
+      - :fetch:kind
       - :docker:build:proxy
       - :docker:build:awskms
       - :docker:build:trousseau
     cmds:
       - task: :docker:run:proxy
       - task: :docker:run:awskms
-      - ENABLED_PROVIDERS="--enabled-providers awskms" task docker:run:trousseau
+      - ENABLED_PROVIDERS="--enabled-providers=awskms" task docker:run:trousseau
       - task: :cluster:create
       - ./bin/kubectl-kuttl test --config tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/kuttl.yaml
       - task: :cluster:delete
   e2e-tests:debug:
     deps:
+      - :fetch:kuttl
+      - :fetch:kind
       - :docker:build:proxy
       - :docker:build:debug
       - :docker:build:trousseau