From 50a106b28c4d2f370100c75c10a4f5e1fc88a85f Mon Sep 17 00:00:00 2001 From: "Cornelius A. Ludmann" Date: Mon, 13 Jun 2022 19:47:34 +0000 Subject: [PATCH 1/3] [kots] Allow multiple docker pull secrets --- .../kots/manifests/gitpod-installer-job.yaml | 27 ++++++++++++++++--- install/kots/manifests/kots-config.yaml | 9 +++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/install/kots/manifests/gitpod-installer-job.yaml b/install/kots/manifests/gitpod-installer-job.yaml index 43219a6d7678db..ccc2736dae448a 100644 --- a/install/kots/manifests/gitpod-installer-job.yaml +++ b/install/kots/manifests/gitpod-installer-job.yaml @@ -148,8 +148,6 @@ spec: yq e -i ".containerRegistry.inCluster = false" "${CONFIG_FILE}" yq e -i ".containerRegistry.external.url = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}" - yq e -i ".containerRegistry.external.certificate.kind = \"secret\"" "${CONFIG_FILE}" - yq e -i ".containerRegistry.external.certificate.name = \"{{repl ImagePullSecretName }}\"" "${CONFIG_FILE}" yq e -i ".repository = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}" yq e -i ".imagePullSecrets[0].kind = \"secret\"" "${CONFIG_FILE}" yq e -i ".imagePullSecrets[0].name = \"{{repl ImagePullSecretName }}\"" "${CONFIG_FILE}" @@ -160,8 +158,6 @@ spec: yq e -i ".containerRegistry.inCluster = false" "${CONFIG_FILE}" yq e -i ".containerRegistry.external.url = \"{{repl ConfigOption "reg_url" }}\"" "${CONFIG_FILE}" - yq e -i ".containerRegistry.external.certificate.kind = \"secret\"" "${CONFIG_FILE}" - yq e -i ".containerRegistry.external.certificate.name = \"container-registry\"" "${CONFIG_FILE}" else if [ '{{repl ConfigOptionEquals "reg_incluster_storage" "s3" }}' = "true" ]; then @@ -174,6 +170,9 @@ spec: yq e -i ".containerRegistry.s3storage.certificate.name = \"container-registry-s3-backend\"" "${CONFIG_FILE}" fi fi + # merged-registry-auths will be created below + yq e -i ".containerRegistry.external.certificate.kind = \"secret\"" "${CONFIG_FILE}" + yq e -i ".containerRegistry.external.certificate.name = \"merged-registry-auths\"" "${CONFIG_FILE}" if [ '{{repl ConfigOptionNotEquals "store_provider" "incluster" }}' = "true" ]; then @@ -273,6 +272,26 @@ spec: 'del(select(.kind == "StatefulSet" and .metadata.name == "openvsx-proxy").status)' \ "${GITPOD_OBJECTS}/templates/gitpod.yaml" + # Merge different docker pull secrets into one secret + if [ '{{repl HasLocalRegistry }}' = "true" ]; + then + kubectl get secret "{{repl ImagePullSecretName }}" -o=jsonpath="{.data['\.dockerconfigjson']}" | base64 -d | yq -P - > registry-auth-airgap.yaml + fi + if [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ]; + then + kubectl get secret "container-registry" -o=jsonpath="{.data['\.dockerconfigjson']}" | base64 -d | yq -P - > registry-auth-external.yaml + else + yq eval 'select(.kind == "Secret" and .metadata.name == "builtin-registry-auth").data.".dockerconfigjson"' \ + "${GITPOD_OBJECTS}/templates/gitpod.yaml" | base64 -d | yq -P - > registry-auth-builtin.yaml + fi + # merge all files together (https://mikefarah.gitbook.io/yq/operators/reduce#merge-all-yaml-files-together) + yq -o=json eval-all '. as $item ireduce ({}; . * $item )' registry-auth-*.yaml > merged-registry-auths.yaml + # create secret and update if exists (https://stackoverflow.com/a/45881259/1364435) + kubectl create secret generic merged-registry-auths \ + --save-config --dry-run=client \ + --from-file=.dockerconfigjson=./merged-registry-auths.yaml \ + -o yaml > "${GITPOD_OBJECTS}/templates/merged-registry-auths.yaml" + echo "Gitpod: Escape any Golang template values" sed -i -r 's/(.*\{\{.*)/{{`\1`}}/' "${GITPOD_OBJECTS}/templates/gitpod.yaml" diff --git a/install/kots/manifests/kots-config.yaml b/install/kots/manifests/kots-config.yaml index 9e0116115af863..e801dea01728a2 100644 --- a/install/kots/manifests/kots-config.yaml +++ b/install/kots/manifests/kots-config.yaml @@ -28,7 +28,6 @@ spec: - name: reg_incluster title: Use in-cluster container registry type: bool - when: '{{repl eq HasLocalRegistry false }}' default: "1" help_text: You may either use an in-cluster container registry or configure your own external container registry for better performance. This container registry must be accessible from your Kubernetes cluster. recommended: false @@ -85,27 +84,27 @@ spec: - name: reg_url title: Container registry URL type: text - when: '{{repl and (eq HasLocalRegistry false) (ConfigOptionEquals "reg_incluster" "0") }}' + when: '{{repl (ConfigOptionEquals "reg_incluster" "0") }}' required: true help_text: The container registry URL. This will usually be the fully qualified domain of your registry. - name: reg_server title: Container registry server type: text - when: '{{repl and (eq HasLocalRegistry false) (ConfigOptionEquals "reg_incluster" "0") }}' + when: '{{repl (ConfigOptionEquals "reg_incluster" "0") }}' help_text: The container registry server. This is used when [generating your credentials](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-secret-by-providing-credentials-on-the-command-line). Depending upon your provider, this may or may not be the same as the registry URL. If not specified, the URL will be used. - name: reg_username title: Container registry username type: text - when: '{{repl and (eq HasLocalRegistry false) (ConfigOptionEquals "reg_incluster" "0") }}' + when: '{{repl (ConfigOptionEquals "reg_incluster" "0") }}' required: true help_text: The username for your container registry. - name: reg_password title: Container registry password type: password - when: '{{repl and (eq HasLocalRegistry false) (ConfigOptionEquals "reg_incluster" "0") }}' + when: '{{repl (ConfigOptionEquals "reg_incluster" "0") }}' required: true help_text: The password for your container registry. From 7dadc947fb40d8cdb49f591a73af63eddf472c8b Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Tue, 14 Jun 2022 17:19:40 +0100 Subject: [PATCH 2/3] add changes --- .../kots/manifests/gitpod-installer-job.yaml | 68 +++++++++++++------ 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/install/kots/manifests/gitpod-installer-job.yaml b/install/kots/manifests/gitpod-installer-job.yaml index ccc2736dae448a..a055a7b712af47 100644 --- a/install/kots/manifests/gitpod-installer-job.yaml +++ b/install/kots/manifests/gitpod-installer-job.yaml @@ -28,7 +28,7 @@ spec: containers: - name: installer # This will normally be the release tag - image: "eu.gcr.io/gitpod-core-dev/build/installer:sje-registry-s3-fork.3" + image: eu.gcr.io/gitpod-core-dev/build/installer:clu-multiple-pull-secrets.1 volumeMounts: - mountPath: /config-patch name: config-patch @@ -57,6 +57,7 @@ spec: args: - | set -e + set -x echo "Gitpod: Killing any in-progress installations" @@ -142,22 +143,26 @@ spec: yq e -i ".database.external.certificate.name = \"database\"" "${CONFIG_FILE}" fi + # we really don't care if we deploy from a mirror if [ '{{repl HasLocalRegistry }}' = "true" ]; then echo "Gitpod: configuring mirrored container registry" - yq e -i ".containerRegistry.inCluster = false" "${CONFIG_FILE}" - yq e -i ".containerRegistry.external.url = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}" - yq e -i ".repository = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}" - yq e -i ".imagePullSecrets[0].kind = \"secret\"" "${CONFIG_FILE}" - yq e -i ".imagePullSecrets[0].name = \"{{repl ImagePullSecretName }}\"" "${CONFIG_FILE}" - yq e -i '.dropImageRepo = true' "${CONFIG_FILE}" - elif [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ]; + # yq e -i ".containerRegistry.external.url = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}" + # yq e -i ".repository = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}" + # yq e -i ".imagePullSecrets[0].kind = \"secret\"" "${CONFIG_FILE}" + # yq e -i ".imagePullSecrets[0].name = \"{{repl ImagePullSecretName }}\"" "${CONFIG_FILE}" + # yq e -i '.dropImageRepo = false' "${CONFIG_FILE}" + fi + + # separate to using the internal registry we need to create an incluster or use an external registry still + if [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ]; then echo "Gitpod: configuring external container registry" - yq e -i ".containerRegistry.inCluster = false" "${CONFIG_FILE}" yq e -i ".containerRegistry.external.url = \"{{repl ConfigOption "reg_url" }}\"" "${CONFIG_FILE}" + yq e -i ".containerRegistry.external.certificate.kind = \"secret\"" "${CONFIG_FILE}" + yq e -i ".containerRegistry.external.certificate.name = \"gitpod-registry\"" "${CONFIG_FILE}" else if [ '{{repl ConfigOptionEquals "reg_incluster_storage" "s3" }}' = "true" ]; then @@ -170,9 +175,7 @@ spec: yq e -i ".containerRegistry.s3storage.certificate.name = \"container-registry-s3-backend\"" "${CONFIG_FILE}" fi fi - # merged-registry-auths will be created below - yq e -i ".containerRegistry.external.certificate.kind = \"secret\"" "${CONFIG_FILE}" - yq e -i ".containerRegistry.external.certificate.name = \"merged-registry-auths\"" "${CONFIG_FILE}" + if [ '{{repl ConfigOptionNotEquals "store_provider" "incluster" }}' = "true" ]; then @@ -212,7 +215,7 @@ spec: if [ '{{repl ConfigOptionEquals "ssh_gateway" "1" }}' = "true" ]; then echo "Gitpod: Generate SSH host key" - ssh-keygen -t rsa -q -N "" -f host.key + ssh-keygen -t ecdsa -q -N "" -f host.key kubectl create secret generic ssh-gateway-host-key --from-file=host.key -n {{repl Namespace }} || echo "SSH Gateway Host Key secret has not been created. Does it exist already?" yq e -i '.sshGatewayHostKey.kind = "secret"' "${CONFIG_FILE}" yq e -i '.sshGatewayHostKey.name = "ssh-gateway-host-key"' "${CONFIG_FILE}" @@ -272,25 +275,48 @@ spec: 'del(select(.kind == "StatefulSet" and .metadata.name == "openvsx-proxy").status)' \ "${GITPOD_OBJECTS}/templates/gitpod.yaml" - # Merge different docker pull secrets into one secret + # If we have a local registry, cache it's credentials to be merged into whichever secret we use + # to access a registry for registry-facade if [ '{{repl HasLocalRegistry }}' = "true" ]; then kubectl get secret "{{repl ImagePullSecretName }}" -o=jsonpath="{.data['\.dockerconfigjson']}" | base64 -d | yq -P - > registry-auth-airgap.yaml fi + if [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ]; then kubectl get secret "container-registry" -o=jsonpath="{.data['\.dockerconfigjson']}" | base64 -d | yq -P - > registry-auth-external.yaml + + if [[ -f ./registry-auth-airgap.yaml ]] + then + yq -o=json eval-all 'select(fi == 0) * select(filename == "registry-auth-airgap.yaml")' registry-auth-external.yaml registry-auth-airgap.yaml > merged-registry-auths.json + # create secret and update if exists (https://stackoverflow.com/a/45881259/1364435) + kubectl create secret generic container-registry \ + --save-config --dry-run=client \ + --from-file=.dockerconfigjson=./merged-registry-auths.json \ + -o yaml | kubectl apply -f - + fi else yq eval 'select(.kind == "Secret" and .metadata.name == "builtin-registry-auth").data.".dockerconfigjson"' \ "${GITPOD_OBJECTS}/templates/gitpod.yaml" | base64 -d | yq -P - > registry-auth-builtin.yaml + + echo " ------------builtin auth output" + cat registry-auth-builtin.yaml + + if [[ -f ./registry-auth-airgap.yaml ]] + then + echo " ------------mirror output" + cat registry-auth-airgap.yaml + yq -o=json eval-all 'select(fi == 0) * select(filename == "registry-auth-airgap.yaml")' registry-auth-builtin.yaml registry-auth-airgap.yaml > merged-registry-auths.json + echo " ------------merged" + cat merged-registry-auths.json + # create secret and update if exists (https://stackoverflow.com/a/45881259/1364435) + export NEW_SECRETS=$(base64 -w 0 merged-registry-auths.json) + yq e -i "select(.kind == \"Secret\" and .metadata.name == \"builtin-registry-auth\").data.\".dockerconfigjson\" = \"${NEW_SECRETS}\"" \ + "${GITPOD_OBJECTS}/templates/gitpod.yaml" + yq e 'select(.kind == "Secret" and .metadata.name == "builtin-registry-auth")' "${GITPOD_OBJECTS}/templates/gitpod.yaml" + fi + fi - # merge all files together (https://mikefarah.gitbook.io/yq/operators/reduce#merge-all-yaml-files-together) - yq -o=json eval-all '. as $item ireduce ({}; . * $item )' registry-auth-*.yaml > merged-registry-auths.yaml - # create secret and update if exists (https://stackoverflow.com/a/45881259/1364435) - kubectl create secret generic merged-registry-auths \ - --save-config --dry-run=client \ - --from-file=.dockerconfigjson=./merged-registry-auths.yaml \ - -o yaml > "${GITPOD_OBJECTS}/templates/merged-registry-auths.yaml" echo "Gitpod: Escape any Golang template values" sed -i -r 's/(.*\{\{.*)/{{`\1`}}/' "${GITPOD_OBJECTS}/templates/gitpod.yaml" From e1a18df737f58554cd2d9aa4633e93873bd48d91 Mon Sep 17 00:00:00 2001 From: Chris Barker Date: Wed, 15 Jun 2022 16:34:38 +0100 Subject: [PATCH 3/3] add cleaned up version --- .../kots/manifests/gitpod-installer-job.yaml | 73 ++++++++----------- 1 file changed, 29 insertions(+), 44 deletions(-) diff --git a/install/kots/manifests/gitpod-installer-job.yaml b/install/kots/manifests/gitpod-installer-job.yaml index a055a7b712af47..6d92a12b58869d 100644 --- a/install/kots/manifests/gitpod-installer-job.yaml +++ b/install/kots/manifests/gitpod-installer-job.yaml @@ -28,7 +28,7 @@ spec: containers: - name: installer # This will normally be the release tag - image: eu.gcr.io/gitpod-core-dev/build/installer:clu-multiple-pull-secrets.1 + image: eu.gcr.io/gitpod-core-dev/build/installer:clu-multiple-pull-secrets.5 volumeMounts: - mountPath: /config-patch name: config-patch @@ -57,7 +57,6 @@ spec: args: - | set -e - set -x echo "Gitpod: Killing any in-progress installations" @@ -141,28 +140,27 @@ spec: yq e -i ".database.inCluster = false" "${CONFIG_FILE}" yq e -i ".database.external.certificate.kind = \"secret\"" "${CONFIG_FILE}" yq e -i ".database.external.certificate.name = \"database\"" "${CONFIG_FILE}" + fi - # we really don't care if we deploy from a mirror if [ '{{repl HasLocalRegistry }}' = "true" ]; then echo "Gitpod: configuring mirrored container registry" - - # yq e -i ".containerRegistry.external.url = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}" - # yq e -i ".repository = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}" - # yq e -i ".imagePullSecrets[0].kind = \"secret\"" "${CONFIG_FILE}" - # yq e -i ".imagePullSecrets[0].name = \"{{repl ImagePullSecretName }}\"" "${CONFIG_FILE}" - # yq e -i '.dropImageRepo = false' "${CONFIG_FILE}" + yq e -i ".repository = \"{{repl LocalRegistryAddress }}\"" "${CONFIG_FILE}" + yq e -i ".imagePullSecrets[0].kind = \"secret\"" "${CONFIG_FILE}" + yq e -i ".imagePullSecrets[0].name = \"{{repl ImagePullSecretName }}\"" "${CONFIG_FILE}" + yq e -i '.dropImageRepo = true' "${CONFIG_FILE}" fi - # separate to using the internal registry we need to create an incluster or use an external registry still if [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ]; then echo "Gitpod: configuring external container registry" + yq e -i ".containerRegistry.inCluster = false" "${CONFIG_FILE}" yq e -i ".containerRegistry.external.url = \"{{repl ConfigOption "reg_url" }}\"" "${CONFIG_FILE}" yq e -i ".containerRegistry.external.certificate.kind = \"secret\"" "${CONFIG_FILE}" - yq e -i ".containerRegistry.external.certificate.name = \"gitpod-registry\"" "${CONFIG_FILE}" + yq e -i ".containerRegistry.external.certificate.name = \"container-registry\"" "${CONFIG_FILE}" + else if [ '{{repl ConfigOptionEquals "reg_incluster_storage" "s3" }}' = "true" ]; then @@ -215,7 +213,7 @@ spec: if [ '{{repl ConfigOptionEquals "ssh_gateway" "1" }}' = "true" ]; then echo "Gitpod: Generate SSH host key" - ssh-keygen -t ecdsa -q -N "" -f host.key + ssh-keygen -t rsa -q -N "" -f host.key kubectl create secret generic ssh-gateway-host-key --from-file=host.key -n {{repl Namespace }} || echo "SSH Gateway Host Key secret has not been created. Does it exist already?" yq e -i '.sshGatewayHostKey.kind = "secret"' "${CONFIG_FILE}" yq e -i '.sshGatewayHostKey.name = "ssh-gateway-host-key"' "${CONFIG_FILE}" @@ -275,47 +273,34 @@ spec: 'del(select(.kind == "StatefulSet" and .metadata.name == "openvsx-proxy").status)' \ "${GITPOD_OBJECTS}/templates/gitpod.yaml" - # If we have a local registry, cache it's credentials to be merged into whichever secret we use + # If we have a airgap registry, cache it's credentials to be merged into whichever secret we use # to access a registry for registry-facade - if [ '{{repl HasLocalRegistry }}' = "true" ]; - then + if [ '{{repl HasLocalRegistry }}' = "true" ]; then + echo "Gitpod: Found airgap registry information" kubectl get secret "{{repl ImagePullSecretName }}" -o=jsonpath="{.data['\.dockerconfigjson']}" | base64 -d | yq -P - > registry-auth-airgap.yaml fi - if [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ]; - then + # if we successfully cached the airgap registry key, append it to the credentials we use for built in or external registry + if [ -f ./registry-auth-airgap.yaml ] && [ '{{repl ConfigOptionEquals "reg_incluster" "0" }}' = "true" ]; then + echo "Gitpod: Appending airgap registry to external registry list" kubectl get secret "container-registry" -o=jsonpath="{.data['\.dockerconfigjson']}" | base64 -d | yq -P - > registry-auth-external.yaml + yq -o=json eval-all 'select(fi == 0) * select(filename == "registry-auth-airgap.yaml")' registry-auth-external.yaml registry-auth-airgap.yaml > merged-registry-auths.json - if [[ -f ./registry-auth-airgap.yaml ]] - then - yq -o=json eval-all 'select(fi == 0) * select(filename == "registry-auth-airgap.yaml")' registry-auth-external.yaml registry-auth-airgap.yaml > merged-registry-auths.json - # create secret and update if exists (https://stackoverflow.com/a/45881259/1364435) - kubectl create secret generic container-registry \ - --save-config --dry-run=client \ - --from-file=.dockerconfigjson=./merged-registry-auths.json \ - -o yaml | kubectl apply -f - - fi - else - yq eval 'select(.kind == "Secret" and .metadata.name == "builtin-registry-auth").data.".dockerconfigjson"' \ - "${GITPOD_OBJECTS}/templates/gitpod.yaml" | base64 -d | yq -P - > registry-auth-builtin.yaml + kubectl create secret generic container-registry \ + --save-config --dry-run=client \ + --from-file=.dockerconfigjson=./merged-registry-auths.json \ + -o yaml | kubectl apply -f - - echo " ------------builtin auth output" - cat registry-auth-builtin.yaml + elif [ -f ./registry-auth-airgap.yaml ]; then + echo "Gitpod: Appending airgap registry to internal registry list" + yq eval 'select(.kind == "Secret" and .metadata.name == "builtin-registry-auth").data.".dockerconfigjson"' \ + "${GITPOD_OBJECTS}/templates/gitpod.yaml" | base64 -d | yq -P - > registry-auth-builtin.yaml - if [[ -f ./registry-auth-airgap.yaml ]] - then - echo " ------------mirror output" - cat registry-auth-airgap.yaml - yq -o=json eval-all 'select(fi == 0) * select(filename == "registry-auth-airgap.yaml")' registry-auth-builtin.yaml registry-auth-airgap.yaml > merged-registry-auths.json - echo " ------------merged" - cat merged-registry-auths.json - # create secret and update if exists (https://stackoverflow.com/a/45881259/1364435) - export NEW_SECRETS=$(base64 -w 0 merged-registry-auths.json) - yq e -i "select(.kind == \"Secret\" and .metadata.name == \"builtin-registry-auth\").data.\".dockerconfigjson\" = \"${NEW_SECRETS}\"" \ - "${GITPOD_OBJECTS}/templates/gitpod.yaml" - yq e 'select(.kind == "Secret" and .metadata.name == "builtin-registry-auth")' "${GITPOD_OBJECTS}/templates/gitpod.yaml" - fi + yq -o=json eval-all 'select(fi == 0) * select(filename == "registry-auth-airgap.yaml")' registry-auth-builtin.yaml registry-auth-airgap.yaml > merged-registry-auths.json + export NEW_SECRETS=$(base64 -w 0 merged-registry-auths.json) + yq e -i "select(.kind == \"Secret\" and .metadata.name == \"builtin-registry-auth\").data.\".dockerconfigjson\" = \"${NEW_SECRETS}\"" \ + "${GITPOD_OBJECTS}/templates/gitpod.yaml" fi echo "Gitpod: Escape any Golang template values"