Improve helm secrets template

packages/grid/devspace.yaml

@@ -74,16 +74,11 @@ deployments:
         global:
           registry: ${CONTAINER_REGISTRY}
           version: dev-${DEVSPACE_TIMESTAMP}
-          useDefaultSecrets: true
-        registry:
-          storageSize: "5Gi"
         node:
           name: ${NODE_NAME}
-          rootEmail: info@openmined.org
-          defaultWorkerPoolCount: 1
-          resourcesPreset: micro
-        veilid:
-          enabled: true
+      # anything that does not need devspace $env vars should go in values.dev.yaml
+      valuesFiles:
+        - ./helm/values.dev.yaml
 
 dev:
   mongo:

packages/grid/helm/syft/templates/_secrets.tpl

@@ -24,17 +24,24 @@ Params:
 {{- end -}}
 
 {{/*
-Re-use or set a new randomly generated secret value from an existing secret.
-If global.useDefaultSecrets is set to true, the default value will be used if the secret does not exist.
+Set a value for a Secret.
+- If the secret exists, the existing value will be re-used.
+- If "randomDefault"=true, a random value will be generated.
+- If "randomDefault"=false, the "default" value will be used.
 
 Usage:
-  {{- include "common.secrets.set " (dict "secret" "some-secret-name" "default" "default-value" "context" $ ) }}
+  Generate random secret of length 64
+  {{- include "common.secrets.set " (dict "secret" "some-secret-name" "randomDefault" true "randomLength" 64 "context" $ ) }}
+
+  Use a static default value (with random disabled)
+  {{- include "common.secrets.set " (dict "secret" "some-secret-name" "default" "default-value" "randomDefault" false "context" $ ) }}
 
 Params:
   secret - String (Required) - Name of the 'Secret' resource where the key is stored.
   key - String - (Required) - Name of the key in the secret.
-  default - String - (Optional) - Default value to use if the secret does not exist.
-  length - Int - (Optional) - The length of the generated secret. Default is 32.
+  randomDefault - Bool - (Optional) - If true, a random value will be generated if secret does note exit.
+  randomLength - Int - (Optional) - The length of the generated secret. Default is 32.
+  default - String - (Optional) - Default value to use if the secret does not exist if "randomDefault" is set to false.
   context - Context (Required) - Parent context.
 */}}
 {{- define "common.secrets.set" -}}
@@ -43,11 +50,11 @@ Params:
 
   {{- if $existingSecret -}}
     {{- $secretVal = $existingSecret -}}
-  {{- else if .context.Values.global.useDefaultSecrets -}}
-    {{- $secretVal = .default | b64enc -}}
-  {{- else -}}
-    {{- $length := .length | default 32 -}}
+  {{- else if .randomDefault -}}
+    {{- $length := .randomLength | default 32 -}}
     {{- $secretVal = randAlphaNum $length | b64enc -}}
+  {{- else -}}
+    {{- $secretVal = .default | required (printf "default value required for secret=%s key=%s" .secret .key) |b64enc -}}
   {{- end -}}
 
   {{- printf "%s" $secretVal -}}

packages/grid/helm/syft/templates/backend/backend-secret.yaml

@@ -11,6 +11,7 @@ data:
   defaultRootPassword: {{ include "common.secrets.set" (dict
     "secret" $secretName
     "key" "defaultRootPassword"
-    "default" .Values.node.defaultSecret.defaultRootPassword
+    "randomDefault" .Values.global.randomizedSecrets
+    "default" .Values.node.secret.defaultRootPassword
     "context" $)
   }}

packages/grid/helm/syft/templates/mongo/mongo-secret.yaml

@@ -11,6 +11,7 @@ data:
   rootPassword: {{ include "common.secrets.set" (dict
     "secret" $secretName
     "key" "rootPassword"
-    "default" .Values.mongo.defaultSecret.rootPassword
+    "randomDefault" .Values.global.randomizedSecrets
+    "default" .Values.mongo.secret.rootPassword
     "context" $)
   }}

packages/grid/helm/syft/templates/seaweedfs/seaweedfs-secret.yaml

@@ -12,7 +12,8 @@ data:
   s3RootPassword: {{ include "common.secrets.set" (dict
     "secret" $secretName
     "key" "s3RootPassword"
-    "default" .Values.seaweedfs.defaultSecret.s3RootPassword
+    "randomDefault" .Values.global.randomizedSecrets
+    "default" .Values.seaweedfs.secret.s3RootPassword
     "context" $)
   }}
 {{ end }}

packages/grid/helm/syft/templates/veilid/veilid-deployment.yaml

@@ -27,12 +27,14 @@ spec:
           resources: {{ include "common.resources.set" (dict "resources" .Values.veilid.resources "preset" .Values.veilid.resourcesPreset) | nindent 12 }}
 
           env:
-            - name: VEILID_FLAGS
-              value: {{ .Values.veilid.serverFlags | quote }}
             - name: UVICORN_LOG_LEVEL
               value: {{ .Values.veilid.uvicornLogLevel }}
             - name: APP_LOG_LEVEL
               value: {{ .Values.veilid.appLogLevel }}
+            {{- if .Values.veilid.serverFlags }}
+            - name: VEILID_FLAGS
+              value: {{ .Values.veilid.serverFlags | quote }}
+            {{- end }}
             {{- if .Values.veilid.env }}
             {{- toYaml .Values.veilid.env | nindent 12 }}
             {{- end }}

packages/grid/helm/syft/templates/veilid/veilid-service.yaml

@@ -13,7 +13,7 @@ spec:
     app.kubernetes.io/component: veilid
   ports:
     - name: python-server
-      port: 80
       protocol: TCP
+      port: 80
       targetPort: 4000
-{{ end }}
+{{ end }}

packages/grid/helm/syft/values.yaml

@@ -3,8 +3,8 @@ global:
   registry: docker.io
   version: 0.8.5-beta.9
 
-  # Force default secret values for development. DO NOT USE IN PRODUCTION
-  useDefaultSecrets: false
+  # Force default secret values for development. DO NOT SET THIS TO FALSE IN PRODUCTION
+  randomizedSecrets: true
 
 mongo:
   # MongoDB config
@@ -24,9 +24,9 @@ mongo:
   # Mongo secret name. Override this if you want to use a self-managed secret.
   secretKeyName: mongo-secret
 
-  # Dev mode default passwords
-  defaultSecret:
-    rootPassword: example
+  # custom secret values
+  secret:
+    rootPassword: null
 
 frontend:
   # Extra environment vars
@@ -62,9 +62,9 @@ seaweedfs:
   resourcesPreset: nano
   resources: null
 
-  # Dev mode default passwords
-  defaultSecret:
-    s3RootPassword: admin
+  # custom secret values
+  secret:
+    s3RootPassword: null
 
 proxy:
   # Extra environment vars
@@ -122,9 +122,9 @@ node:
   # - defaultRootPassword
   secretKeyName: backend-secret
 
-  # Dev mode default passwords
-  defaultSecret:
-    defaultRootPassword: changethis
+  # custom secret values
+  secret:
+    defaultRootPassword: null
 
 ingress:
   hostname: null # do not make this localhost
@@ -152,7 +152,7 @@ ingress:
 # ----------------------------------------
 veilid:
   enabled: false
-  serverFlags: ""
+  serverFlags: null
   appLogLevel: "info"
   uvicornLogLevel: "info"
 
@@ -161,4 +161,4 @@ veilid:
 
   # Pod Resource Limits
   resourcesPreset: nano
-  resources: null
+  resources: null

packages/grid/helm/values.dev.yaml

@@ -0,0 +1,26 @@
+# Helm chart values used for development and testing
+# Can be used through `helm install -f values.dev.yaml` or devspace `valuesFiles`
+
+global:
+  randomizedSecrets: false
+
+registry:
+  storageSize: "5Gi"
+
+node:
+  rootEmail: info@openmined.org
+  defaultWorkerPoolCount: 1
+
+  secret:
+    defaultRootPassword: changethis
+
+mongo:
+  secret:
+    rootPassword: example
+
+seaweedfs:
+  secret:
+    s3RootPassword: admin
+
+veilid:
+  enabled: true

packages/grid/veilid/start.sh

@@ -16,6 +16,6 @@ then
     RELOAD="--reload"
 fi
 
-/veilid/veilid-server -c /veilid/veilid-server.conf  $VEILID_FLAGS &
+/veilid/veilid-server -c /veilid/veilid-server.conf $VEILID_FLAGS &
 
 exec uvicorn $RELOAD --host $HOST --port $PORT --log-level $UVICORN_LOG_LEVEL "$APP_MODULE"

tox.ini

@@ -797,11 +797,9 @@ commands =
 
 [testenv:syft.test.helm]
 description = Test Helm Chart for Kubernetes
-changedir = {toxinidir}
-passenv=HOME,USER,EXTERNAL_REGISTRY_USERNAME,EXTERNAL_REGISTRY_PASSWORD
+changedir = {toxinidir}/packages/grid
+passenv=HOME, USER, EXTERNAL_REGISTRY_USERNAME, EXTERNAL_REGISTRY_PASSWORD
 allowlist_externals =
-    grep
-    sleep
     bash
     tox
 setenv =
@@ -811,59 +809,38 @@ setenv =
     EXCLUDE_NOTEBOOKS = {env:EXCLUDE_NOTEBOOKS:not 10-container-images.ipynb}
     SYFT_VERSION = {env:SYFT_VERSION:local}
     EXTERNAL_REGISTRY = {env:EXTERNAL_REGISTRY:k3d-registry.localhost:5800}
+    ; env vars for dev.k8s.start
+    CLUSTER_NAME = testdomain
+    CLUSTER_HTTP_PORT = {env:NODE_PORT:8080}
 commands =
-    bash -c "echo Running with ORCHESTRA_DEPLOYMENT_TYPE=$ORCHESTRA_DEPLOYMENT_TYPE NODE_PORT=$NODE_PORT NODE_URL=$NODE_URL \
-    Excluding notebooks: $EXCLUDE_NOTEBOOKS SYFT_VERSION=$SYFT_VERSION \
-    EXTERNAL_REGISTRY=$EXTERNAL_REGISTRY; date"
+    bash -c "env; date; k3d version"
 
+    bash -c "k3d cluster delete ${CLUSTER_NAME} || true"
 
-    bash -c "k3d version"
+    tox -e dev.k8s.start
 
-    # Remvoing old clusters and volumes and registry
-    ; bash -c "docker rm $(docker ps -aq) --force || true"
-    bash -c "k3d cluster delete syft || true"
-    bash -c "docker volume rm k3d-syft-images --force || true"
-    bash -c "k3d registry delete k3d-registry.localhost || true"
-
-    # Creating registry
-    bash -c '\
-        export CLUSTER_NAME=syft CLUSTER_HTTP_PORT=${NODE_PORT} && \
-        tox -e dev.k8s.start'
-
-    # Creating registry and cluster
-    bash -c 'NODE_NAME=syft NODE_PORT=${NODE_PORT} && \
-        k3d cluster create syft -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
-        k3d cluster start syft'
-
-    sleep 10
-    bash -c "kubectl --context k3d-syft create namespace syft || true"
-
-    # if syft version is local, then install local helm charts
-    # else install the helm charts from the openmined gh-pages branch
     bash -c 'if [[ $SYFT_VERSION == "local" ]]; then \
         echo "Installing local helm charts"; \
-        bash -c "cd packages/grid/helm && helm install --kube-context k3d-syft --namespace syft syft ./syft --set global.useDefaultSecrets=true"; \
+        helm install ${CLUSTER_NAME} ./helm/syft -f ./helm/values.dev.yaml --kube-context k3d-${CLUSTER_NAME} --namespace syft --create-namespace; \
     else \
         echo "Installing helm charts from repo for syft version: ${SYFT_VERSION}"; \
-        bash -c "helm repo add openmined https://openmined.github.io/PySyft/helm && helm repo update openmined"; \
-        bash -c "helm install --kube-context k3d-syft --namespace syft syft openmined/syft --version=${SYFT_VERSION} --set global.useDefaultSecrets=true"; \
+        helm repo add openmined https://openmined.github.io/PySyft/helm; \
+        helm repo update openmined; \
+        helm install ${CLUSTER_NAME} openmined/syft --version=${SYFT_VERSION} -f ./helm/values.dev.yaml --kube-context k3d-${CLUSTER_NAME} --namespace syft --create-namespace; \
     fi'
 
     ; wait for everything else to be loaded
-    bash packages/grid/scripts/wait_for.sh service frontend --context k3d-syft --namespace syft
-    bash -c '(kubectl logs service/frontend --context k3d-syft --namespace syft -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
-    bash packages/grid/scripts/wait_for.sh service mongo --context k3d-syft --namespace syft
-    bash packages/grid/scripts/wait_for.sh service backend --context k3d-syft --namespace syft
-    bash packages/grid/scripts/wait_for.sh service proxy --context k3d-syft --namespace syft
-    bash -c '(kubectl logs service/backend --context k3d-syft --namespace syft -f &) | grep -q "Application startup complete" || true'
-
+    bash -c './scripts/wait_for.sh service frontend --context k3d-$CLUSTER_NAME --namespace syft'
+    bash -c '(kubectl logs service/frontend --context k3d-$CLUSTER_NAME --namespace syft -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
+    bash -c './scripts/wait_for.sh service mongo --context k3d-$CLUSTER_NAME --namespace syft'
+    bash -c './scripts/wait_for.sh service backend --context k3d-$CLUSTER_NAME --namespace syft'
+    bash -c './scripts/wait_for.sh service proxy --context k3d-$CLUSTER_NAME --namespace syft'
+    bash -c '(kubectl logs service/backend --context k3d-$CLUSTER_NAME --namespace syft -f &) | grep -q "Application startup complete" || true'
 
     # Run Notebook tests
     tox -e e2e.test.notebook
 
-    # Cleanup
-    bash -c "k3d cluster delete syft || true"
-    bash -c "docker volume rm k3d-syft-images --force || true"
+    bash -c "k3d cluster delete ${CLUSTER_NAME} || true"
 
 [testenv:syft.test.helm.upgrade]
 description = Test helm upgrade
@@ -925,7 +902,7 @@ commands =
 [testenv:dev.k8s.start]
 description = Start local Kubernetes registry & cluster with k3d
 changedir = {toxinidir}
-passenv = *
+passenv = HOME, USER
 setenv =
     CLUSTER_NAME = {env:CLUSTER_NAME:syft-dev}
     CLUSTER_HTTP_PORT = {env:CLUSTER_HTTP_PORT:8080}
@@ -1017,7 +994,7 @@ commands =
     ; destroy cluster
     bash -c '\
         rm -rf .devspace; echo ""; \
-        k3d cluster delete ${CLUSTER_NAME}'
+        k3d cluster delete ${CLUSTER_NAME};'
 
 [testenv:dev.k8s.destroyall]
 description = Destroy both local Kubernetes cluster and registry
@@ -1103,7 +1080,7 @@ commands =
 
     # If the syft version is local install the local version
     # else install the version of syft specified
-    bash -c " if [[ $SYFT_VERSION == 'local' ]]; then \
+    bash -c "if [[ $SYFT_VERSION == 'local' ]]; then \
         echo 'Using local syft'; \
     else \
         echo 'Installing syft version: ${SYFT_VERSION}'; \