[K8s] Parametrize tox task

.github/workflows/pr-tests-stack.yml

@@ -645,10 +645,10 @@ jobs:
         shell: bash
         run: |
           mkdir -p ./k8s-logs
-          kubectl describe all -A --context k3d-testgateway1 --namespace testgateway1 > ./k8s-logs/testgateway1-desc-${{ steps.date.outputs.date }}.txt
-          kubectl describe all -A --context k3d-testdomain1 --namespace testdomain1 > ./k8s-logs/testdomain1-desc-${{ steps.date.outputs.date }}.txt
-          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testgateway1 --namespace testgateway1 > ./k8s-logs/testgateway1-logs-${{ steps.date.outputs.date }}.txt
-          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testdomain1 --namespace testdomain1 > ./k8s-logs/testdomain1-logs-${{ steps.date.outputs.date }}.txt
+          kubectl describe all -A --context k3d-testgateway1 --namespace syft > ./k8s-logs/testgateway1-desc-${{ steps.date.outputs.date }}.txt
+          kubectl describe all -A --context k3d-testdomain1 --namespace syft > ./k8s-logs/testdomain1-desc-${{ steps.date.outputs.date }}.txt
+          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testgateway1 --namespace syft > ./k8s-logs/testgateway1-logs-${{ steps.date.outputs.date }}.txt
+          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-testdomain1 --namespace syft > ./k8s-logs/testdomain1-logs-${{ steps.date.outputs.date }}.txt
           ls -la ./k8s-logs
 
       - name: Upload logs to GitHub

scripts/get_k8s_secret_ci.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
-export SYFT_LOGIN_testgateway1_PASSWORD=$(kubectl --context=k3d-testgateway1 get secret syft-default-secret -n testgateway1 \
+export SYFT_LOGIN_testgateway1_PASSWORD=$(kubectl --context=k3d-testgateway1 get secret syft-default-secret -n syft \
     -o jsonpath='{.data.defaultRootPassword}' | base64 --decode)
 
-export SYFT_LOGIN_testdomain1_PASSWORD=$(kubectl get --context=k3d-testdomain1 secret syft-default-secret -n testdomain1 \
+export SYFT_LOGIN_testdomain1_PASSWORD=$(kubectl get --context=k3d-testdomain1 secret syft-default-secret -n syft \
     -o jsonpath='{.data.defaultRootPassword}' | base64 --decode)

tox.ini

@@ -690,49 +690,25 @@ commands =
     bash -c "k3d cluster delete testgateway1 || true"
     bash -c "k3d cluster delete testdomain1 || true"
 
-    # Deleting registery & volumes
+    # Deleting registry & volumes
     bash -c "k3d registry delete k3d-registry.localhost || true"
     bash -c "docker volume rm k3d-testgateway1-images --force || true"
     bash -c "docker volume rm k3d-testdomain1-images --force || true"
 
-
-    # Creating registry
-    bash -c 'k3d registry create registry.localhost --port 5800 -v `pwd`/k3d-registry:/var/lib/registry || true'
+    # Create registry
+    tox -e dev.k8s.registry
 
     # Creating testgateway1 cluster on port 9081
-    bash -c 'NODE_NAME=testgateway1 NODE_PORT=9081 && \
-        k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
-        k3d cluster start $NODE_NAME'
-
-    bash -c 'NODE_NAME=testgateway1 NODE_PORT=9081 && \
-        cd packages/grid && \
-        (r=5;while ! \
-        devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
-        -p gateway \
-        --var NODE_NAME=$NODE_NAME \
-        --var TEST_MODE=1 \
-        --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 \
-        --var NODE_TYPE=gateway \
-        deploy -b; \
-        do ((--r))||exit;echo "retrying" && sleep 20;done)'
+    bash -c '\
+        export CLUSTER_NAME=testgateway1 CLUSTER_HTTP_PORT=9081 DEVSPACE_PROFILE=gateway && \
+        tox -e dev.k8s.start && \
+        tox -e dev.k8s.deploy'
 
     # Creating testdomain1 cluster on port 9082
-    bash -c 'NODE_NAME=testdomain1 NODE_PORT=9082 && \
-        k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
-        k3d cluster start $NODE_NAME'
-
-    # Patches CoreDNS
-    tox -e dev.k8s.patch.coredns
-
-    bash -c 'NODE_NAME=testdomain1 NODE_PORT=9082 && \
-        cd packages/grid && \
-        (r=5;while ! \
-        devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
-        --var NODE_NAME=$NODE_NAME \
-        --var TEST_MODE=1 \
-        --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 \
-        deploy -b; \
-        do ((--r))||exit;echo "retrying" && sleep 20;done)'
+    bash -c '\
+        export CLUSTER_NAME=testdomain1 CLUSTER_HTTP_PORT=9082 && \
+        tox -e dev.k8s.start && \
+        tox -e dev.k8s.deploy'
 
     # free up build cache after build of images
     bash -c 'if [[ "$GITHUB_CI" != "false" ]]; then \
@@ -743,25 +719,24 @@ commands =
     sleep 30
 
     # wait for front end
-    bash packages/grid/scripts/wait_for.sh service frontend --context k3d-testdomain1 --namespace testdomain1
-    bash -c '(kubectl logs service/frontend --context k3d-testdomain1 --namespace testdomain1 -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
+    bash packages/grid/scripts/wait_for.sh service frontend --context k3d-testdomain1 --namespace syft
+    bash -c '(kubectl logs service/frontend --context k3d-testdomain1 --namespace syft -f &) | grep -q -E "Network:\s+https?://[a-zA-Z0-9.-]+:[0-9]+/" || true'
 
     # wait for test gateway 1
-    bash packages/grid/scripts/wait_for.sh service mongo --context k3d-testgateway1 --namespace testgateway1
-    bash packages/grid/scripts/wait_for.sh service backend --context k3d-testgateway1 --namespace testgateway1
-    bash packages/grid/scripts/wait_for.sh service proxy --context k3d-testgateway1 --namespace testgateway1
+    bash packages/grid/scripts/wait_for.sh service mongo --context k3d-testgateway1 --namespace syft
+    bash packages/grid/scripts/wait_for.sh service backend --context k3d-testgateway1 --namespace syft
+    bash packages/grid/scripts/wait_for.sh service proxy --context k3d-testgateway1 --namespace syft
 
     # wait for test domain 1
-    bash packages/grid/scripts/wait_for.sh service mongo --context k3d-testdomain1 --namespace testdomain1
-    bash packages/grid/scripts/wait_for.sh service backend --context k3d-testdomain1 --namespace testdomain1
-    bash packages/grid/scripts/wait_for.sh service proxy --context k3d-testdomain1 --namespace testdomain1
-    bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-testdomain1 --namespace testdomain1
+    bash packages/grid/scripts/wait_for.sh service mongo --context k3d-testdomain1 --namespace syft
+    bash packages/grid/scripts/wait_for.sh service backend --context k3d-testdomain1 --namespace syft
+    bash packages/grid/scripts/wait_for.sh service proxy --context k3d-testdomain1 --namespace syft
+    bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-testdomain1 --namespace syft
 
     # Checking logs generated & startup of test-domain 1
-    bash -c '(kubectl logs service/backend --context k3d-testdomain1 --namespace testdomain1 -f &) | grep -q "Application startup complete" || true'
+    bash -c '(kubectl logs service/backend --context k3d-testdomain1 --namespace syft -f &) | grep -q "Application startup complete" || true'
     # Checking logs generated & startup of testgateway1
-    bash -c '(kubectl logs service/backend --context k3d-testgateway1 --namespace testgateway1 -f &) | grep -q "Application startup complete" || true'
-
+    bash -c '(kubectl logs service/backend --context k3d-testgateway1 --namespace syft -f &) | grep -q "Application startup complete" || true'
 
     # frontend
     bash -c 'if [[ "$PYTEST_MODULES" == *"frontend"* ]]; then \
@@ -773,18 +748,18 @@ commands =
         exit $return; \
     fi'
 
-    #Integration + Gateway Connection Tests
+    # Integration + Gateway Connection Tests
     # Gateway tests are not run in kuberetes, as currently,it does not have a way to configure
     # high/low side warning flag.
-    bash -c " source ./scripts/get_k8s_secret_ci.sh; \
+    bash -c "source ./scripts/get_k8s_secret_ci.sh; \
     pytest tests/integration/network -k 'not test_domain_gateway_user_code' -p no:randomly -vvvv"
 
     # Shutting down the gateway cluster to free up space, as the
     # below code does not require gateway cluster
-    bash -c "k3d cluster delete testgateway1 || true"
+    bash -c "CLUSTER_NAME=testgateway1 tox -e dev.k8s.destroy || true"
     bash -c "docker volume rm k3d-testgateway1-images --force || true"
 
-    ; ; container workload
+    ; container workload
     ; bash -c 'if [[ "$PYTEST_MODULES" == *"container_workload"* ]]; then \
     ;     echo "Starting Container Workload test"; date; \
     ;     pytest tests/integration -m container_workload -p no:randomly --co; \
@@ -794,27 +769,16 @@ commands =
     ;     exit $return; \
     ; fi'
 
-    # Since we randomize the password, we retrieve them and store as environment variables
-    # which would then be used by the notebook
-
-
-    # ignore 06 because of opendp on arm64
-    # Run 0.8 notebooks
-
-    bash -c " source ./scripts/get_k8s_secret_ci.sh; \
-     pytest --nbmake notebooks/api/0.8 -p no:randomly -k 'not 10-container-images.ipynb' -vvvv --nbmake-timeout=1000"
-
+    bash -c "source ./scripts/get_k8s_secret_ci.sh; \
+        pytest --nbmake notebooks/api/0.8 -p no:randomly -k 'not 10-container-images.ipynb' -vvvv --nbmake-timeout=1000"
 
     # deleting clusters created
-    bash -c "k3d cluster delete testgateway1 || true"
-    bash -c "k3d cluster delete testdomain1 || true"
+    bash -c "CLUSTER_NAME=testdomain1 tox -e dev.k8s.destroy || true"
     bash -c "k3d registry delete k3d-registry.localhost || true"
     bash -c "docker rm $(docker ps -aq) --force || true"
-    bash -c "docker volume rm k3d-testgateway1-images --force || true"
     bash -c "docker volume rm k3d-testdomain1-images --force || true"
 
 
-
 [testenv:syft.build.helm]
 description = Build Helm Chart for Kubernetes
 changedir = {toxinidir}
@@ -982,7 +946,10 @@ commands =
 [testenv:dev.k8s.start]
 description = Start local Kubernetes registry & cluster with k3d
 changedir = {toxinidir}
-passenv=*
+passenv = *
+setenv =
+    CLUSTER_NAME = {env:CLUSTER_NAME:syft-dev}
+    CLUSTER_HTTP_PORT = {env:CLUSTER_HTTP_PORT:8080}
 allowlist_externals =
     bash
     sleep
@@ -992,8 +959,8 @@ commands =
     tox -e dev.k8s.registry
 
     ; for NodePort to work add the following --> -p "NodePort:NodePort@loadbalancer"
-    bash -c 'k3d cluster create syft-dev -p "8080:80@loadbalancer" --registry-use k3d-registry.localhost:5800; \
-        kubectl create namespace syft || true'
+    bash -c 'k3d cluster create ${CLUSTER_NAME} -p "${CLUSTER_HTTP_PORT}:80@loadbalancer" --registry-use k3d-registry.localhost:5800 && \
+        kubectl --context k3d-${CLUSTER_NAME} create namespace syft || true'
 
     ; patch coredns
     tox -e dev.k8s.patch.coredns
@@ -1004,34 +971,39 @@ commands =
 [testenv:dev.k8s.deploy]
 description = Deploy Syft to a local Kubernetes cluster with Devspace
 changedir = {toxinidir}/packages/grid
-passenv=HOME, USER
+passenv = HOME, USER, DEVSPACE_PROFILE
+setenv=
+    CLUSTER_NAME = {env:CLUSTER_NAME:syft-dev}
 allowlist_externals =
-    tox
     bash
 commands =
     ; deploy syft helm charts
-    bash -c 'devspace deploy -b --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800'
+    bash -c '\
+        if [[ -n "${DEVSPACE_PROFILE}" ]]; then export DEVSPACE_PROFILE="-p ${DEVSPACE_PROFILE}"; fi && \
+        devspace deploy -b --kube-context k3d-${CLUSTER_NAME} --no-warn ${DEVSPACE_PROFILE} --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800'
 
 [testenv:dev.k8s.hotreload]
 description = Start development with hot-reload in Kubernetes
 changedir = {toxinidir}/packages/grid
-passenv=HOME, USER
+passenv = HOME, USER, DEVSPACE_PROFILE
+setenv=
+    CLUSTER_NAME = {env:CLUSTER_NAME:syft-dev}
 allowlist_externals =
     bash
-    tox
 commands =
     ; deploy syft helm charts with hot-reload
-    bash -c 'devspace dev --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800'
+    bash -c '\
+        if [[ -n "${DEVSPACE_PROFILE}" ]]; then export DEVSPACE_PROFILE="-p ${DEVSPACE_PROFILE}"; fi && \
+        devspace dev --kube-context k3d-${CLUSTER_NAME} --no-warn ${DEVSPACE_PROFILE} --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800'
 
 [testenv:dev.k8s.info]
 description = Gather info about the localKubernetes cluster
-passenv=HOME, USER
+passenv = HOME, USER
 ignore_errors = True
 allowlist_externals =
     k3d
     kubectl
 commands =
-    kubectl config view
     k3d cluster list
     kubectl cluster-info
     kubectl config current-context
@@ -1041,22 +1013,21 @@ commands =
 description = Cleanup Syft deployment and associated resources, but keep the cluster running
 changedir = {toxinidir}/packages/grid
 passenv=HOME, USER
+setenv=
+    CLUSTER_NAME = {env:CLUSTER_NAME:syft-dev}
 allowlist_externals =
     bash
 commands =
-    bash -c 'devspace purge --force-purge --kube-context k3d-syft-dev --namespace syft; sleep 3'
-    bash -c 'devspace cleanup images --kube-context k3d-syft-dev --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 || true'
-    bash -c 'kubectl config use-context k3d-syft-dev'
-    bash -c 'kubectl delete all --all --namespace syft || true'
-    bash -c 'kubectl delete pvc --all --namespace syft || true'
-    bash -c 'kubectl delete secret --all --namespace syft || true'
-    bash -c 'kubectl delete configmap --all --namespace syft || true'
-    bash -c 'kubectl delete serviceaccount --all --namespace syft || true'
+    bash -c 'devspace purge --force-purge --kube-context k3d-${CLUSTER_NAME} --no-warn --namespace syft; sleep 3'
+    bash -c 'devspace cleanup images --kube-context k3d-${CLUSTER_NAME} --no-warn --namespace syft --var CONTAINER_REGISTRY=k3d-registry.localhost:5800 || true'
+    bash -c 'kubectl --context k3d-${CLUSTER_NAME} delete namespace syft --now=true || true'
 
 [testenv:dev.k8s.destroy]
 description = Destroy local Kubernetes cluster
 changedir = {toxinidir}/packages/grid
-passenv=HOME, USER
+passenv = HOME, USER
+setenv=
+    CLUSTER_NAME = {env:CLUSTER_NAME:syft-dev}
 allowlist_externals =
     tox
     bash
@@ -1067,18 +1038,16 @@ commands =
     ; destroy cluster
     bash -c '\
         rm -rf .devspace; echo ""; \
-        k3d cluster delete syft-dev; echo ""; \
-        kubectl config view'
+        k3d cluster delete ${CLUSTER_NAME}'
 
 [testenv:dev.k8s.destroyall]
 description = Destroy both local Kubernetes cluster and registry
 changedir = {toxinidir}
-passenv=HOME, USER
+passenv = HOME, USER, CLUSTER_NAME
 ignore_errors=True
 allowlist_externals =
     bash
     tox
-    rm
 commands =
     ; destroy cluster
     tox -e dev.k8s.destroy