From 621bb284bbfcc9c51b1ac575daec7770fce52b36 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 9 May 2026 13:05:56 +0200 Subject: [PATCH 1/5] ci: add LocalStack-on-Kubernetes workflow (k3d + kubectl + Lambda pod executor) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Renames ls-k8s/ → ls-on-k8s/ - Adds .github/workflows/test-ls-on-k8s.yml: spins up a k3d cluster, deploys LocalStack via kubectl apply, runs Lambda integration tests, tears the cluster down; reads auth token from K8S_LOCALSTACK_AUTH_TOKEN secret - Adds k8s/localstack.yaml: ServiceAccount, Role, RoleBinding, Deployment, NodePort Service; auth token injected via optional k8s Secret - cluster-up.sh: creates k3d cluster, optional auth secret, kubectl apply, waits for health endpoint - Makefile: kubectl-based deploy-ls target; logs/logs-follow split to avoid CI hang from -f flag Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/test-ls-on-k8s.yml | 55 ++++++++ ls-k8s/k8s/values.yaml | 26 ---- {ls-k8s => ls-on-k8s}/Makefile | 16 +-- ls-on-k8s/k8s/localstack.yaml | 129 ++++++++++++++++++ {ls-k8s => ls-on-k8s}/scripts/cluster-down.sh | 0 {ls-k8s => ls-on-k8s}/scripts/cluster-up.sh | 28 ++-- {ls-k8s => ls-on-k8s}/tests/requirements.txt | 0 {ls-k8s => ls-on-k8s}/tests/test_lambda.py | 0 8 files changed, 207 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/test-ls-on-k8s.yml delete mode 100644 ls-k8s/k8s/values.yaml rename {ls-k8s => ls-on-k8s}/Makefile (74%) create mode 100644 ls-on-k8s/k8s/localstack.yaml rename {ls-k8s => ls-on-k8s}/scripts/cluster-down.sh (100%) rename {ls-k8s => ls-on-k8s}/scripts/cluster-up.sh (66%) rename {ls-k8s => ls-on-k8s}/tests/requirements.txt (100%) rename {ls-k8s => ls-on-k8s}/tests/test_lambda.py (100%) diff --git a/.github/workflows/test-ls-on-k8s.yml b/.github/workflows/test-ls-on-k8s.yml new file mode 100644 index 0000000..afbf510 --- /dev/null +++ b/.github/workflows/test-ls-on-k8s.yml @@ -0,0 +1,55 @@ +name: LocalStack on Kubernetes + +on: + pull_request: + branches: [main] + paths: + - 'ls-on-k8s/**' + - '.github/workflows/test-ls-on-k8s.yml' + push: + branches: [main] + paths: + - 'ls-on-k8s/**' + - '.github/workflows/test-ls-on-k8s.yml' + workflow_dispatch: + +env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.K8S_LOCALSTACK_AUTH_TOKEN }} + LOCALSTACK_DISABLE_EVENTS: "1" + +jobs: + test-ls-on-k8s: + name: k3d + LocalStack + Lambda pod executor + runs-on: ubuntu-latest + timeout-minutes: 25 + defaults: + run: + working-directory: ls-on-k8s + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install k3d + run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + + - name: Spin up cluster and deploy LocalStack + run: make cluster-up + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Run Lambda integration tests + run: make test + + - name: Dump LocalStack logs on failure + if: failure() + run: | + make logs 2>/dev/null || true + make pods 2>/dev/null || true + + - name: Tear down cluster + if: always() + run: make cluster-down diff --git a/ls-k8s/k8s/values.yaml b/ls-k8s/k8s/values.yaml deleted file mode 100644 index df9b58d..0000000 --- a/ls-k8s/k8s/values.yaml +++ /dev/null @@ -1,26 +0,0 @@ -replicaCount: 1 - -image: - repository: localstack/localstack - tag: "latest" - pullPolicy: IfNotPresent - -# Role + ServiceAccount give LocalStack permission to create/watch pods, -# which is required by the Kubernetes Lambda executor. -serviceAccount: - create: true - -role: - create: true - -service: - type: NodePort - edgeService: - name: edge - targetPort: 4566 - nodePort: 31566 - -# Use the Kubernetes executor so every Lambda invocation runs in its own pod. -lambdaExecutor: "kubernetes" - -debug: false diff --git a/ls-k8s/Makefile b/ls-on-k8s/Makefile similarity index 74% rename from ls-k8s/Makefile rename to ls-on-k8s/Makefile index 4e7da29..3c7d81b 100644 --- a/ls-k8s/Makefile +++ b/ls-on-k8s/Makefile @@ -18,13 +18,10 @@ cluster-up: ## Create k3d cluster and deploy LocalStack with Kubernetes Lambda e cluster-down: ## Tear down the k3d cluster @bash scripts/cluster-down.sh $(CLUSTER) -deploy-ls: ## Re-deploy / upgrade LocalStack (cluster must already be running) - helm repo update localstack - helm upgrade --install $(RELEASE) localstack/localstack \ - --kube-context $(CONTEXT) \ - --namespace $(NAMESPACE) \ - --values k8s/values.yaml \ - --wait --timeout 180s +deploy-ls: ## Re-deploy LocalStack (cluster must already be running) + kubectl --context $(CONTEXT) apply -f k8s/localstack.yaml + kubectl --context $(CONTEXT) rollout status deployment/$(RELEASE) \ + --namespace $(NAMESPACE) --timeout=180s $(VENV): python3 -m venv $(VENV) @@ -33,7 +30,10 @@ $(VENV): test: $(VENV) ## Run Lambda-on-Kubernetes integration tests $(VENV)/bin/pytest tests/ -v -s -logs: ## Tail LocalStack pod logs +logs: ## Dump LocalStack pod logs + kubectl --context $(CONTEXT) -n $(NAMESPACE) logs deploy/$(RELEASE) --tail=200 + +logs-follow: ## Tail LocalStack pod logs (interactive) kubectl --context $(CONTEXT) -n $(NAMESPACE) logs -f deploy/$(RELEASE) pods: ## List all pods in the LocalStack namespace diff --git a/ls-on-k8s/k8s/localstack.yaml b/ls-on-k8s/k8s/localstack.yaml new file mode 100644 index 0000000..451e514 --- /dev/null +++ b/ls-on-k8s/k8s/localstack.yaml @@ -0,0 +1,129 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: localstack + namespace: localstack +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: localstack + namespace: localstack +rules: + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["create", "delete", "get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: localstack + namespace: localstack +subjects: + - kind: ServiceAccount + name: localstack + namespace: localstack +roleRef: + kind: Role + name: localstack + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: localstack + namespace: localstack +spec: + replicas: 1 + selector: + matchLabels: + app: localstack + template: + metadata: + labels: + app: localstack + spec: + serviceAccountName: localstack + # Writes a kubeconfig with the actual token + CA embedded so LocalStack's + # k8s client connects to kubernetes.default.svc instead of localhost:80. + initContainers: + - name: gen-kubeconfig + image: busybox + command: + - sh + - -c + - | + TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + CA=$(base64 /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | tr -d '\n') + cat > /kube/config </dev/null | awk 'NR>1{print $1}' | grep -qx "${CLUSTER}"; echo " Cluster '${CLUSTER}' already exists – skipping creation." else k3d cluster create "${CLUSTER}" \ - --port "${HOST_PORT}:${NODE_PORT}@server[0]" \ + --port "${HOST_PORT}:${NODE_PORT}@server:0" \ --wait fi @@ -22,17 +21,20 @@ echo "==> Ensuring namespace '${NAMESPACE}'..." kubectl --context "${CONTEXT}" create namespace "${NAMESPACE}" \ --dry-run=client -o yaml | kubectl --context "${CONTEXT}" apply -f - -echo "==> Adding/updating LocalStack Helm repo..." -helm repo add localstack https://localstack.github.io/helm-charts 2>/dev/null || true -helm repo update localstack 2>/dev/null - -echo "==> Deploying LocalStack via Helm..." -helm upgrade --install "${RELEASE}" localstack/localstack \ - --kube-context "${CONTEXT}" \ - --namespace "${NAMESPACE}" \ - --values "${SCRIPT_DIR}/../k8s/values.yaml" \ - --wait \ - --timeout 180s +if [[ -n "${LOCALSTACK_AUTH_TOKEN:-}" ]]; then + echo "==> Creating auth token secret..." + kubectl --context "${CONTEXT}" create secret generic localstack-auth \ + --namespace "${NAMESPACE}" \ + --from-literal=token="${LOCALSTACK_AUTH_TOKEN}" \ + --dry-run=client -o yaml | kubectl --context "${CONTEXT}" apply -f - +fi + +echo "==> Deploying LocalStack..." +kubectl --context "${CONTEXT}" apply -f "${SCRIPT_DIR}/../k8s/localstack.yaml" + +echo "==> Waiting for LocalStack to be ready..." +kubectl --context "${CONTEXT}" rollout status deployment/localstack \ + --namespace "${NAMESPACE}" --timeout=180s echo "==> Waiting for LocalStack health endpoint on localhost:${HOST_PORT}..." for i in $(seq 1 60); do diff --git a/ls-k8s/tests/requirements.txt b/ls-on-k8s/tests/requirements.txt similarity index 100% rename from ls-k8s/tests/requirements.txt rename to ls-on-k8s/tests/requirements.txt diff --git a/ls-k8s/tests/test_lambda.py b/ls-on-k8s/tests/test_lambda.py similarity index 100% rename from ls-k8s/tests/test_lambda.py rename to ls-on-k8s/tests/test_lambda.py From b2920a9c26a247d79bc611766722815c15a8d5f6 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 9 May 2026 13:12:35 +0200 Subject: [PATCH 2/5] =?UTF-8?q?fix:=20k8s=20executor=20connectivity=20?= =?UTF-8?q?=E2=80=94=20simpler=20kubeconfig=20+=20CI=20debug=20step?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Init container: drop base64 (busybox compat issue), use certificate-authority file path + embedded token instead of certificate-authority-data - Add 'Debug LocalStack pod k8s setup' CI step that execs into the pod and prints KUBERNETES_*/KUBECONFIG env vars, /kube/config, and SA file listing Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/test-ls-on-k8s.yml | 11 +++++++++++ ls-on-k8s/k8s/localstack.yaml | 11 +++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-ls-on-k8s.yml b/.github/workflows/test-ls-on-k8s.yml index afbf510..e8fa4e5 100644 --- a/.github/workflows/test-ls-on-k8s.yml +++ b/.github/workflows/test-ls-on-k8s.yml @@ -41,6 +41,17 @@ jobs: with: python-version: "3.11" + - name: Debug LocalStack pod k8s setup + run: | + PODNAME=$(kubectl --context k3d-ls-k8s-cluster -n localstack get pod -l app=localstack -o jsonpath='{.items[0].metadata.name}') + echo "Pod: $PODNAME" + echo "--- env ---" + kubectl --context k3d-ls-k8s-cluster -n localstack exec "$PODNAME" -- env | grep -E 'KUBERNETES|KUBECONFIG|LAMBDA' | sort + echo "--- /kube/config ---" + kubectl --context k3d-ls-k8s-cluster -n localstack exec "$PODNAME" -- cat /kube/config 2>/dev/null || echo "(not found)" + echo "--- serviceaccount files ---" + kubectl --context k3d-ls-k8s-cluster -n localstack exec "$PODNAME" -- ls /var/run/secrets/kubernetes.io/serviceaccount/ 2>/dev/null || echo "(not found)" + - name: Run Lambda integration tests run: make test diff --git a/ls-on-k8s/k8s/localstack.yaml b/ls-on-k8s/k8s/localstack.yaml index 451e514..805d0d1 100644 --- a/ls-on-k8s/k8s/localstack.yaml +++ b/ls-on-k8s/k8s/localstack.yaml @@ -45,8 +45,9 @@ spec: app: localstack spec: serviceAccountName: localstack - # Writes a kubeconfig with the actual token + CA embedded so LocalStack's - # k8s client connects to kubernetes.default.svc instead of localhost:80. + # Writes a kubeconfig pointing to kubernetes.default.svc so LocalStack's + # k8s client doesn't fall back to localhost:80. + # Uses certificate-authority file path (avoids base64 portability issues). initContainers: - name: gen-kubeconfig image: busybox @@ -54,8 +55,8 @@ spec: - sh - -c - | + set -e TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) - CA=$(base64 /var/run/secrets/kubernetes.io/serviceaccount/ca.crt | tr -d '\n') cat > /kube/config < Date: Sat, 9 May 2026 13:18:28 +0200 Subject: [PATCH 3/5] fix: set LOCALSTACK_K8S_SERVICE_NAME so executor endpoint resolution doesn't return None --- ls-on-k8s/k8s/localstack.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ls-on-k8s/k8s/localstack.yaml b/ls-on-k8s/k8s/localstack.yaml index 805d0d1..dac26a3 100644 --- a/ls-on-k8s/k8s/localstack.yaml +++ b/ls-on-k8s/k8s/localstack.yaml @@ -93,6 +93,8 @@ spec: value: kubernetes - name: LOCALSTACK_K8S_NAMESPACE value: localstack + - name: LOCALSTACK_K8S_SERVICE_NAME + value: localstack - name: KUBECONFIG value: /kube/config - name: LOCALSTACK_DISABLE_EVENTS From 91d10dd3093fda4630fe3f9caabc5d1b08f766d2 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 9 May 2026 13:35:53 +0200 Subject: [PATCH 4/5] fix: use localstack/lambda- image prefix for k8s executor (includes init binary) --- .github/workflows/test-ls-on-k8s.yml | 8 +++++++- ls-on-k8s/k8s/localstack.yaml | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-ls-on-k8s.yml b/.github/workflows/test-ls-on-k8s.yml index e8fa4e5..32c77d5 100644 --- a/.github/workflows/test-ls-on-k8s.yml +++ b/.github/workflows/test-ls-on-k8s.yml @@ -55,11 +55,17 @@ jobs: - name: Run Lambda integration tests run: make test - - name: Dump LocalStack logs on failure + - name: Dump logs on failure if: failure() run: | make logs 2>/dev/null || true make pods 2>/dev/null || true + echo "--- lambda pod logs ---" + kubectl --context k3d-ls-k8s-cluster -n localstack get pods | grep lambda || true + for pod in $(kubectl --context k3d-ls-k8s-cluster -n localstack get pods -o name | grep lambda); do + echo "=== $pod ===" + kubectl --context k3d-ls-k8s-cluster -n localstack logs $pod --all-containers 2>/dev/null || true + done - name: Tear down cluster if: always() diff --git a/ls-on-k8s/k8s/localstack.yaml b/ls-on-k8s/k8s/localstack.yaml index dac26a3..d4b63a3 100644 --- a/ls-on-k8s/k8s/localstack.yaml +++ b/ls-on-k8s/k8s/localstack.yaml @@ -95,6 +95,8 @@ spec: value: localstack - name: LOCALSTACK_K8S_SERVICE_NAME value: localstack + - name: LAMBDA_K8S_IMAGE_PREFIX + value: localstack/lambda- - name: KUBECONFIG value: /kube/config - name: LOCALSTACK_DISABLE_EVENTS From dc9b75e0528b023fb1ded31a0726497d994579f6 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Sat, 9 May 2026 13:42:22 +0200 Subject: [PATCH 5/5] simplify: remove init container and kubeconfig volume (load_incluster_config works natively) --- ls-on-k8s/k8s/localstack.yaml | 45 ----------------------------------- 1 file changed, 45 deletions(-) diff --git a/ls-on-k8s/k8s/localstack.yaml b/ls-on-k8s/k8s/localstack.yaml index d4b63a3..9e6f307 100644 --- a/ls-on-k8s/k8s/localstack.yaml +++ b/ls-on-k8s/k8s/localstack.yaml @@ -45,43 +45,6 @@ spec: app: localstack spec: serviceAccountName: localstack - # Writes a kubeconfig pointing to kubernetes.default.svc so LocalStack's - # k8s client doesn't fall back to localhost:80. - # Uses certificate-authority file path (avoids base64 portability issues). - initContainers: - - name: gen-kubeconfig - image: busybox - command: - - sh - - -c - - | - set -e - TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) - cat > /kube/config <