From 44aed3d6c2fee5242de54660c05478ab9f86782c Mon Sep 17 00:00:00 2001 From: Pavlov Alexandr Date: Sat, 13 Jun 2026 10:47:58 +0700 Subject: [PATCH 1/7] ci(06): pin CI WireGuard peer to dedicated 10.8.0.3 (avoid laptop 10.8.0.2 collision) The operator laptop already occupies 10.8.0.2 on the WG network; the CI runner must use a distinct peer IP so both tunnels can coexist. Pass WG_LOCAL_IP to the wg-tunnel-up.sh step in both dry-run and deploy jobs. Pairs with a dedicated [Peer] at 10.8.0.3/32 added on the staging VPS WG server. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/deploy-staging.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index ddc52c9..d895a67 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -54,6 +54,7 @@ jobs: WG_PRIVATE_KEY: ${{ secrets.WG_PRIVATE_KEY }} WG_PEER_PUBLIC_KEY: ${{ secrets.WG_PEER_PUBLIC_KEY }} WG_ENDPOINT: ${{ secrets.WG_ENDPOINT }} + WG_LOCAL_IP: 10.8.0.3/32 run: bash scripts/wg-tunnel-up.sh - name: Setup kubeconfig @@ -88,6 +89,7 @@ jobs: WG_PRIVATE_KEY: ${{ secrets.WG_PRIVATE_KEY }} WG_PEER_PUBLIC_KEY: ${{ secrets.WG_PEER_PUBLIC_KEY }} WG_ENDPOINT: ${{ secrets.WG_ENDPOINT }} + WG_LOCAL_IP: 10.8.0.3/32 run: bash scripts/wg-tunnel-up.sh - name: Setup kubeconfig From f7c44e854b938877002c0b1fa3f0e62898848b91 Mon Sep 17 00:00:00 2001 From: Pavlov Alexandr Date: Sat, 13 Jun 2026 10:56:28 +0700 Subject: [PATCH 2/7] =?UTF-8?q?fix(06):=20WG=20handshake=20=E2=80=94=20pas?= =?UTF-8?q?s=20private-key=20via=20/dev/stdin,=20not=20process=20substitut?= =?UTF-8?q?ion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First live CI run exposed `fopen: No such file or directory` at `wg set private-key <(...)`: sudo closes inherited FDs (closefrom=3), so the process-substitution /dev/fd/NN path vanishes in the wg process. /dev/stdin (FD 0) is preserved by sudo, keeping the key off disk while fixing the handshake step. Found by the dry-run job in PR #1. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/wg-tunnel-up.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/wg-tunnel-up.sh b/scripts/wg-tunnel-up.sh index 373b118..7a77c90 100755 --- a/scripts/wg-tunnel-up.sh +++ b/scripts/wg-tunnel-up.sh @@ -53,10 +53,14 @@ sudo ip link add dev "$WG_INTERFACE" type wireguard echo "Assigning local IP $WG_LOCAL_IP..." sudo ip address add "$WG_LOCAL_IP" dev "$WG_INTERFACE" -# --- 3. Configure peer (private key via process substitution — never on disk) +# --- 3. Configure peer (private key via /dev/stdin — never on disk) --------- +# NB: process substitution <(...) does NOT survive `sudo` — sudo closes +# inherited FDs (closefrom=3), so the FD-backed /dev/fd/NN path disappears in +# the wg process ("fopen: No such file or directory"). /dev/stdin (FD 0) IS +# preserved by sudo, so the key still never touches disk. echo "Configuring peer $WG_PEER_PUBLIC_KEY..." -sudo wg set "$WG_INTERFACE" \ - private-key <(printf '%s' "$WG_PRIVATE_KEY") \ +printf '%s' "$WG_PRIVATE_KEY" | sudo wg set "$WG_INTERFACE" \ + private-key /dev/stdin \ peer "$WG_PEER_PUBLIC_KEY" \ endpoint "$WG_ENDPOINT" \ allowed-ips "$WG_ALLOWED_IPS" From 3841680f37844dd8e32a34ae931c640190ac63e7 Mon Sep 17 00:00:00 2001 From: Pavlov Alexandr Date: Sat, 13 Jun 2026 10:59:46 +0700 Subject: [PATCH 3/7] =?UTF-8?q?fix(06):=20WG=20handshake=20=E2=80=94=20ini?= =?UTF-8?q?tiate=20it=20(keepalive=20+=20prime=20packet),=20don't=20just?= =?UTF-8?q?=20wait?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second live run: interface showed no `transfer:` line — zero bytes sent, so the handshake was never initiated. WireGuard is lazy: it only sends a handshake when an outbound packet is destined for the peer. The wait loop polled latest-handshakes without generating any traffic, so it always timed out. Fix: set `persistent-keepalive 25` on the peer and poke the API server (/dev/tcp/10.8.0.1:6443) at the top of the wait loop to force immediate handshake initiation. Found by the dry-run job in PR #1. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/wg-tunnel-up.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/wg-tunnel-up.sh b/scripts/wg-tunnel-up.sh index 7a77c90..bc61957 100755 --- a/scripts/wg-tunnel-up.sh +++ b/scripts/wg-tunnel-up.sh @@ -63,6 +63,7 @@ printf '%s' "$WG_PRIVATE_KEY" | sudo wg set "$WG_INTERFACE" \ private-key /dev/stdin \ peer "$WG_PEER_PUBLIC_KEY" \ endpoint "$WG_ENDPOINT" \ + persistent-keepalive 25 \ allowed-ips "$WG_ALLOWED_IPS" # --- 4. Bring interface up -------------------------------------------------- @@ -76,6 +77,12 @@ sudo ip link set up dev "$WG_INTERFACE" echo "Waiting for handshake (timeout: ${HANDSHAKE_TIMEOUT_SECS}s)..." start_epoch=$(date +%s) while true; do + # Prime: WireGuard initiates a handshake only when an outbound packet needs + # to reach the peer. Poke the API server through the tunnel to force that + # initiation (the connect may fail until the tunnel is up — we only need it + # to queue a packet so the handshake starts). + timeout 1 bash -c "echo > /dev/tcp/$K8S_API_HOST/$K8S_API_PORT" 2>/dev/null || true + if sudo wg show "$WG_INTERFACE" latest-handshakes | grep -qP '\t[1-9][0-9]*$'; then echo "Handshake complete." break From bf155d3e4906dd3cd104543c8966fda56b7bf2ce Mon Sep 17 00:00:00 2001 From: Pavlov Alexandr Date: Sat, 13 Jun 2026 11:03:29 +0700 Subject: [PATCH 4/7] =?UTF-8?q?fix(06):=20WG=20tunnel=20=E2=80=94=20add=20?= =?UTF-8?q?kernel=20route=20to=20allowed-ips=20(raw=20wg=20set=20adds=20no?= =?UTF-8?q?ne)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third live run: handshake completed but the API reachability check timed out. Root cause: `wg set allowed-ips` only sets WireGuard cryptokey routing, not a kernel route, and the /32 local address adds no subnet route — so 10.8.0.1 had no route via wg0 (the handshake still succeeded because keepalive/handshake go endpoint-to-endpoint over the public net, bypassing the tunnel IP). Add an explicit `ip route replace dev wg0` after link-up. Found by the dry-run job in PR #1. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/wg-tunnel-up.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/wg-tunnel-up.sh b/scripts/wg-tunnel-up.sh index bc61957..cc610b8 100755 --- a/scripts/wg-tunnel-up.sh +++ b/scripts/wg-tunnel-up.sh @@ -70,6 +70,17 @@ printf '%s' "$WG_PRIVATE_KEY" | sudo wg set "$WG_INTERFACE" \ echo "Bringing up $WG_INTERFACE..." sudo ip link set up dev "$WG_INTERFACE" +# --- 4b. Route allowed-ips through the tunnel ------------------------------- +# `wg set ... allowed-ips` only configures WireGuard's cryptokey routing — it +# does NOT add a kernel route (that is wg-quick's job, and we don't use it). +# With a /32 local address there is no subnet route either, so the API server +# IP would be unreachable at the IP layer even though the endpoint-to-endpoint +# handshake succeeds. Add an explicit route for each allowed IP. +for cidr in ${WG_ALLOWED_IPS//,/ }; do + echo "Routing $cidr via $WG_INTERFACE..." + sudo ip route replace "$cidr" dev "$WG_INTERFACE" +done + # --- 5. Handshake polling loop (fail-closed) -------------------------------- # latest-handshakes output: "\t" # Handshake is complete when timestamp_epoch is non-zero (> 0). From 091bcceb9faa7825ebe780d62e4a23c32f23ce5e Mon Sep 17 00:00:00 2001 From: Pavlov Alexandr Date: Sat, 13 Jun 2026 11:06:06 +0700 Subject: [PATCH 5/7] =?UTF-8?q?fix(06):=20CD=20apply=20=E2=80=94=20give=20?= =?UTF-8?q?each=20manifest=20its=20own=20-f=20(xargs=20appended=20bare=20f?= =?UTF-8?q?iles)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fourth live run: `xargs kubectl apply ... -f` placed all filenames after a single -f, so kubectl took only the first and rejected the rest ("Unexpected args"). Prefix each path with `-f ` via sed so every manifest gets its own flag. Same fix in both the dry-run and deploy jobs. Found by the dry-run job in PR #1. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/deploy-staging.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index d895a67..f407dfe 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -67,8 +67,8 @@ jobs: run: | set -euo pipefail # Exclude 01-ci-rbac.yaml — operator-bootstrapped, not CI-managed - find k8s/staging -maxdepth 1 -name '*.yaml' ! -name '01-ci-rbac.yaml' | sort | \ - xargs kubectl apply -n "$K8S_NAMESPACE" --dry-run=server -f + find k8s/staging -maxdepth 1 -name '*.yaml' ! -name '01-ci-rbac.yaml' | sort | sed 's/^/-f /' | \ + xargs kubectl apply -n "$K8S_NAMESPACE" --dry-run=server deploy: name: Deploy to k3s @@ -123,8 +123,8 @@ jobs: run: | set -euo pipefail # Exclude 01-ci-rbac.yaml — operator-bootstrapped, not CI-managed - find k8s/staging -maxdepth 1 -name '*.yaml' ! -name '01-ci-rbac.yaml' | sort | \ - xargs kubectl apply -n "$K8S_NAMESPACE" -f + find k8s/staging -maxdepth 1 -name '*.yaml' ! -name '01-ci-rbac.yaml' | sort | sed 's/^/-f /' | \ + xargs kubectl apply -n "$K8S_NAMESPACE" - name: Verify rollouts run: | From d701f6d1fedf97866de5510ec2d078d884897df7 Mon Sep 17 00:00:00 2001 From: Pavlov Alexandr Date: Sat, 13 Jun 2026 11:07:50 +0700 Subject: [PATCH 6/7] =?UTF-8?q?fix(06):=20kubeconfig=20=E2=80=94=20embed?= =?UTF-8?q?=20CA=20data=20so=20it=20survives=20temp-file=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifth live run: the dry-run apply failed with "unable to read certificate-authority /tmp/tmp.XXX: no such file or directory". set-cluster stored the CA *path*, but the temp CA file is rm'd by the script's EXIT trap; the in-script `auth whoami` passed only because the trap had not fired yet. Add --embed-certs=true so the CA is inlined into the kubeconfig and later steps work after cleanup. Found by the dry-run job in PR #1. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/kubeconfig-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/kubeconfig-setup.sh b/scripts/kubeconfig-setup.sh index 8b48bf6..5f7bafe 100755 --- a/scripts/kubeconfig-setup.sh +++ b/scripts/kubeconfig-setup.sh @@ -46,6 +46,7 @@ printf '%s' "$K8S_CA_CERT" > "$ca_file" echo "Configuring cluster $K8S_CLUSTER_NAME -> $K8S_API_SERVER..." kubectl config set-cluster "$K8S_CLUSTER_NAME" \ --certificate-authority="$ca_file" \ + --embed-certs=true \ --server="$K8S_API_SERVER" \ --kubeconfig="$KUBECONFIG" From cfa8a5e63171cc24a20eee9b41d3717e04874797 Mon Sep 17 00:00:00 2001 From: Pavlov Alexandr Date: Sat, 13 Jun 2026 11:10:46 +0700 Subject: [PATCH 7/7] =?UTF-8?q?fix(06):=20CD=20glob=20=E2=80=94=20also=20e?= =?UTF-8?q?xclude=2000-namespace.yaml=20(operator-bootstrapped)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sixth live run: every workload dry-ran cleanly, but 00-namespace.yaml was Forbidden — the namespace-scoped ci-deployer Role cannot get/create the cluster-scoped Namespace (by design: "CI never creates the namespace"). The glob already excluded 01-ci-rbac.yaml; add 00-namespace.yaml to the same operator-managed exclusion in both jobs. Found by the dry-run job in PR #1. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/deploy-staging.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index f407dfe..b5e843c 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -66,8 +66,10 @@ jobs: - name: Dry-run kubectl apply (server-side) run: | set -euo pipefail - # Exclude 01-ci-rbac.yaml — operator-bootstrapped, not CI-managed - find k8s/staging -maxdepth 1 -name '*.yaml' ! -name '01-ci-rbac.yaml' | sort | sed 's/^/-f /' | \ + # Exclude operator-bootstrapped, non-CI-managed manifests: + # 00-namespace.yaml (cluster-scoped Namespace) and 01-ci-rbac.yaml + # (the namespace-scoped ci-deployer Role cannot get/create either). + find k8s/staging -maxdepth 1 -name '*.yaml' ! -name '00-namespace.yaml' ! -name '01-ci-rbac.yaml' | sort | sed 's/^/-f /' | \ xargs kubectl apply -n "$K8S_NAMESPACE" --dry-run=server deploy: @@ -122,8 +124,10 @@ jobs: - name: Apply staging manifests run: | set -euo pipefail - # Exclude 01-ci-rbac.yaml — operator-bootstrapped, not CI-managed - find k8s/staging -maxdepth 1 -name '*.yaml' ! -name '01-ci-rbac.yaml' | sort | sed 's/^/-f /' | \ + # Exclude operator-bootstrapped, non-CI-managed manifests: + # 00-namespace.yaml (cluster-scoped Namespace) and 01-ci-rbac.yaml + # (the namespace-scoped ci-deployer Role cannot get/create either). + find k8s/staging -maxdepth 1 -name '*.yaml' ! -name '00-namespace.yaml' ! -name '01-ci-rbac.yaml' | sort | sed 's/^/-f /' | \ xargs kubectl apply -n "$K8S_NAMESPACE" - name: Verify rollouts