From b446c6f1cabd828865a28dba85185b045d2c3627 Mon Sep 17 00:00:00 2001 From: Alejandro Pedraza Date: Wed, 24 May 2023 05:04:53 -0500 Subject: [PATCH] Test CNI ordering Followup to #242 This adds a new integration test that verifies that pods don't get scheduled until all the networking has been setup, making sure having linkerd-cni run run first doesn't trigger scheduling. Note that for now we just consider the linkerd-cni+calico combo. Worklow: 1. Install calico and linkerd-cni. 2. Label node and then add node selectors to calico and linkerd-cni to run only on the current node. 3. Create a new node. Calico nor linkerd-cni will be triggered in it. 4. Persist an nginx pod in that node. Verify it doesn't get scheduled. 5. Add label to linkerd-cni to have it trigger in the new node. 6. Verify the nginx pod still isn't scheduled. 7. Add label to calico to have it trigger in the new node. 8. Verify the nginx pod is finally scheduled. Note that currently the test fails at step 6, but will pass as soon #242 gets merged. In order to test succesfully, you can apply this patch: ```diff diff --git a/cni-plugin/integration/manifests/calico/linkerd-cni.yaml b/cni-plugin/integration/manifests/calico/linkerd-cni.yaml index 05f110b..419abdd 100644 --- a/cni-plugin/integration/manifests/calico/linkerd-cni.yaml +++ b/cni-plugin/integration/manifests/calico/linkerd-cni.yaml @@ -123,7 +123,7 @@ spec: # script copies the files into place and then sleeps so # that Kubernetes doesn't keep trying to restart it. - name: install-cni - image: test.l5d.io/linkerd/cni-plugin:test + image: ghcr.io/alpeb/cni-plugin:v1.2.8 #image: cr.l5d.io/linkerd/cni-plugin:edge-22.12.1 env: - name: DEST_CNI_NET_DIR ``` --- .github/workflows/cni-plugin-integration.yml | 11 ++- cni-plugin/integration/run-ordering.sh | 71 ++++++++++++++++++++ justfile | 8 +++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100755 cni-plugin/integration/run-ordering.sh diff --git a/.github/workflows/cni-plugin-integration.yml b/.github/workflows/cni-plugin-integration.yml index 046983d0..1766d544 100644 --- a/.github/workflows/cni-plugin-integration.yml +++ b/.github/workflows/cni-plugin-integration.yml @@ -36,4 +36,13 @@ jobs: - uses: linkerd/dev/actions/setup-tools@v41 - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab - name: Run CNI integration tests - run: just cni-plugin-test-integration-cilium \ No newline at end of file + run: just cni-plugin-test-integration-cilium + ordering-test: + continue-on-error: true + timeout-minutes: 15 + runs-on: ubuntu-20.04 + steps: + - uses: linkerd/dev/actions/setup-tools@v41 + - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 + - name: Run CNI ordering tests + run: just cni-plugin-test-ordering diff --git a/cni-plugin/integration/run-ordering.sh b/cni-plugin/integration/run-ordering.sh new file mode 100755 index 00000000..0b57c80f --- /dev/null +++ b/cni-plugin/integration/run-ordering.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd "${BASH_SOURCE[0]%/*}" + +NODE_NAME=l5d-server-extra + +kubectl config use-context "k3d-$K3D_CLUSTER_NAME" + +printf '\n# Install calico and linkerd-cni...\n' +kubectl apply -f manifests/calico/calico-install.yaml +kubectl apply -f manifests/calico/linkerd-cni.yaml + +printf '\n# Label node and then add node selectors to calico and linkerd-cni to run only on the current node...\n' +kubectl label node "k3d-$K3D_CLUSTER_NAME-server-0" allow-calico=true +kubectl label node "k3d-$K3D_CLUSTER_NAME-server-0" allow-linkerd-cni=true +kubectl -n kube-system patch daemonsets calico-node --type=json \ + -p='[{"op": "add", "path": "/spec/template/spec/nodeSelector", "value": {"allow-calico": "true"}}]' +kubectl -n linkerd-cni patch daemonsets linkerd-cni --type=json \ + -p='[{"op": "add", "path": "/spec/template/spec/nodeSelector", "value": {"allow-linkerd-cni": "true"}}]' +kubectl rollout status daemonset -n kube-system +kubectl rollout status daemonset -n linkerd-cni + +printf '\n# Create new node...\n' +k3d node create "$NODE_NAME" -c "$K3D_CLUSTER_NAME" +k3d image import test.l5d.io/linkerd/cni-plugin:test -c "$K3D_CLUSTER_NAME" + +printf '\n# Start pod; k8s should not schedule it just yet...\n' +selector="{ + \"spec\": { + \"nodeSelector\": { + \"kubernetes.io/hostname\": \"k3d-$NODE_NAME-0\" + } + } +}" +kubectl run nginx --image nginx --restart Never --overrides="$selector" +sleep 10s +status=$(kubectl get po nginx -ojson | jq -r .status.containerStatuses[0]) +if [[ "$status" == "null" ]]; then + echo "Pod not scheduled as expected" +else + echo "Unexpected pod container status: $status" + exit 1 +fi + +printf '\n# Trigger linkerd-cni; k8s should still not schedule the pod...\n' +kubectl label node "k3d-$NODE_NAME-0" allow-linkerd-cni=true +kubectl rollout status daemonset -n linkerd-cni +sleep 10s +status=$(kubectl get po nginx -ojson | jq -r .status.containerStatuses[0]) +if [[ "$status" == "null" ]]; then + echo "Pod not scheduled as expected" +else + echo "Unexpected pod container status: $status" + exit 1 +fi + +printf '\n# Trigger calico; k8s should now schedule the pod...\n' +kubectl label node "k3d-$NODE_NAME-0" allow-calico=true +kubectl rollout status daemonset -n kube-system +sleep 10s +status=$(kubectl get po nginx -ojson | jq -c .status.containerStatuses[0]) +if [[ "$status" == "null" ]]; then + echo "Pod not scheduled unexpectedly" + exit1 +elif [[ $(echo "$status" | jq .state.running) == "null" ]]; then + echo "Unexpected pod container status: $status" + exit 1 +fi +echo 'Pod scheduled as expected.' diff --git a/justfile b/justfile index 9a2a3038..b9c7b4fd 100644 --- a/justfile +++ b/justfile @@ -184,6 +184,14 @@ cni-plugin-test-integration-cilium: K3D_CREATE_FLAGS='{{ _K3D_CREATE_FLAGS_NO_CNI_NO_POLICY_ENFORCER }}' \ cni-plugin-test-integration-with-cilium +cni-plugin-test-ordering: build-cni-plugin-image + @{{ just_executable() }} K3D_CLUSTER_NAME='l5d-calico-ordering-test' _cni-plugin-test-ordering-run + +_cni-plugin-test-ordering-run: + @{{ just_executable() }} K3D_CREATE_FLAGS='{{ _K3D_CREATE_FLAGS_NO_CNI }}' _k3d-cni-create + @just-k3d import {{ cni-plugin-image }} + ./cni-plugin/integration/run-ordering.sh + _cni-plugin-setup-cilium: #!/usr/bin/env bash set -euxo pipefail