Test CNI ordering

.github/workflows/cni-plugin-integration.yml

@@ -36,4 +36,13 @@ jobs:
       - uses: linkerd/dev/actions/setup-tools@v41
       - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab
       - name: Run CNI integration tests
-        run: just cni-plugin-test-integration-cilium
+        run: just cni-plugin-test-integration-cilium
+  ordering-test:
+    continue-on-error: true
+    timeout-minutes: 15
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: linkerd/dev/actions/setup-tools@v41
+      - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3
+      - name: Run CNI ordering tests
+        run: just cni-plugin-test-ordering

cni-plugin/integration/run-ordering.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+cd "${BASH_SOURCE[0]%/*}"
+
+NODE_NAME=l5d-server-extra
+
+kubectl config use-context "k3d-$K3D_CLUSTER_NAME"
+
+printf '\n# Install calico and linkerd-cni...\n'
+kubectl apply -f manifests/calico/calico-install.yaml
+kubectl apply -f manifests/calico/linkerd-cni.yaml
+
+printf '\n# Label node and then add node selectors to calico and linkerd-cni to run only on the current node...\n'
+kubectl label node "k3d-$K3D_CLUSTER_NAME-server-0" allow-calico=true
+kubectl label node "k3d-$K3D_CLUSTER_NAME-server-0" allow-linkerd-cni=true
+kubectl -n kube-system patch daemonsets calico-node --type=json \
+	-p='[{"op": "add", "path": "/spec/template/spec/nodeSelector", "value": {"allow-calico": "true"}}]'
+kubectl -n linkerd-cni patch daemonsets linkerd-cni --type=json \
+	-p='[{"op": "add", "path": "/spec/template/spec/nodeSelector", "value": {"allow-linkerd-cni": "true"}}]'
+kubectl rollout status daemonset -n kube-system
+kubectl rollout status daemonset -n linkerd-cni
+
+printf '\n# Create new node...\n'
+k3d node create "$NODE_NAME" -c "$K3D_CLUSTER_NAME"
+k3d image import test.l5d.io/linkerd/cni-plugin:test -c "$K3D_CLUSTER_NAME"
+
+printf '\n# Start pod; k8s should not schedule it just yet...\n'
+selector="{
+  \"spec\": {
+    \"nodeSelector\": {
+      \"kubernetes.io/hostname\": \"k3d-$NODE_NAME-0\"
+    }
+  }
+}"
+kubectl run nginx --image nginx --restart Never --overrides="$selector"
+sleep 10s
+status=$(kubectl get po nginx -ojson | jq -r .status.containerStatuses[0])
+if [[ "$status" == "null" ]]; then
+		echo "Pod not scheduled as expected"
+else
+		echo "Unexpected pod container status: $status"
+		exit 1
+fi
+
+printf '\n# Trigger linkerd-cni; k8s should still not schedule the pod...\n'
+kubectl label node "k3d-$NODE_NAME-0" allow-linkerd-cni=true
+kubectl rollout status daemonset -n linkerd-cni
+sleep 10s
+status=$(kubectl get po nginx -ojson | jq -r .status.containerStatuses[0])
+if [[ "$status" == "null" ]]; then
+		echo "Pod not scheduled as expected"
+else
+		echo "Unexpected pod container status: $status"
+		exit 1
+fi
+
+printf '\n# Trigger calico; k8s should now schedule the pod...\n'
+kubectl label node "k3d-$NODE_NAME-0" allow-calico=true
+kubectl rollout status daemonset -n kube-system
+sleep 10s
+status=$(kubectl get po nginx -ojson | jq -c .status.containerStatuses[0])
+if [[ "$status" == "null" ]]; then
+		echo "Pod not scheduled unexpectedly"
+    exit1
+elif [[ $(echo "$status" | jq .state.running) == "null" ]]; then
+		echo "Unexpected pod container status: $status"
+		exit 1
+fi
+echo 'Pod scheduled as expected.'

justfile

@@ -184,6 +184,14 @@ cni-plugin-test-integration-cilium:
         K3D_CREATE_FLAGS='{{ _K3D_CREATE_FLAGS_NO_CNI_NO_POLICY_ENFORCER }}' \
         cni-plugin-test-integration-with-cilium
 
+cni-plugin-test-ordering: build-cni-plugin-image
+    @{{ just_executable() }} K3D_CLUSTER_NAME='l5d-calico-ordering-test' _cni-plugin-test-ordering-run
+
+_cni-plugin-test-ordering-run:
+    @{{ just_executable() }} K3D_CREATE_FLAGS='{{ _K3D_CREATE_FLAGS_NO_CNI }}' _k3d-cni-create
+    @just-k3d import {{ cni-plugin-image }}
+    ./cni-plugin/integration/run-ordering.sh
+
 _cni-plugin-setup-cilium:
     #!/usr/bin/env bash
     set -euxo pipefail