PoC spike for k8s-based linkerd-cni testing

.github/workflows/cni-plugin-integration.yml

@@ -0,0 +1,22 @@
+name: cni-plugin-integration
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - Dockerfile-cni-plugin
+      - cni-plugin/integration/flannel/Dockerfile-tester
+      - cni-plugin/integration/run.sh
+      - cni-plugin/**
+
+jobs:
+  integration:
+    timeout-minutes: 15
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: linkerd/dev/actions/setup-tools@v38
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
+      - run: just k3d-create
+      - run: just build-cni-plugin-image
+      - run: just build-cni-plugin-test-image
+      - run: just cni-plugin-test-integration

cni-plugin/integration/Dockerfile-tester

@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1.4
+#
+# A single container holds all of the test code and it must be
+# specified in `run.sh` which tests you want to run.
+#
+# There's no ENTRYPOINT as integration test runners will require
+# two things:
+#  1) a specific k3d cluster configured with CNI
+#  2) a test suite (e.g. `flannel.go`) runs with a configured CNI plugin.
+
+FROM golang:1.18-alpine AS build
+ENV GOCACHE=/tmp/
+WORKDIR /src
+COPY --link go.mod go.sum .
+COPY --link cni-plugin cni-plugin
+COPY --link internal internal
+COPY --link proxy-init proxy-init
+RUN go mod tidy && go mod download

cni-plugin/integration/flannel/flannel_test.go

@@ -0,0 +1,91 @@
+package flannel
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+)
+
+const (
+	ConfigDirectory = "/var/lib/rancher/k3s/agent/etc/cni/net.d"
+	FlannelConflist = "10-flannel.conflist"
+)
+
+// Given a directory, return a map of filename->struct{}
+func files(directory string) (map[string]struct{}, error) {
+	files, err := os.ReadDir(directory)
+	if err != nil {
+		return nil, err
+	}
+
+	fileNames := make(map[string]struct{}, len(files))
+	for _, f := range files {
+		fileNames[f.Name()] = struct{}{}
+	}
+
+	return fileNames, nil
+}
+
+func TestMain(m *testing.M) {
+	runTests := flag.Bool("integration-tests", false, "must be provided to run the integration tests")
+	flag.Parse()
+
+	if !*runTests {
+		fmt.Fprintln(os.Stderr, "integration tests not enabled: enable with -integration-tests")
+		os.Exit(0)
+	}
+
+	os.Exit(m.Run())
+}
+
+// TODO(stevej): this could be a test helper as we want it to be true for every CNI integration
+func TestLinkerdIsLastCNIPlugin(t *testing.T) {
+	t.Parallel()
+
+	t.Run("succeeds when linkerd-cni is the last plugin", func(t *testing.T) {
+		if _, err := os.Stat(ConfigDirectory); os.IsNotExist(err) {
+			t.Fatalf("Directory does not exist. Check if volume mount exists: %s", ConfigDirectory)
+		}
+
+		filenames, err := files(ConfigDirectory)
+
+		if err != nil {
+			t.Fatalf("unable to read files from directory %s due to error: %e", ConfigDirectory, err)
+		}
+
+		if len(filenames) == 0 {
+			t.Fatalf("no files found in %s", ConfigDirectory)
+		}
+
+		if len(filenames) > 2 {
+			t.Fatalf("too many files found in %s: %s ", ConfigDirectory, filenames)
+		}
+
+		if _, ok := filenames[FlannelConflist]; !ok {
+			t.Fatalf("filenames does not contain %s, instead it contains: %s", FlannelConflist, filenames)
+		}
+
+		conflistFile, err := os.ReadFile(ConfigDirectory + "/" + FlannelConflist)
+		if err != nil {
+			t.Fatalf("could not read %s: %e", FlannelConflist, err)
+		}
+
+		var conflist map[string]any
+		err = json.Unmarshal(conflistFile, &conflist)
+		if err != nil {
+			t.Fatalf("unmarshaling conflist json failed: %e", err)
+		}
+
+		if conflist["cniVersion"] != "1.0.0" {
+			t.Fatalf("expected cniVersion 1.0.0, instead saw %s", conflistFile)
+		}
+
+		plugins := conflist["plugins"].([]interface{})
+		lastPlugin := plugins[len(plugins)-1].(map[string]any)
+		if lastPlugin["name"] != "linkerd-cni" {
+			t.Fatalf("linkerd-cni was not last in the plugins list")
+		}
+	})
+}

cni-plugin/integration/manifests/linkerd-cni.yaml

@@ -0,0 +1,177 @@
+##
+## Everything below here is generated from the output `linkerd install-cni`
+## and modified with the test image of the cni-plugin.
+##
+## `linkerd install-cni \
+##    --dest-cni-net-dir "/var/lib/rancher/k3s/agent/etc/cni/net.d/" \
+##    --dest-cni-bin-dir "/bin"`
+## These flags are meant to enable cni to work properly with k3d/k3s.
+## Also the log level is set to debug to simplify development.
+##
+## DO NOT hand edit.
+##
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: linkerd-cni
+  labels:
+    linkerd.io/cni-resource: "true"
+    config.linkerd.io/admission-webhooks: disabled
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: linkerd-cni
+  namespace: linkerd-cni
+  labels:
+    linkerd.io/cni-resource: "true"
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: linkerd-cni
+  labels:
+    linkerd.io/cni-resource: "true"
+rules:
+- apiGroups: [""]
+  resources: ["pods", "nodes", "namespaces", "services"]
+  verbs: ["list", "get", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: linkerd-cni
+  labels:
+    linkerd.io/cni-resource: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: linkerd-cni
+subjects:
+- kind: ServiceAccount
+  name: linkerd-cni
+  namespace: linkerd-cni
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: linkerd-cni-config
+  namespace: linkerd-cni
+  labels:
+    linkerd.io/cni-resource: "true"
+data:
+  dest_cni_net_dir: "/var/lib/rancher/k3s/agent/etc/cni/net.d"
+  dest_cni_bin_dir: "/bin"
+  # The CNI network configuration to install on each node. The special
+  # values in this config will be automatically populated.
+  cni_network_config: |-
+    {
+      "name": "linkerd-cni",
+      "type": "linkerd-cni",
+      "log_level": "debug",
+      "policy": {
+          "type": "k8s",
+          "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
+          "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
+      },
+      "kubernetes": {
+          "kubeconfig": "__KUBECONFIG_FILEPATH__"
+      },
+      "linkerd": {
+        "incoming-proxy-port": 4143,
+        "outgoing-proxy-port": 4140,
+        "proxy-uid": 2102,
+        "ports-to-redirect": [],
+        "inbound-ports-to-ignore": ["4191","4190"],
+        "simulate": false,
+        "use-wait-flag": false
+      }
+    }
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: linkerd-cni
+  namespace: linkerd-cni
+  labels:
+    k8s-app: linkerd-cni
+    linkerd.io/cni-resource: "true"
+  annotations:
+    linkerd.io/created-by: linkerd/cli edge-22.12.1
+spec:
+  selector:
+    matchLabels:
+      k8s-app: linkerd-cni
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: linkerd-cni
+      annotations:
+        linkerd.io/created-by: linkerd/cli edge-22.12.1
+        linkerd.io/cni-resource: "true"
+        linkerd.io/inject: disabled
+    spec:
+      tolerations:
+        - operator: Exists
+      nodeSelector:
+        kubernetes.io/os: linux
+      hostNetwork: true
+      serviceAccountName: linkerd-cni
+      containers:
+      # This container installs the linkerd CNI binaries
+      # and CNI network config file on each node. The install
+      # script copies the files into place and then sleeps so
+      # that Kubernetes doesn't keep trying to restart it.
+      - name: install-cni
+        #image: test.l5d.io/linkerd/cni-plugin:test
+        image: cr.l5d.io/linkerd/cni-plugin:edge-22.12.1
+        env:
+        - name: DEST_CNI_NET_DIR
+          valueFrom:
+            configMapKeyRef:
+              name: linkerd-cni-config
+              key: dest_cni_net_dir
+        - name: DEST_CNI_BIN_DIR
+          valueFrom:
+            configMapKeyRef:
+              name: linkerd-cni-config
+              key: dest_cni_bin_dir
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: linkerd-cni-config
+              key: cni_network_config
+        - name: SLEEP
+          value: "true"
+        lifecycle:
+          # In some edge-cases this helps ensure that cleanup() is called in the container's script
+          # https://github.com/linkerd/linkerd2/issues/2355
+          preStop:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - kill -15 1; sleep 15s
+        volumeMounts:
+        - mountPath: /host/bin
+          name: cni-bin-dir
+        - mountPath: /host/var/lib/rancher/k3s/agent/etc/cni/net.d
+          name: cni-net-dir
+        - mountPath: /tmp
+          name: linkerd-tmp-dir
+        securityContext:
+          readOnlyRootFilesystem: true
+          privileged:
+      volumes:
+      - name: cni-bin-dir
+        hostPath:
+          path: /bin
+      - name: cni-net-dir
+        hostPath:
+          path: /var/lib/rancher/k3s/agent/etc/cni/net.d
+      - name: linkerd-tmp-dir
+        emptyDir: {}