test: Add GH Action for Cilium L4LB XDP tests

This commit introduces a new GH action called "Cilium L4LB XDP" which is
responsible for running the standalone LB tests.

The action starts a Fedora VM with vagrant. We do that because we need
to run Kind on cgroupv2-only machine (otherwise, bpf_sock which is
required by the LB health check is not guaranteed to work).
Unfortunately, GH Action does not support any runner with cgroupv2-only.
So instead we run Fedora 34 which has cgroupv1 disabled on the MacOS
runner which supports nested virtualisation.

For now the test issues 10 requests to LB VIP from the Fedora VM. See
test.sh for more details.

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Martynas Pumputis <m@lambda.lt>

.github/workflows/l4lb.yaml

@@ -0,0 +1,65 @@
+name: Cilium L4LB XDP
+
+# Any change in triggers needs to be reflected in the concurrency group.
+on:
+  pull_request:
+    paths-ignore:
+      - 'Documentation/**'
+      - 'test/**'
+  push:
+    branches:
+      - master
+    paths-ignore:
+      - 'Documentation/**'
+      - 'test/**'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.after }}
+  cancel-in-progress: true
+
+jobs:
+  setup-and-test:
+    name: Cilium L4LB XDP
+    # We need nested virtualisation which is supported only by MacOS runner
+    runs-on: macos-10.15
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Boot Fedora
+        run: |
+          ln -sf ./test/l4lb-xdp/Vagrantfile ./Vagrantfile
+          # Retry if it fails (download.fedoraproject.org returns 404 sometimes)
+          # Spend up to 10 seconds on this
+          for i in {1..4}; do
+            if vagrant up; then
+              break
+            fi
+            sleep $i
+          done
+          vagrant ssh-config >> ~/.ssh/config
+
+      - name: Set image tag
+        id: vars
+        run: |
+          if [ ${{ github.event.pull_request.head.sha }} != "" ]; then
+            echo ::set-output name=tag::${{ github.event.pull_request.head.sha }}
+          else
+            echo ::set-output name=tag::${{ github.sha }}
+          fi
+
+
+      - name: Wait for image to be available
+        timeout-minutes: 10
+        shell: bash
+        run: |
+          until curl --silent -f -lSL "https://quay.io/api/v1/repository/${{ github.repository_owner }}/cilium-ci/tag/${{ steps.vars.outputs.tag }}/images" &> /dev/null; do sleep 45s; done
+
+      - name: Run tests
+        run: |
+          ssh default "sudo /bin/sh -c 'cd /vagrant/test/l4lb-xdp && ./test.sh quay.io/${{ github.repository_owner}}/cilium-ci:${{ steps.vars.outputs.tag }}'"

test/l4lb-xdp/Vagrantfile

@@ -0,0 +1,30 @@
+# Vagrant box for testing kind with cgroup v2
+# Adopted from https://github.com/kubernetes-sigs/kind/blob/main/hack/ci/Vagrantfile
+Vagrant.configure("2") do |config|
+  config.vm.box = "fedora/34-cloud-base"
+  memory = 2048
+  cpus = 2
+  config.vm.provider :virtualbox do |v|
+    v.memory = memory
+    v.cpus = cpus
+  end
+  config.vm.provider :libvirt do |v|
+    v.memory = memory
+    v.cpus = cpus
+  end
+  config.vm.provision "install-packages", type: "shell", run: "once" do |sh|
+    sh.inline = <<~SHELL
+    set -eux -o pipefail
+    dnf install -y make kubernetes-client clang llvm glibc-devel.i686 libbpf-devel ethtool
+
+    # cgroupv2 requires >= Docker 20.10 from the upstream.
+    curl -fsSL https://get.docker.com | sh
+    systemctl enable --now docker
+
+    # cgroupv2 requires >= Kind 0.11.0
+    curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.0/kind-linux-amd64
+    chmod +x ./kind
+    mv ./kind /usr/local/bin
+    SHELL
+  end
+end

test/l4lb-xdp/bpf_xdp_veth_host.c

@@ -0,0 +1,14 @@
+#include <linux/bpf.h>
+
+#ifndef __section
+# define __section(NAME)                  \
+	   __attribute__((section(NAME), used))
+#endif
+
+__section("prog")
+int xdp_foo(struct xdp_md *ctx)
+{
+	    return XDP_PASS;
+}
+
+char __license[] __section("license") = "GPL";

test/l4lb-xdp/cilium-lb.yaml

@@ -0,0 +1,206 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cilium-config
+  namespace: kube-system
+data:
+  datapath-mode: "lb-only"
+  devices: "eth0"
+  kube-proxy-replacement: "partial"
+  enable-node-port: "true"
+  node-port-algorithm: "maglev"
+  bpf-lb-mode: "dsr"
+  bpf-lb-dsr-dispatch: "ipip"
+  bpf-lb-acceleration: "native"
+  enable-health-check-nodeport: "false"
+  node-port-bind-protection: "false"
+  enable-host-reachable-services: "false"
+
+  enable-ipv4: "true"
+  enable-ipv6: "false"
+
+  enable-hubble: "true"
+  enable-recorder: "true"
+
+  debug: "false"
+  enable-l7-proxy: "false"
+  enable-bpf-clock-probe: "true"
+  preallocate-bpf-maps: "false"
+  tunnel: "disabled"
+  install-iptables-rules: "false"
+  auto-direct-node-routes: "false"
+  enable-bandwidth-manager: "false"
+  enable-local-redirect-policy: "false"
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: cilium
+  name: cilium
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cilium
+  template:
+    metadata:
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+      labels:
+        k8s-app: cilium
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: In
+                values:
+                - kind-control-plane
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: k8s-app
+                operator: In
+                values:
+                - cilium
+            topologyKey: kubernetes.io/hostname
+      containers:
+      - args:
+        - --config-dir=/tmp/cilium/config-map
+        command:
+        - cilium-agent
+        startupProbe:
+          httpGet:
+            host: '127.0.0.1'
+            path: /healthz
+            port: 9876
+            scheme: HTTP
+            httpHeaders:
+            - name: "brief"
+              value: "true"
+          failureThreshold: 24
+          periodSeconds: 2
+          successThreshold: 1
+        livenessProbe:
+          httpGet:
+            host: '127.0.0.1'
+            path: /healthz
+            port: 9876
+            scheme: HTTP
+            httpHeaders:
+            - name: "brief"
+              value: "true"
+          failureThreshold: 10
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            host: '127.0.0.1'
+            path: /healthz
+            port: 9876
+            scheme: HTTP
+            httpHeaders:
+            - name: "brief"
+              value: "true"
+          failureThreshold: 3
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 5
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        image: "{{IMG}}"
+        imagePullPolicy: IfNotPresent
+        name: cilium-agent
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - SYS_MODULE
+          privileged: true
+        volumeMounts:
+        - mountPath: /sys/fs/bpf
+          name: bpf-maps
+        - mountPath: /var/run/cilium
+          name: cilium-run
+        - mountPath: /tmp/cilium/config-map
+          name: cilium-config-path
+          readOnly: true
+        - mountPath: /lib/modules
+          name: lib-modules
+          readOnly: true
+      hostNetwork: true
+      initContainers:
+      - command:
+        - /init-container.sh
+        env:
+        - name: CILIUM_ALL_STATE
+          valueFrom:
+            configMapKeyRef:
+              key: clean-cilium-state
+              name: cilium-config
+              optional: true
+        - name: CILIUM_BPF_STATE
+          valueFrom:
+            configMapKeyRef:
+              key: clean-cilium-bpf-state
+              name: cilium-config
+              optional: true
+        - name: CILIUM_WAIT_BPF_MOUNT
+          valueFrom:
+            configMapKeyRef:
+              key: wait-bpf-mount
+              name: cilium-config
+              optional: true
+        image: "{{IMG}}"
+        imagePullPolicy: IfNotPresent
+        name: clean-cilium-state
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+          privileged: true
+        volumeMounts:
+        - mountPath: /sys/fs/bpf
+          name: bpf-maps
+          mountPropagation: HostToContainer
+        - mountPath: /var/run/cilium
+          name: cilium-run
+        resources:
+          requests:
+            cpu: 100m
+            memory: 100Mi
+      restartPolicy: Always
+      priorityClassName: system-node-critical
+      terminationGracePeriodSeconds: 1
+      tolerations:
+      - operator: Exists
+      volumes:
+      - hostPath:
+          path: /var/run/cilium
+          type: DirectoryOrCreate
+        name: cilium-run
+      - hostPath:
+          path: /sys/fs/bpf
+          type: DirectoryOrCreate
+        name: bpf-maps
+      - hostPath:
+          path: /lib/modules
+        name: lib-modules
+      - configMap:
+          name: cilium-config
+        name: cilium-config-path

test/l4lb-xdp/kind-config.yaml

@@ -0,0 +1,8 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane # L4LB (kind-control-plane)
+- role: worker # nginx backend (kind-worker)
+networking:
+  disableDefaultCNI: true
+  kubeProxyMode: none

test/l4lb-xdp/test.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -eux
+
+IMG=${1:-quay.io/cilium/cilium-ci:latest}
+
+###########
+#  SETUP  #
+###########
+
+# bpf_xdp_veth_host is a dummy XDP program which is going to be attached to LB
+# node's veth pair end in the host netns. When bpf_xdp, which is attached in
+# the container netns, forwards a LB request with XDP_TX, the request needs to
+# be picked in the host netns by a NAPI handler. To register the handler, we
+# attach the dummy program.
+clang -O2 -Wall -target bpf -c bpf_xdp_veth_host.c -o bpf_xdp_veth_host.o
+
+# The worker (aka backend node) will receive IPIP packets from the LB node.
+# To decapsulate the packets instead of creating an ipip dev which would
+# complicate network setup, we will attach the following program which
+# terminates the tunnel.
+# The program is taken from the Linux kernel selftests.
+clang -O2 -Wall -target bpf -c test_tc_tunnel.c -o test_tc_tunnel.o
+
+# With Kind we create two nodes cluster:
+#
+# * "kind-control-plane" runs cilium in the LB-only mode.
+# * "kind-worker" runs the nginx server.
+#
+# The LB cilium does not connect to the kube-apiserver. For now we use Kind
+# just to create Docker-in-Docker containers.
+kind create cluster --config kind-config.yaml
+cat cilium-lb.yaml | sed "s#{{IMG}}#$IMG#g" | kubectl apply -f -
+
+IFIDX=$(docker exec -i kind-control-plane \
+    /bin/sh -c 'echo $(( $(ip -o l show eth0 | awk "{print $1}" | cut -d: -f1) ))')
+LB_VETH_HOST=$(ip -o l | grep "if$IFIDX" | awk '{print $2}' | cut -d@ -f1)
+ip l set dev $LB_VETH_HOST  xdp obj bpf_xdp_veth_host.o
+
+# Disable TX and RX csum offloading, as veth does not support it. Otherwise,
+# the forwarded packets by the LB to the worker node will have invalid csums.
+ethtool -K $LB_VETH_HOST rx off tx off
+
+docker exec kind-worker /bin/sh -c 'apt-get update && apt-get install -y nginx && systemctl start nginx'
+WORKER_IP=$(docker exec kind-worker ip -o -4 a s eth0 | awk '{print $4}' | cut -d/ -f1)
+nsenter -t $(docker inspect kind-worker -f '{{ .State.Pid }}') -n /bin/sh -c \
+    'tc qdisc add dev eth0 clsact && tc filter add dev eth0 ingress bpf direct-action object-file ./test_tc_tunnel.o section decap'
+
+CILIUM_POD_NAME=$(kubectl -n kube-system get pod -l k8s-app=cilium -o=jsonpath='{.items[0].metadata.name}')
+kubectl -n kube-system wait --for=condition=Ready pod "$CILIUM_POD_NAME" --timeout=5m
+
+##########
+#  TEST #
+##########
+
+LB_VIP="2.2.2.2"
+
+nsenter -t $(docker inspect kind-worker -f '{{ .State.Pid }}') -n /bin/sh -c \
+    "ip a a dev eth0 ${LB_VIP}/32"
+
+kubectl -n kube-system exec $CILIUM_POD_NAME -- \
+    cilium service update --id 1 --frontend "${LB_VIP}:80" --backends "${WORKER_IP}:80" --k8s-node-port
+
+LB_NODE_IP=$(docker exec kind-control-plane ip -o -4 a s eth0 | awk '{print $4}' | cut -d/ -f1)
+ip r a "${LB_VIP}/32" via "$LB_NODE_IP"
+
+# Issue 10 requests to LB
+for i in $(seq 1 10); do
+    curl -o /dev/null "${LB_VIP}:80"
+done