conformance-ipsec-e2e: add leaked unencrypted packets check

Extend the conformance-ipsec-e2e GHA workflow to additionally check that
we don't leak any unencrypted packets during the connectivity test.
This aims to complement the validation already performed as part of
the connectivity tests by the Cilium CLI.

Specifically, we leverage bpftrace to analyze the packets forwarded by
the bridge device (used by kind), and report those that are not encrypted.
We flag packets with both the source and the destination belonging to
the IPv4/6 PodCIDR, and we consider the inner headers if packets are
encapsulated. In this case, we additionally skip packets originating
or targeting CiliumInternalIP addresses (as these are used for node-to-pod
traffic when running in tunnel mode, which is not encrypted by design).

Extra checks are finally added to always include packets originating
from the L7 and DNS proxies, as their source IP is not that of a pod.

Signed-off-by: Marco Iorio <marco.iorio@isovalent.com>

.github/actions/bpftrace/check/action.yaml

@@ -0,0 +1,35 @@
+name: Assert bpftrace script output
+description: Stops the background bpftrace process, asserts that it completed successfully and did not write anything to stdout
+
+inputs:
+  output-path:
+    description: "Directory where the output files are stored to"
+    default: "."
+
+runs:
+  using: composite
+  steps:
+    - name: Assert that bpftrace completed successfully
+      uses: cilium/little-vm-helper@8410a93e544b7e180a2365e5fdab0724a39bc02a # v0.0.13
+      with:
+        provision: 'false'
+        cmd: |
+          cd /host/
+
+          if [[ "\$(wc -l < ${{ inputs.output-path }}/bpftrace.err)" -ne 0 ]];
+          then
+            echo "Unexpected error reported by bpftrace"
+            cat ${{ inputs.output-path }}/bpftrace.err
+            exit 1
+          fi
+
+          pkill -F ${{ inputs.output-path }}/bpftrace.pid || { echo "Failed to stop bpftrace"; exit 1; }
+          # Wait until bpftrace terminates, so that the output is complete
+          while pgrep -F ${{ inputs.output-path }}/bpftrace.pid > /dev/null; do sleep 1; done
+
+          if [[ "\$(grep -cvE '(^\s*$)' ${{ inputs.output-path }}/bpftrace.out)" -ne 0 ]];
+          then
+            echo "Error: bpftrace output is not empty"
+            cat ${{ inputs.output-path }}/bpftrace.out
+            exit 1
+          fi

.github/actions/bpftrace/scripts/check-ipsec-leaks.bt

@@ -0,0 +1,223 @@
+// Six parameters expected:
+// $1: IPv4 CiliumInternalIP - Node1
+// $2: IPv6 CiliumInternalIP - Node1
+// $3: IPv4 CiliumInternalIP - Node2
+// $4: IPv6 CiliumInternalIP - Node2
+// $5: IPv4 CiliumInternalIP - Node3
+// $6: IPv6 CiliumInternalIP - Node3
+
+#define CIDR4 (uint32)0x0A000000 // 10.0.0.0/8
+#define MASK4 (uint32)0xFF000000
+#define CIDR6 (uint32)0xfd00
+
+#define PROTO_IPV4 0x0800
+#define PROTO_IPV6 0x86DD
+#define PROTO_TCP 6
+#define PROTO_UDP 17
+#define PROTO_ESP 50
+
+#define AF_INET	2
+#define AF_INET6 10
+
+#define PORT_DNS 53
+
+#define TYPE_PROXY_L7_IP4 1
+#define TYPE_PROXY_L7_IP6 2
+#define TYPE_PROXY_DNS_IP4 3
+#define TYPE_PROXY_DNS_IP6 4
+
+kprobe:br_forward
+{
+  $skb = ((struct sk_buff *) arg1);
+
+  $proto = bswap($skb->protocol);
+  $ip4h = ((struct iphdr *) ($skb->head + $skb->network_header));
+  $ip6h = ((struct ipv6hdr *) ($skb->head + $skb->network_header));
+  $udph = ((struct udphdr*) ($skb->head + $skb->transport_header));
+
+  if ($skb->encapsulation) {
+    // $skb->inner_protocol does not appear to be correctly initialized
+    $proto = bswap(*((uint16*) ($skb->head + $skb->inner_mac_header + 12)));
+    $ip4h = ((struct iphdr*) ($skb->head + $skb->inner_network_header));
+    $ip6h = ((struct ipv6hdr*) ($skb->head + $skb->inner_network_header));
+    $udph = ((struct udphdr*) ($skb->head + $skb->inner_transport_header));
+
+    if ($proto == PROTO_IPV4) {
+      $trace_override =
+        @trace_ip4[$ip4h->saddr, $udph->source, $ip4h->protocol] ||
+        @trace_ip4[$ip4h->daddr, $udph->dest, $ip4h->protocol];
+
+      // Skip CiliumInternalIP addresses, as they belong to the PodCIDR,
+      // unless the given flow is explicitly marked as traced (i.e., from proxy).
+      if (!$trace_override &&
+           ($ip4h->saddr == (uint32)pton(str($1)) || $ip4h->daddr == (uint32)pton(str($1)) ||
+            $ip4h->saddr == (uint32)pton(str($3)) || $ip4h->daddr == (uint32)pton(str($3)) ||
+            $ip4h->saddr == (uint32)pton(str($5)) || $ip4h->daddr == (uint32)pton(str($5)))) {
+        return;
+      }
+    }
+
+    if ($proto == PROTO_IPV6) {
+      $trace_override =
+        @trace_ip6[$ip6h->saddr.in6_u.u6_addr8, $udph->source, $ip6h->nexthdr] ||
+        @trace_ip6[$ip6h->daddr.in6_u.u6_addr8, $udph->dest, $ip6h->nexthdr];
+
+      // Skip CiliumInternalIP addresses, as they belong to the PodCIDR
+      // unless the given flow is explicitly marked as traced (i.e., from proxy).
+      if (!$trace_override &&
+           ($ip6h->saddr.in6_u.u6_addr8 == pton(str($2)) || $ip6h->daddr.in6_u.u6_addr8 == pton(str($2)) ||
+            $ip6h->saddr.in6_u.u6_addr8 == pton(str($4)) || $ip6h->daddr.in6_u.u6_addr8 == pton(str($4)) ||
+            $ip6h->saddr.in6_u.u6_addr8 == pton(str($6)) || $ip6h->daddr.in6_u.u6_addr8 == pton(str($6)))) {
+        return;
+      }
+    }
+  }
+
+  if ($proto == PROTO_IPV4 && $ip4h->protocol != PROTO_ESP) {
+    $src_is_pod = (bswap($ip4h->saddr) & MASK4) == CIDR4;
+    $dst_is_pod = (bswap($ip4h->daddr) & MASK4) == CIDR4;
+
+    $trace_override =
+        @trace_ip4[$ip4h->saddr, $udph->source, $ip4h->protocol] ||
+        @trace_ip4[$ip4h->daddr, $udph->dest, $ip4h->protocol];
+
+    if (($src_is_pod && $dst_is_pod) || ($trace_override && ($src_is_pod || $dst_is_pod))) {
+      printf("[%s] %s:%d -> %s:%d (proto: %d, ifindex: %d, netns: %x)\n",
+        strftime("%H:%M:%S:%f", nsecs),
+        ntop($ip4h->saddr), bswap($udph->source),
+        ntop($ip4h->daddr), bswap($udph->dest),
+        $ip4h->protocol,
+        $skb->dev->ifindex,
+        $skb->dev->nd_net.net->ns.inum);
+    }
+  }
+
+  if ($proto == PROTO_IPV6 && $ip6h->nexthdr != PROTO_ESP) {
+    $src_is_pod = bswap($ip6h->saddr.in6_u.u6_addr16[0]) == CIDR6;
+    $dst_is_pod = bswap($ip6h->daddr.in6_u.u6_addr16[0]) == CIDR6;
+
+    $trace_override =
+        @trace_ip6[$ip6h->saddr.in6_u.u6_addr8, $udph->source, $ip6h->nexthdr] ||
+        @trace_ip6[$ip6h->daddr.in6_u.u6_addr8, $udph->dest, $ip6h->nexthdr];
+
+    if (($src_is_pod && $dst_is_pod) || ($trace_override && ($src_is_pod || $dst_is_pod))) {
+      printf("[%s] %s:%d -> %s:%d (proto: %d, ifindex: %d, netns: %x)\n",
+        strftime("%H:%M:%S:%f", nsecs),
+        ntop($ip6h->saddr.in6_u.u6_addr8), bswap($udph->source),
+        ntop($ip6h->daddr.in6_u.u6_addr8), bswap($udph->dest),
+        $ip6h->nexthdr,
+        $skb->dev->ifindex,
+        $skb->dev->nd_net.net->ns.inum);
+    }
+  }
+}
+
+// Trace TCP connections established by the L7 proxy, even if the source address belongs to the host.
+kprobe:tcp_connect
+{
+  if (strncmp(comm, "wrk:", 4) != 0) {
+    return;
+  }
+
+  $sk = ((struct sock *) arg0);
+  $inet_family = $sk->__sk_common.skc_family;
+
+  if ($inet_family == AF_INET) {
+    @trace_ip4[$sk->__sk_common.skc_rcv_saddr, bswap($sk->__sk_common.skc_num), PROTO_TCP] = true;
+    @trace_sk[$sk] = true;
+    @sanity[TYPE_PROXY_L7_IP4] = true;
+  }
+
+  if ($inet_family == AF_INET6) {
+    @trace_ip6[$sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8, bswap($sk->__sk_common.skc_num), PROTO_TCP] = true;
+    @trace_sk[$sk] = true;
+    @sanity[TYPE_PROXY_L7_IP6] = true;
+  }
+}
+
+kprobe:tcp_close
+{
+  $sk = ((struct sock *) arg0);
+  $inet_family = $sk->__sk_common.skc_family;
+
+  if ($inet_family == AF_INET) {
+    delete(@trace_ip4[$sk->__sk_common.skc_rcv_saddr, bswap($sk->__sk_common.skc_num), PROTO_TCP]);
+  }
+
+  if ($inet_family == AF_INET6) {
+    delete(@trace_ip6[$sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8, bswap($sk->__sk_common.skc_num), PROTO_TCP]);
+  }
+}
+
+// Trace UDP messages sent by the DNS proxy, even if the source address belongs to the host.
+kprobe:udp_sendmsg /comm == "cilium-agent" || comm == "dnsproxy"/
+{
+  $sk = ((struct sock *) arg0);
+  if (bswap($sk->__sk_common.skc_dport) == PORT_DNS) {
+    @trace_ip4[$sk->__sk_common.skc_rcv_saddr, bswap($sk->__sk_common.skc_num), PROTO_UDP] = true;
+    @trace_sk[$sk] = true;
+    @sanity[TYPE_PROXY_DNS_IP4] = true;
+  }
+}
+
+// Trace UDP6 messages sent by the DNS proxy, even if the source address belongs to the host.
+kprobe:udpv6_sendmsg /comm == "cilium-agent" || comm == "dnsproxy"/
+{
+  $sk = ((struct sock *) arg0);
+  if ($sk->__sk_common.skc_num == PORT_DNS) {
+    @trace_ip6[$sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8, bswap($sk->__sk_common.skc_num), PROTO_UDP] = true;
+    @trace_sk[$sk] = true;
+    @sanity[TYPE_PROXY_DNS_IP6] = true;
+  }
+}
+
+// Additionally trace traffic flows in which the source got masquerated.
+kprobe:__dev_queue_xmit
+{
+  $skb = ((struct sk_buff *) arg0);
+  $sk = $skb->sk;
+
+  if ($sk == 0 || !@trace_sk[$sk]) {
+    return;
+  }
+
+  $proto = bswap($skb->protocol);
+  $ip4h = ((struct iphdr *) ($skb->head + $skb->network_header));
+  $ip6h = ((struct ipv6hdr *) ($skb->head + $skb->network_header));
+  $udph = ((struct udphdr*) ($skb->head + $skb->transport_header));
+  $l4proto = $proto == PROTO_IPV4 ? $ip4h->protocol : $ip6h->nexthdr;
+
+  if ($l4proto == PROTO_TCP) {
+    @sanity[$proto == PROTO_IPV4 ? TYPE_PROXY_L7_IP4 : TYPE_PROXY_L7_IP6] = true;
+  } else {
+    @sanity[$proto == PROTO_IPV4 ? TYPE_PROXY_DNS_IP4 : TYPE_PROXY_DNS_IP6] = true;
+  }
+
+  if ($proto == PROTO_IPV4) {
+    @trace_ip4[$ip4h->saddr, $udph->source, $l4proto] = true;
+  } else {
+    @trace_ip6[$ip6h->saddr.in6_u.u6_addr8, $udph->source, $l4proto] = true;
+  }
+
+  delete(@trace_sk[$sk])
+}
+
+END
+{
+  if (!@sanity[TYPE_PROXY_L7_IP4]) {
+    printf("Sanity check failed: detected no IPv4 connections from the L7 proxy. Is the filter correct?\n")
+  }
+
+  if (!@sanity[TYPE_PROXY_L7_IP6] && str($2) != "::1") {
+    printf("Sanity check failed: detected no IPv6 connections from the L7 proxy. Is the filter correct?\n")
+  }
+
+  if (!(@sanity[TYPE_PROXY_DNS_IP4] || @sanity[TYPE_PROXY_DNS_IP6])) {
+    printf("Sanity check failed: detected no messages sent by the DNS proxy. Is the filter correct?\n")
+  }
+
+  clear(@trace_ip4);
+  clear(@trace_ip6);
+  clear(@trace_sk);
+  clear(@sanity);
+}

.github/actions/bpftrace/start/action.yaml

@@ -0,0 +1,51 @@
+name: Start bpftrace script in background
+description: Starts the given bpftrace script in background
+
+inputs:
+  script:
+    description: "The path of the bpftrace program to execute"
+    required: true
+  args:
+    description: "The arguments propagated to the bpftrace script"
+    default: ""
+  output-path:
+    description: "Directory where the output files are stored to"
+    default: "."
+
+runs:
+  using: composite
+  steps:
+    - name: Install bpftrace if not already present
+      uses: cilium/little-vm-helper@8410a93e544b7e180a2365e5fdab0724a39bc02a # v0.0.13
+      with:
+        provision: 'false'
+        cmd: |
+          if ! command -v bpftrace &> /dev/null; then
+            # bpftrace v0.20.1 doesn't seem to play well with Linux 4.19
+            # https://github.com/bpftrace/bpftrace/issues/3011
+            # Let's buy us some time, and keep installing v0.19.1 for the moment.
+
+            curl -L https://github.com/bpftrace/bpftrace/releases/download/v0.19.1/bpftrace -o bpftrace
+            install -m 755 bpftrace /usr/local/bin/bpftrace
+
+          fi
+
+    - name: Start bpftrace in background
+      id: run
+      uses: cilium/little-vm-helper@8410a93e544b7e180a2365e5fdab0724a39bc02a # v0.0.13
+      with:
+        provision: 'false'
+        cmd: |
+          cd /host/
+
+          if [[ -f "/boot/btf-\$(uname -r)" ]]; then
+            export BPFTRACE_BTF="/boot/btf-\$(uname -r)"
+          fi
+
+          bpftrace ${{ inputs.script }} -q \
+            ${{ inputs.args }} \
+            > ${{ inputs.output-path }}/bpftrace.out \
+            2> ${{ inputs.output-path }}/bpftrace.err \
+            < /dev/null &
+
+          echo \$! > ${{ inputs.output-path }}/bpftrace.pid

.github/workflows/conformance-ipsec-e2e.yaml

@@ -287,7 +287,7 @@ jobs:
             until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
           done
 
-      - name: Run tests (${{ join(matrix.*, ', ') }})
+      - name: Install Cilium
         shell: bash
         run: |
           kubectl patch node kind-worker3 --type=json -p='[{"op":"add","path":"/metadata/labels/cilium.io~1no-schedule","value":"true"}]'
@@ -310,6 +310,25 @@ jobs:
           kubectl get pods --all-namespaces -o wide
           kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium-dbg status
 
+      - name: Prepare the bpftrace parameters
+        id: bpftrace-params
+        run: |
+          CILIUM_INTERNAL_IPS=$(kubectl get ciliumnode -o jsonpath='{.items[*].spec.addresses[?(@.type=="CiliumInternalIP")].ip}')
+          if [[ "${{ matrix.ipv6 }}" == "false" ]]; then
+            CILIUM_INTERNAL_IPS="${CILIUM_INTERNAL_IPS// / ::1 } ::1"
+          fi
+
+          echo "params=$CILIUM_INTERNAL_IPS" >> $GITHUB_OUTPUT
+
+      - name: Start unencrypted packets check
+        uses: ./.github/actions/bpftrace/start
+        with:
+          script: ./.github/actions/bpftrace/scripts/check-ipsec-leaks.bt
+          args: ${{ steps.bpftrace-params.outputs.params }}
+
+      - name: Run tests (${{ join(matrix.*, ', ') }})
+        shell: bash
+        run: |
           mkdir -p cilium-junits
 
           ./cilium-cli connectivity test --include-unsafe-tests --collect-sysdump-on-failure \
@@ -335,6 +354,9 @@ jobs:
         with:
           full-test: 'true'
 
+      - name: Assert that no unencrypted packets are leaked
+        uses: ./.github/actions/bpftrace/check
+
       - name: Fetch artifacts
         if: ${{ !success() }}
         shell: bash