[1.13] ci: Add IPsec leak detection for ci-ipsec-e2e

.github/actions/bpftrace/check/action.yaml

@@ -0,0 +1,35 @@
+name: Assert bpftrace script output
+description: Stops the background bpftrace process, asserts that it completed successfully and did not write anything to stdout
+
+inputs:
+  output-path:
+    description: "Directory where the output files are stored to"
+    default: "."
+
+runs:
+  using: composite
+  steps:
+    - name: Assert that bpftrace completed successfully
+      uses: cilium/little-vm-helper@8410a93e544b7e180a2365e5fdab0724a39bc02a # v0.0.13
+      with:
+        provision: 'false'
+        cmd: |
+          cd /host/
+
+          if [[ "\$(wc -l < ${{ inputs.output-path }}/bpftrace.err)" -ne 0 ]];
+          then
+            echo "Unexpected error reported by bpftrace"
+            cat ${{ inputs.output-path }}/bpftrace.err
+            exit 1
+          fi
+
+          pkill -F ${{ inputs.output-path }}/bpftrace.pid || { echo "Failed to stop bpftrace"; exit 1; }
+          # Wait until bpftrace terminates, so that the output is complete
+          while pgrep -F ${{ inputs.output-path }}/bpftrace.pid > /dev/null; do sleep 1; done
+
+          if [[ "\$(grep -cvE '(^\s*$)' ${{ inputs.output-path }}/bpftrace.out)" -ne 0 ]];
+          then
+            echo "Error: bpftrace output is not empty"
+            cat ${{ inputs.output-path }}/bpftrace.out
+            exit 1
+          fi

.github/actions/bpftrace/scripts/check-ipsec-leaks.bt

@@ -0,0 +1,224 @@
+// Six parameters expected:
+// $1: IPv4 CiliumInternalIP - Node1
+// $2: IPv6 CiliumInternalIP - Node1
+// $3: IPv4 CiliumInternalIP - Node2
+// $4: IPv6 CiliumInternalIP - Node2
+// $5: IPv4 CiliumInternalIP - Node3
+// $6: IPv6 CiliumInternalIP - Node3
+// $7: Report errors if proxy traffic not found - [true|false]
+
+#define CIDR4 (uint32)0x0A000000 // 10.0.0.0/8
+#define MASK4 (uint32)0xFF000000
+#define CIDR6 (uint32)0xfd00
+
+#define PROTO_IPV4 0x0800
+#define PROTO_IPV6 0x86DD
+#define PROTO_TCP 6
+#define PROTO_UDP 17
+#define PROTO_ESP 50
+
+#define AF_INET	2
+#define AF_INET6 10
+
+#define PORT_DNS 53
+
+#define TYPE_PROXY_L7_IP4 1
+#define TYPE_PROXY_L7_IP6 2
+#define TYPE_PROXY_DNS_IP4 3
+#define TYPE_PROXY_DNS_IP6 4
+
+kprobe:br_forward
+{
+  $skb = ((struct sk_buff *) arg1);
+
+  $proto = bswap($skb->protocol);
+  $ip4h = ((struct iphdr *) ($skb->head + $skb->network_header));
+  $ip6h = ((struct ipv6hdr *) ($skb->head + $skb->network_header));
+  $udph = ((struct udphdr*) ($skb->head + $skb->transport_header));
+
+  if ($skb->encapsulation) {
+    // $skb->inner_protocol does not appear to be correctly initialized
+    $proto = bswap(*((uint16*) ($skb->head + $skb->inner_mac_header + 12)));
+    $ip4h = ((struct iphdr*) ($skb->head + $skb->inner_network_header));
+    $ip6h = ((struct ipv6hdr*) ($skb->head + $skb->inner_network_header));
+    $udph = ((struct udphdr*) ($skb->head + $skb->inner_transport_header));
+
+    if ($proto == PROTO_IPV4) {
+      $trace_override =
+        @trace_ip4[$ip4h->saddr, $udph->source, $ip4h->protocol] ||
+        @trace_ip4[$ip4h->daddr, $udph->dest, $ip4h->protocol];
+
+      // Skip CiliumInternalIP addresses, as they belong to the PodCIDR,
+      // unless the given flow is explicitly marked as traced (i.e., from proxy).
+      if (!$trace_override &&
+           ($ip4h->saddr == (uint32)pton(str($1)) || $ip4h->daddr == (uint32)pton(str($1)) ||
+            $ip4h->saddr == (uint32)pton(str($3)) || $ip4h->daddr == (uint32)pton(str($3)) ||
+            $ip4h->saddr == (uint32)pton(str($5)) || $ip4h->daddr == (uint32)pton(str($5)))) {
+        return;
+      }
+    }
+
+    if ($proto == PROTO_IPV6) {
+      $trace_override =
+        @trace_ip6[$ip6h->saddr.in6_u.u6_addr8, $udph->source, $ip6h->nexthdr] ||
+        @trace_ip6[$ip6h->daddr.in6_u.u6_addr8, $udph->dest, $ip6h->nexthdr];
+
+      // Skip CiliumInternalIP addresses, as they belong to the PodCIDR
+      // unless the given flow is explicitly marked as traced (i.e., from proxy).
+      if (!$trace_override &&
+           ($ip6h->saddr.in6_u.u6_addr8 == pton(str($2)) || $ip6h->daddr.in6_u.u6_addr8 == pton(str($2)) ||
+            $ip6h->saddr.in6_u.u6_addr8 == pton(str($4)) || $ip6h->daddr.in6_u.u6_addr8 == pton(str($4)) ||
+            $ip6h->saddr.in6_u.u6_addr8 == pton(str($6)) || $ip6h->daddr.in6_u.u6_addr8 == pton(str($6)))) {
+        return;
+      }
+    }
+  }
+
+  if ($proto == PROTO_IPV4 && $ip4h->protocol != PROTO_ESP) {
+    $src_is_pod = (bswap($ip4h->saddr) & MASK4) == CIDR4;
+    $dst_is_pod = (bswap($ip4h->daddr) & MASK4) == CIDR4;
+
+    $trace_override =
+        @trace_ip4[$ip4h->saddr, $udph->source, $ip4h->protocol] ||
+        @trace_ip4[$ip4h->daddr, $udph->dest, $ip4h->protocol];
+
+    if (($src_is_pod && $dst_is_pod) || ($trace_override && ($src_is_pod || $dst_is_pod))) {
+      printf("[%s] %s:%d -> %s:%d (proto: %d, ifindex: %d, netns: %x)\n",
+        strftime("%H:%M:%S:%f", nsecs),
+        ntop($ip4h->saddr), bswap($udph->source),
+        ntop($ip4h->daddr), bswap($udph->dest),
+        $ip4h->protocol,
+        $skb->dev->ifindex,
+        $skb->dev->nd_net.net->ns.inum);
+    }
+  }
+
+  if ($proto == PROTO_IPV6 && $ip6h->nexthdr != PROTO_ESP) {
+    $src_is_pod = bswap($ip6h->saddr.in6_u.u6_addr16[0]) == CIDR6;
+    $dst_is_pod = bswap($ip6h->daddr.in6_u.u6_addr16[0]) == CIDR6;
+
+    $trace_override =
+        @trace_ip6[$ip6h->saddr.in6_u.u6_addr8, $udph->source, $ip6h->nexthdr] ||
+        @trace_ip6[$ip6h->daddr.in6_u.u6_addr8, $udph->dest, $ip6h->nexthdr];
+
+    if (($src_is_pod && $dst_is_pod) || ($trace_override && ($src_is_pod || $dst_is_pod))) {
+      printf("[%s] %s:%d -> %s:%d (proto: %d, ifindex: %d, netns: %x)\n",
+        strftime("%H:%M:%S:%f", nsecs),
+        ntop($ip6h->saddr.in6_u.u6_addr8), bswap($udph->source),
+        ntop($ip6h->daddr.in6_u.u6_addr8), bswap($udph->dest),
+        $ip6h->nexthdr,
+        $skb->dev->ifindex,
+        $skb->dev->nd_net.net->ns.inum);
+    }
+  }
+}
+
+// Trace TCP connections established by the L7 proxy, even if the source address belongs to the host.
+kprobe:tcp_connect
+{
+  if (strncmp(comm, "wrk:", 4) != 0) {
+    return;
+  }
+
+  $sk = ((struct sock *) arg0);
+  $inet_family = $sk->__sk_common.skc_family;
+
+  if ($inet_family == AF_INET) {
+    @trace_ip4[$sk->__sk_common.skc_rcv_saddr, bswap($sk->__sk_common.skc_num), PROTO_TCP] = true;
+    @trace_sk[$sk] = true;
+    @sanity[TYPE_PROXY_L7_IP4] = true;
+  }
+
+  if ($inet_family == AF_INET6) {
+    @trace_ip6[$sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8, bswap($sk->__sk_common.skc_num), PROTO_TCP] = true;
+    @trace_sk[$sk] = true;
+    @sanity[TYPE_PROXY_L7_IP6] = true;
+  }
+}
+
+kprobe:tcp_close
+{
+  $sk = ((struct sock *) arg0);
+  $inet_family = $sk->__sk_common.skc_family;
+
+  if ($inet_family == AF_INET) {
+    delete(@trace_ip4[$sk->__sk_common.skc_rcv_saddr, bswap($sk->__sk_common.skc_num), PROTO_TCP]);
+  }
+
+  if ($inet_family == AF_INET6) {
+    delete(@trace_ip6[$sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8, bswap($sk->__sk_common.skc_num), PROTO_TCP]);
+  }
+}
+
+// Trace UDP messages sent by the DNS proxy, even if the source address belongs to the host.
+kprobe:udp_sendmsg /comm == "cilium-agent" || comm == "dnsproxy"/
+{
+  $sk = ((struct sock *) arg0);
+  if (bswap($sk->__sk_common.skc_dport) == PORT_DNS) {
+    @trace_ip4[$sk->__sk_common.skc_rcv_saddr, bswap($sk->__sk_common.skc_num), PROTO_UDP] = true;
+    @trace_sk[$sk] = true;
+    @sanity[TYPE_PROXY_DNS_IP4] = true;
+  }
+}
+
+// Trace UDP6 messages sent by the DNS proxy, even if the source address belongs to the host.
+kprobe:udpv6_sendmsg /comm == "cilium-agent" || comm == "dnsproxy"/
+{
+  $sk = ((struct sock *) arg0);
+  if ($sk->__sk_common.skc_num == PORT_DNS) {
+    @trace_ip6[$sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr8, bswap($sk->__sk_common.skc_num), PROTO_UDP] = true;
+    @trace_sk[$sk] = true;
+    @sanity[TYPE_PROXY_DNS_IP6] = true;
+  }
+}
+
+// Additionally trace traffic flows in which the source got masquerated.
+kprobe:__dev_queue_xmit
+{
+  $skb = ((struct sk_buff *) arg0);
+  $sk = $skb->sk;
+
+  if ($sk == 0 || !@trace_sk[$sk]) {
+    return;
+  }
+
+  $proto = bswap($skb->protocol);
+  $ip4h = ((struct iphdr *) ($skb->head + $skb->network_header));
+  $ip6h = ((struct ipv6hdr *) ($skb->head + $skb->network_header));
+  $udph = ((struct udphdr*) ($skb->head + $skb->transport_header));
+  $l4proto = $proto == PROTO_IPV4 ? $ip4h->protocol : $ip6h->nexthdr;
+
+  if ($l4proto == PROTO_TCP) {
+    @sanity[$proto == PROTO_IPV4 ? TYPE_PROXY_L7_IP4 : TYPE_PROXY_L7_IP6] = true;
+  } else {
+    @sanity[$proto == PROTO_IPV4 ? TYPE_PROXY_DNS_IP4 : TYPE_PROXY_DNS_IP6] = true;
+  }
+
+  if ($proto == PROTO_IPV4) {
+    @trace_ip4[$ip4h->saddr, $udph->source, $l4proto] = true;
+  } else {
+    @trace_ip6[$ip6h->saddr.in6_u.u6_addr8, $udph->source, $l4proto] = true;
+  }
+
+  delete(@trace_sk[$sk])
+}
+
+END
+{
+  if (str($7) == "true" && !@sanity[TYPE_PROXY_L7_IP4]) {
+    printf("Sanity check failed: detected no IPv4 connections from the L7 proxy. Is the filter correct?\n")
+  }
+
+  if (str($7) == "true" && !@sanity[TYPE_PROXY_L7_IP6] && str($2) != "::1") {
+    printf("Sanity check failed: detected no IPv6 connections from the L7 proxy. Is the filter correct?\n")
+  }
+
+  if (str($7) == "true" && !(@sanity[TYPE_PROXY_DNS_IP4] || @sanity[TYPE_PROXY_DNS_IP6])) {
+    printf("Sanity check failed: detected no messages sent by the DNS proxy. Is the filter correct?\n")
+  }
+
+  clear(@trace_ip4);
+  clear(@trace_ip6);
+  clear(@trace_sk);
+  clear(@sanity);
+}

.github/actions/bpftrace/start/action.yaml

@@ -0,0 +1,51 @@
+name: Start bpftrace script in background
+description: Starts the given bpftrace script in background
+
+inputs:
+  script:
+    description: "The path of the bpftrace program to execute"
+    required: true
+  args:
+    description: "The arguments propagated to the bpftrace script"
+    default: ""
+  output-path:
+    description: "Directory where the output files are stored to"
+    default: "."
+
+runs:
+  using: composite
+  steps:
+    - name: Install bpftrace if not already present
+      uses: cilium/little-vm-helper@8410a93e544b7e180a2365e5fdab0724a39bc02a # v0.0.13
+      with:
+        provision: 'false'
+        cmd: |
+          if ! command -v bpftrace &> /dev/null; then
+            # bpftrace v0.20.1 doesn't seem to play well with Linux 4.19
+            # https://github.com/bpftrace/bpftrace/issues/3011
+            # Let's buy us some time, and keep installing v0.19.1 for the moment.
+
+            curl -L https://github.com/bpftrace/bpftrace/releases/download/v0.19.1/bpftrace -o bpftrace
+            install -m 755 bpftrace /usr/local/bin/bpftrace
+
+          fi
+
+    - name: Start bpftrace in background
+      id: run
+      uses: cilium/little-vm-helper@8410a93e544b7e180a2365e5fdab0724a39bc02a # v0.0.13
+      with:
+        provision: 'false'
+        cmd: |
+          cd /host/
+
+          if [[ -f "/boot/btf-\$(uname -r)" ]]; then
+            export BPFTRACE_BTF="/boot/btf-\$(uname -r)"
+          fi
+
+          bpftrace ${{ inputs.script }} -q \
+            ${{ inputs.args }} \
+            > ${{ inputs.output-path }}/bpftrace.out \
+            2> ${{ inputs.output-path }}/bpftrace.err \
+            < /dev/null &
+
+          echo \$! > ${{ inputs.output-path }}/bpftrace.pid

.github/actions/conn-disrupt-test-check/action.yaml

@@ -0,0 +1,39 @@
+name: Cilium Conn Disrupt Check
+description: Check cilium connection disruption status
+
+inputs:
+  job-name:
+    required: false
+    description: 'Job name used in Cilium sysdump filename'
+  cilium-cli:
+    required: false
+    default: "./cilium-cli"
+    description: 'Path to the Cilium CLI binary'
+  extra-connectivity-test-flags:
+    required: false
+    description: 'Cilium CLI connectivity tests extra flags'
+  full-test:
+    required: false
+    default: 'false'
+    description: 'Run full connectivity test suite'
+
+runs:
+  using: composite
+  steps:
+    - name: Perform Conn Disrupt Test
+      shell: bash
+      run: |
+        TEST_ARG="no-interrupted-connections"
+        if [[ "${{ inputs.full-test }}" == "true" ]]; then
+          TEST_ARG=""
+        fi
+        ${{ inputs.cilium-cli }} connectivity test --include-unsafe-tests --collect-sysdump-on-failure \
+          --include-conn-disrupt-test \
+          --flush-ct \
+          --sysdump-hubble-flows-count=1000000 --sysdump-hubble-flows-timeout=5m \
+          --sysdump-output-filename "cilium-sysdump-conn-disrupt-test-${{ inputs.job-name }}-<ts>" \
+          --junit-file "cilium-junits/conn-disrupt-test-${{ inputs.job-name }}.xml" \
+          ${{ inputs.extra-connectivity-test-flags }} \
+          --junit-property github_job_step="Run conn disrupt tests (${{ inputs.job-name }})" \
+          --test "$TEST_ARG" \
+          --expected-xfrm-errors "+inbound_no_state"

.github/actions/conn-disrupt-test-setup/action.yaml

@@ -0,0 +1,21 @@
+name: Cilium Conn Disrupt Setup
+description: Setup cilium connection disruption test
+
+inputs:
+  cilium-cli:
+    required: false
+    default: "./cilium-cli"
+    description: 'Path to the Cilium CLI binary'
+
+runs:
+  using: composite
+  steps:
+    - name: Setup Conn Disrupt Test
+      shell: bash
+      run: |
+        # Create pods which establish long lived connections. It will be used by
+        # subsequent connectivity tests with --include-conn-disrupt-test to catch any
+        # interruption in such flows.
+        ${{ inputs.cilium-cli }} connectivity test --include-conn-disrupt-test --conn-disrupt-test-setup \
+          --conn-disrupt-dispatch-interval 0ms \
+          --expected-xfrm-errors "+inbound_no_state"