Flesh out README

README.md

@@ -1,5 +1,100 @@
 # ape - an XDP packet manipulation tool
 
-Ape is a tool to manipulate UDP packets.
+Ape is a tool to manipulate UDP (both ipv4 and ipv6) packets.
 
 The learning resources I used to create this project is https://github.com/xdp-project/xdp-tutorial (highly recommended if you want to get started with XDP). `headers/` and `common/` are copied from it.
+
+### architecture
+
+Ape looks stitched together, because it is. It consists of the following pieces:
+
+1. Userspace XDP loaders, named `xdp_user_*.c` - these must be built **before** using ape, with `make clean all`
+2. Kernel XDP programs, named `xdp_kern_*.c` - these are compiled at runtime by `ape.py` based on command-line arguments which are converted to `-D` compiler switches
+3. `ape.py`, the main Python script and entrypoint of this project, which parses command-line arguments, builds and attaches the XDP kernel objects, and exposes XDP metrics to Prometheus using `bpftool`
+
+`ape.py` must be run with sudo to execute privileged actions e.g. attaching XDP programs to NICs, but it drops privileges to `SUDO_UID` when compiling the kernel modules to not create root-owned files in the repository.
+
+### drop
+
+Drop UDP4/6 packets from an interface, with an optional port.
+
+ape, dropping 50% of UDP packets on port 1337 on lo:
+```
+sevagh:ape $ sudo ./ape.py --udp-drop 50 --udp-port 1337 lo
+b'Success: Loaded BPF-object(xdp_kern_drop.o) and used section(xdp_ape_drop)\n - XDP prog attached on device:lo(ifindex:1)\n - Unpinning (remove) prev maps in /sys/fs/bpf/lo/\n - Pinning maps in /sys/fs/bpf/lo/\n'
+Starting bpftool map listener in thread
+Started prometheus metrics server at http://localhost:8000
+```
+
+socat sender:
+```
+sevagh:~ $ for x in 1 2 3 4 5 6 7 8 9 10; do echo "hello world ${x}" | socat - UDP6-SENDTO:[::1]:1337; done
+```
+
+socat receiver:
+```
+sevagh:~ $ socat - UDP6-LISTEN:1337,bind=[::1],fork
+hello world 1
+hello world 2
+hello world 4
+hello world 5
+hello world 7
+hello world 9
+hello world 10
+```
+
+We can see packets `3, 6, 8` missing. The Prometheus metrics show it, at `http://127.0.0.1:8000`:
+
+```
+ape_total_udp_packets{action="drop",device="lo",port="1337"} 10.0
+# HELP ape_manipulated_udp_packets UDP packets manipulated by ape
+# TYPE ape_manipulated_udp_packets gauge
+ape_manipulated_udp_packets{action="drop",device="lo",port="1337"} 3.0
+```
+
+### scramble
+
+Scramble UDP4/6 packets from an interface, with an optional port.
+
+The implementation of scramble is much more complex than drop. There's no way to copy a packet, or "sleep", in XDP.
+
+I need to use [`AF_XDP`](https://www.kernel.org/doc/html/latest/networking/af_xdp.html), which is a way to redirect packets from the XDP kernel program to userspace. Inside `xdp_user_scramble.c`, I set up UDP4 and UDP6 sender sockets. When the XDP kernel scramble program redirects a packet to the XSK map and I receive it in the user scramble program, I sleep randomly between 0-MAX_SLEEP_MS ms before re-sending it on the appropriate UDP4 or UDP6 socket. This results in packets seemingly being received out of order.
+
+ape, scrambling 50% of UDP packets on port 1337 on lo:
+```
+sevagh:ape $ sudo ./ape.py --udp-scramble 50 --udp-port 1337 lo
+sleeping 2s to let module load
+Starting bpftool map listener in thread
+Started prometheus metrics server at http://localhost:8000
+```
+
+socat sender:
+```
+sevagh:~ $ for x in 1 2 3 4 5 6 7 8 9 10; do echo "hello world ${x}" | socat - UDP6-SENDTO:[::1]:1337; done
+```
+
+socat receiver:
+```
+sevagh:~ $ socat - UDP6-LISTEN:1337,bind=[::1],fork
+hello world 2
+hello world 5
+hello world 3
+hello world 6
+hello world 1
+hello world 8
+hello world 9
+hello world 10
+hello world 7
+hello world 4
+```
+
+We can see the packets arrive out of order. The Prometheus metrics show it, at `http://127.0.0.1:8000`:
+
+```
+ape_total_udp_packets{action="scramble",device="lo",port="1337"} 18.0
+# HELP ape_manipulated_udp_packets UDP packets manipulated by ape
+# TYPE ape_manipulated_udp_packets gauge
+ape_manipulated_udp_packets{action="scramble",device="lo",port="1337"} 8.0
+```
+
+There are more than 10 total packets, because `scramble` will feed back to itself (the XDP kernel program will probably re-scramble packets sent by the user scramble program).

ape.py

@@ -71,7 +71,7 @@ def stats_thread(
             .encode("utf-8")
         ),
     )[0]
-    total_metric.labels(device, "drop", port).set(total_packets)
+    total_metric.labels(device, action, port).set(total_packets)
     dropped_packets = unpack(
         "<2i",
         unhexlify(

xdp_user_scramble.c

@@ -38,6 +38,10 @@
 #define RX_BATCH_SIZE 64
 #define INVALID_UMEM_FRAME UINT64_MAX
 
+#ifndef MAX_SLEEP_MS
+#define MAX_SLEEP_MS 1000
+#endif
+
 struct xsk_umem_info {
 	struct xsk_ring_prod fq;
 	struct xsk_ring_cons cq;
@@ -120,7 +124,7 @@ static void dawdle()
 {
 	/* https://gist.github.com/justinloundagin/5536640 */
 	struct timespec tv;
-	int msec = (int)(((double)random() / INT_MAX) * 1000);
+	int msec = (int)(((double)random() / INT_MAX) * MAX_SLEEP_MS);
 	tv.tv_sec = 0;
 	tv.tv_nsec = 1000000 * msec;
 	if (nanosleep(&tv, NULL) == -1) {
@@ -294,8 +298,6 @@ static bool process_packet(struct xsk_socket_info *xsk, uint64_t addr,
 		return false;
 	}
 
-	printf("dest port is: %d\n", bpf_ntohs(udphdr->dest));
-
 	// sleep randomly to scramble
 	dawdle();
 
@@ -305,19 +307,17 @@ static bool process_packet(struct xsk_socket_info *xsk, uint64_t addr,
 		sin.sin6_family = AF_INET6;
 		sin.sin6_port = udphdr->dest;
 		inet_pton(AF_INET6, "::1", &sin.sin6_addr);
-		sent_bytes = sendto(udp6_out, (void *)pkt, len, 0,
+		sent_bytes = sendto(udp6_out, (void *)nh.pos, len, 0,
 				    (struct sockaddr *)&sin, sizeof(sin));
 	} else {
 		struct sockaddr_in sin;
 		sin.sin_family = AF_INET;
 		sin.sin_port = udphdr->dest;
 		sin.sin_addr.s_addr = inet_addr("127.0.0.1");
-		sent_bytes = sendto(udp4_out, (void *)pkt, len, 0,
+		sent_bytes = sendto(udp4_out, (void *)nh.pos, len, 0,
 				    (struct sockaddr *)&sin, sizeof(sin));
 	}
 
-	printf("sent %lu bytes\n", sent_bytes);
-
 	return true;
 }
 
@@ -354,7 +354,6 @@ static void handle_receive_packets(struct xsk_socket_info *xsk, int udp4_out,
 
 	/* Process received packets */
 	for (i = 0; i < rcvd; i++) {
-		printf("processing this packet!\n");
 		uint64_t addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
 		uint32_t len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;