icept: add skmsg interception

Avoid the need for iptables rules. Splice traffic from client/server directly into icept egress socket. Attach a bpf program to the client/server cgroups to intercept all connection establishment. Insert the new sockets into a sockmap. Attach a bpf program to this sockmap to intercept all send calls. Call bpf_skb_redirect_msg to redirect to the icept egress socket. Signed-off-by: Willem de Bruijn <willemb@google.com>
wdebruij · Nov 16, 2019 · 603382b · 603382b
1 parent e1504fa
commit 603382b
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 5 deletions.
diff --git a/tools/icept/icept.c b/tools/icept/icept.c
@@ -38,6 +38,7 @@ static socklen_t cfg_addr_len;
 static struct sockaddr *cfg_addr_dst;
 static struct sockaddr *cfg_addr_listen;
 static struct sockaddr *cfg_addr_src;
+static const char *cfg_cgroup_path;
 static int cfg_mark;
 static const char *cfg_role;
 static void (*cfg_role_fn)(void);
@@ -254,14 +255,16 @@ static void do_bpf_attach_prog(struct bpf_program *prog, int fd,
 
 static void do_bpf_setup(void)
 {
-	struct bpf_program *prog_parse, *prog_verdict;
+	struct bpf_program *prog_parse, *prog_verdict, *prog_cgroup, *prog_skmsg;
 	struct bpf_map *map;
 
 	obj = bpf_object__open("icept_bpf.o");
 	bpf_check_ptr(obj, "obj");
 
 	prog_parse = do_bpf_get_prog("sk_skb_parser", BPF_PROG_TYPE_SK_SKB);
 	prog_verdict = do_bpf_get_prog("sk_skb_verdict", BPF_PROG_TYPE_SK_SKB);
+	prog_cgroup = do_bpf_get_prog("sockops", BPF_PROG_TYPE_SOCK_OPS);
+	prog_skmsg = do_bpf_get_prog("sk_msg", BPF_PROG_TYPE_SK_MSG);
 
 	if (bpf_object__load(obj))
 		error(1, 0, "bpf object load: %ld", libbpf_get_error(obj));
@@ -272,6 +275,25 @@ static void do_bpf_setup(void)
 
 	do_bpf_attach_prog(prog_parse, map_fd, BPF_SK_SKB_STREAM_PARSER);
 	do_bpf_attach_prog(prog_verdict, map_fd, BPF_SK_SKB_STREAM_VERDICT);
+
+	if (cfg_cgroup_path) {
+		int map_tx_fd, cgroup_fd;
+
+		cgroup_fd = open(cfg_cgroup_path, O_DIRECTORY, O_RDONLY);
+		if (cgroup_fd == -1)
+			error(1, errno, "open cgroup");
+
+		do_bpf_attach_prog(prog_cgroup, cgroup_fd, BPF_CGROUP_SOCK_OPS);
+
+		if (close(cgroup_fd))
+			error(1, errno, "close cgroup");
+
+		map = bpf_object__find_map_by_name(obj, "sock_map_tx");
+		bpf_check_ptr(map, "map_tx");
+		map_tx_fd = bpf_map__fd(map);
+
+		do_bpf_attach_prog(prog_skmsg, map_tx_fd, BPF_SK_MSG_VERDICT);
+	}
 }
 
 static void do_bpf_cleanup(void)
@@ -361,7 +383,7 @@ static void do_intercept(void)
 
 static void __attribute__((noreturn)) usage(const char *filepath)
 {
-	error(1, 0, "Usage: %s [-46sS] [-d addr] [-D port] [-L port] [-m mark] <client|server|intercept>",
+	error(1, 0, "Usage: %s [-46sS] [-C cgroup_path ] [-d addr] [-D port] [-L port] [-m mark] <client|server|intercept>",
 		    filepath);
 
 	/* suppress compiler warning */
@@ -374,7 +396,7 @@ static void parse_opts(int argc, char **argv)
 	const char *addr_dst = NULL;
 	int c;
 
-	while ((c = getopt(argc, argv, "46d:D:L:m:sS")) != -1) {
+	while ((c = getopt(argc, argv, "46C:d:D:L:m:sS")) != -1) {
 		switch (c) {
 		case '4':
 			if (cfg_addr_dst)
@@ -392,6 +414,9 @@ static void parse_opts(int argc, char **argv)
 			cfg_addr_src = (struct sockaddr *)&cfg_addr6_src;
 			cfg_addr_listen = (struct sockaddr *)&cfg_addr6_listen;
 			break;
+		case 'C':
+			cfg_cgroup_path = optarg;
+			break;
 		case 'd':
 			addr_dst = optarg;
 			break;

diff --git a/tools/icept/icept.sh b/tools/icept/icept.sh
@@ -46,6 +46,7 @@ do_intercept() {
 	local -r ipt_bin=$2
 	local -r family=$3
 	local -r mode=$4
+	local -r cgroup=$5
 
 	local role="intercept"
 
@@ -54,6 +55,8 @@ do_intercept() {
 		role="-s ${role}"
 	elif [[ "${mode}" == "sockmap" ]]; then
 		role="-S ${role}"
+	elif [[ "${mode}" == "skmsg" ]]; then
+		role="-S -C ${cgroup} ${role}"
 	fi
 
 	ip netns exec "${ns}" ./icept "-${family}" \
@@ -89,8 +92,9 @@ do_main() {
 
 	# Start intercept service (optionally)
 	if [[ "${cfg_do_icept}" != "none" ]]; then
-		do_intercept "${ns1}" "${ipt_bin}" "${cfg_family}" "${cfg_do_icept}"
-		do_intercept "${ns2}" "${ipt_bin}" "${cfg_family}" "${cfg_do_icept}"
+		do_intercept "${ns1}" "${ipt_bin}" "${cfg_family}" "${cfg_do_icept}" "${cfg_cgroup_ns1}"
+		sleep 0.2
+		do_intercept "${ns2}" "${ipt_bin}" "${cfg_family}" "${cfg_do_icept}" "${cfg_cgroup_ns2}"
 	fi
 
 	# Wait for servers to be up
@@ -174,4 +178,8 @@ echo "Test Intercept (sockmap)"
 run_in_namespace bare ip6tables 6 "fd::2" sockmap
 run_in_namespace bare iptables 4 "192.168.1.2" sockmap
 
+echo "Test Intercept (skmsg)"
+run_in_namespace bare ip6tables 6 "fd::2" skmsg
+run_in_namespace bare iptables 4 "192.168.1.2" skmsg
+
 echo "OK. All passed"
diff --git a/tools/icept/icept_bpf.c b/tools/icept/icept_bpf.c
@@ -16,6 +16,13 @@ struct bpf_map_def SEC("maps") sock_map = {
 	.max_entries = 2,
 };
 
+struct bpf_map_def SEC("maps") sock_map_tx = {
+	.type = BPF_MAP_TYPE_SOCKMAP,
+	.key_size = sizeof(int),
+	.value_size = sizeof(int),
+	.max_entries = 1,
+};
+
 SEC("sk_skb_parser")
 int _prog_parser(struct __sk_buff *skb)
 {
@@ -49,3 +56,36 @@ int _prog_verdict(struct __sk_buff *skb)
 	return bpf_sk_redirect_map(skb, &sock_map, key, 0);
 }
 
+SEC("sockops")
+int _prog_cgroup_sockops(struct bpf_sock_ops *ops)
+{
+	if (ops->op == BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB ||
+	    ops->op == BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB) {
+		uint32_t key = 0;
+
+		bpf_sock_map_update(ops, &sock_map_tx, &key, BPF_NOEXIST);
+	}
+
+	return 1;
+}
+
+SEC("sk_msg")
+int _prog_skmsg(struct sk_msg_md *msg)
+{
+	void *data, *data_end;
+	uint32_t key;
+	char *d;
+
+	data = (void *)(long) msg->data;
+	data_end = (void *)(long) msg->data_end;
+	d = data;
+
+	if (data_end < data + 1)
+		return SK_DROP;
+
+	key = d[0] == 'a' ? 1 : 0;
+	d[0]++;
+
+	return bpf_msg_redirect_map(msg, &sock_map, key, 0);
+}
+