ebpf unit testing -- handle tailcalls and support user-space map emul…

…ation

This commit contains a demo on how to handle tailcalls and also supports
user-space eBPF map emulation which is a part of our fake kernel model library.

We resolve tailcalls by mocking the tailcall helper function and stubbing a
callback since tailcalls are achieved by the tailcall helper function. In this
way, the callback will be called instead whenever the tailcall helper function
is encountered.

We also provide a library to support user-space map emulation. Since eBPF maps
are maintained by the kernel, we do not actually know if the map operations are
safe or not without testing in the actual kernel. Considering that eBPF maps
are actually hashmaps, we create a user-space hashmap and emulate the eBPF map
operations in the user space via callbacks.

new file:
bpf/mock/fake_maps.h: wrap up raw hashmap operations.
bpf/tests/drop_notify_test.h: include a demo test showing how to handle
tailcalls
test/bpf/drop_notify_test.c: contains main function to run test functions in drop_notify_test.h

modified:
bpf/mock/Dockerfile: add hashmap library
bpf/tests/nat_test.h: add a demo test showing how to use user-space map emulation
test/bpf/nat-test.c: add the new test in bpf/tests/nat_test.h into main
function
bpf/mock/Makefile: put all the generated files into folder "mocks" for
later cleaning
bpf/mock/mock_helpers.sh: put all the generated files into folder "mocks" for
later cleaning
test/bpf/Makefile: clean bpf/mock/mocks

Signed-off-by: Xinyuan Zhang <zhangxinyuan@google.com>

bpf/mock/Dockerfile

@@ -6,3 +6,5 @@ RUN apt-get -y install ruby-full
 RUN apt-get -y install git
 RUN git clone --recursive https://github.com/ThrowTheSwitch/CMock.git
 RUN git -C CMock reset --hard 3d4ba8d20b8958da5dace7dd5d31155c94b60819
+RUN git clone https://github.com/DavidLeeds/hashmap.git
+RUN git -C hashmap reset --hard 137d60b3818c22c79d2be5560150eb2eff981a68

bpf/mock/Makefile

@@ -22,26 +22,23 @@ check_helper_headers: generate_helper_headers mock_helpers
 	# Generate temp headers from lib/helpers.h, lib/helpers_skb.h and lib/helpers_xdp.h.
 	# Compare them to the headers within the current directory.
 	# The temp headers are removed after finishing.
-	rm helpers.h helpers_skb.h helpers_xdp.h mock_helpers.h mock_helpers.c mock_helpers_skb.h mock_helpers_skb.c mock_helpers_xdp.h mock_helpers_xdp.c;
+	rm -r mocks;
 
 generate_helper_headers:
 	# Auto-generate helpers.h, helpers_skb.h, and helpers_xdp.h.
 	# Take contents from bpf/helpers.h, bpf/helpers_skb.h and bpf/helpers_xdp.h.
 	# Prune to be ready to mock and save to the current directory.
-	sed -f helpers.sed ../include/bpf/helpers.h > helpers.h; \
-	sed -f helpers_skb.sed ../include/bpf/helpers_skb.h > helpers_skb.h; \
-	sed -f helpers_xdp.sed ../include/bpf/helpers_xdp.h > helpers_xdp.h;
+	mkdir -p mocks; \
+	sed -f helpers.sed ../include/bpf/helpers.h > mocks/helpers.h; \
+	sed -f helpers_skb.sed ../include/bpf/helpers_skb.h > mocks/helpers_skb.h; \
+	sed -f helpers_xdp.sed ../include/bpf/helpers_xdp.h > mocks/helpers_xdp.h;
 
 mock_helpers: builder-image
 	# Generate mock libraries for helpers.h, helpers_skb.h and helpers_xdp.h with CMock.
-	$(QUIET) $(DOCKER_RUN) ./mock_helpers.sh; \
-	mv mocks/* ./; \
-	rmdir mocks;
+	$(QUIET) $(DOCKER_RUN) ./mock_helpers.sh;
 
 mock_customized: builder-image
 	# Take the name of the header to be mocked.
 	# Generate the corresponding mock library.
-	$(QUIET) $(DOCKER_RUN) ./mock_customized.sh $(filename); \
-	mv mocks/* ./; \
-	rmdir mocks;
+	$(QUIET) $(DOCKER_RUN) ./mock_customized.sh $(filename);
 

bpf/mock/fake_maps.h

@@ -0,0 +1,56 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2021 Authors of Cilium */
+
+// library for user-space map emulation
+// It emulates the eBPF map operations in the user space by wrapping up the raw
+// hashmap functions.
+
+#include "hashmap.h"
+
+typedef HASHMAP(void, void) hashmap_void_t;
+
+
+// Get the number of entries inside map.
+size_t fake_get_size(hashmap_void_t *map) {
+  return hashmap_size(map);
+}
+
+// Initiate map with given length of keys and capacity.
+void fake_init_map(hashmap_void_t *map, size_t (*hash)(const void *key), int (*compare)(const void *a, const void *b)) {
+  hashmap_init(map, hash, compare);
+}
+
+// Loop up the value of given key.
+void *fake_lookup_elem(hashmap_void_t *map, void *key) {
+  return hashmap_get(map, key);
+}
+
+// Update the value of given key.
+int fake_update_elem(hashmap_void_t *map, void *key, void *value, __u32 flags, size_t size) {
+  void *entry = hashmap_get(map, key);
+  if (entry == NULL && hashmap_size(map) >= size) {
+      printf("Update failed: Map is full\n");
+      return -1;
+    }
+
+  int r = hashmap_put(map, key, value);
+  if (r == -EEXIST) {
+    if (flags == BPF_NOEXIST) return -1;
+    hashmap_remove(map, key);
+    hashmap_put(map, key, value);
+    return 0;
+  } else if (r == 0) {
+    if (flags == BPF_EXIST) {
+      hashmap_remove(map, key);
+      return -1;
+    }
+    return 0;
+  }
+
+  return r;
+}
+
+// Delete given key.
+int fake_delete_elem(hashmap_void_t *map, void *key) {
+  return hashmap_remove(map, key);
+}

bpf/mock/mock_helpers.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 
-ruby ../../CMock/lib/cmock.rb -obpf.yaml helpers.h
-ruby ../../CMock/lib/cmock.rb -obpf.yaml helpers_skb.h
-ruby ../../CMock/lib/cmock.rb -obpf.yaml helpers_xdp.h
+ruby ../../CMock/lib/cmock.rb -obpf.yaml mocks/helpers.h
+ruby ../../CMock/lib/cmock.rb -obpf.yaml mocks/helpers_skb.h
+ruby ../../CMock/lib/cmock.rb -obpf.yaml mocks/helpers_xdp.h

bpf/tests/drop_notify_test.h

@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2021 Authors of Cilium */
+
+// sample unit test program for some functions in drop.h
+// It contains function definitions for testing "send_drop_notify" and "__send_drop_notify".
+// It is used to perform unit test on the above two functions to demonstrate how
+// to handle tailcalls.
+// There is a tailcall at the end of function send_drop_notif which actually
+// calls function __send_drop_notify. We can stub the tailcall and actually call
+// the function with callback.
+// If other functions in drop.h need to be tested, please add the function definitions at the bottom.
+
+#define __BPF_HELPERS_SKB__
+#define __BPF_HELPERS__
+#define DROP_NOTIFY
+
+#include <stdio.h>
+#include <assert.h>
+
+// Include unity test framework and all the mock libraries.
+#include "unity.h"
+#include "mocks/mock_helpers.h"
+#include "mocks/mock_helpers_skb.h"
+
+#include "bpf/ctx/skb.h"
+#include "node_config.h"
+
+// Include lib/metrics.h which contains the definition of ep_tail_call first to
+// avoid it to be included again in lib/drop.h.
+#include "lib/metrics.h"
+
+// Define macros like the followings to make sure the original tailcall is redirected
+// to the mock tailcall function, the last 0 does not matter because we do not
+// actually use the arguments.
+#define ep_tail_call(a, b) tail_call(a, b, 0)
+
+// The file containing the functions to be tested must be included after
+// defining the above macros.
+#include "lib/drop.h"
+
+// Undefine ep_tail_call to stop redirecting to the mock. It is not necessary
+// unless you would like to include something else that might conflict with the
+// redirection.
+#undef ep_tail_call
+
+
+// This is the function we use as the callback when stubbing the tailcall.
+void __send_drop_notify_tailcall(void* ctx, const void* map, __u32 index, int cmock_num_calls) {
+
+  // We can even unit-test the function which is actually called by the tailcall
+  // within the callback.
+  skb_event_output_IgnoreAndReturn(0);
+  assert(!__send_drop_notify(ctx));
+}
+
+// A sample test for function send_drop_notify
+// It is a demo to show how we handle tailcalls.
+void test_send_drop_notify() {
+  struct __ctx_buff ctx;
+
+  // Set the expectations for the helpers functions called before the tailcall.
+  map_lookup_elem_IgnoreAndReturn(NULL);
+  map_update_elem_IgnoreAndReturn(0);
+
+  // We stub the tailcall here by calling callback.
+  tail_call_Stub(__send_drop_notify_tailcall);
+  assert(!send_drop_notify(&ctx, 0, 0, 0, 0, 0, 0));
+}
+

bpf/tests/nat_test.h

@@ -18,12 +18,14 @@
 
 #include <stdio.h>
 #include <assert.h>
+#include <errno.h>
 
 // Include unity test framework and all the mock libraries.
 #include "unity.h"
-#include "mock/mock_helpers.h"
-#include "mock/mock_helpers_skb.h"
-#include "mock/mock_conntrack_stub.h"
+#include "mocks/mock_helpers.h"
+#include "mocks/mock_helpers_skb.h"
+#include "mocks/mock_conntrack_stub.h"
+#include "fake_maps.h"
 
 // To avoid conflict between functions in bpf/builtins.h and string.h, define
 // __BPF_BUILTINS__ in conntrack_stub.h to make sure the mock library can be
@@ -99,3 +101,58 @@ void test_snat_v4_track_local() {
     assert(!snat_v4_track_local(&ctx, &tuple, &state, NAT_DIR_EGRESS, 0,
                                 &target));
 }
+
+// Create a fake map
+HASHMAP(struct ipv4_ct_tuple, struct ipv4_nat_entry) ipv4_ct_tuple_map;
+
+// Define the compare_func
+// We need to wrap the cmp func upon using since memcmp requires the length.
+int bpf_compare_ipv4_ct_tuple(const void *a, const void *b)
+{
+    return __bpf_memcmp_builtin(a, b, sizeof(struct ipv4_ct_tuple));
+}
+
+// Define the hash_func
+// We need to wrap the cmp func upon using since hashmap_hash_default also
+// requires the length.
+size_t bpf_hash_ipv4_ct_tuple(const void *key)
+{
+    return hashmap_hash_default(key, sizeof(struct ipv4_ct_tuple));
+}
+
+// Wrap the fake map operations into callbacks
+// The "map" argument (the real map) is actually not used, but it has to be
+// encoded in the callback functions as required by the stub functions in CMock.
+// So we need different callback funcs for different fake maps.
+void *ipv4_ct_tuple_map_lookup_elem_callback(const void* map, const void* key, int cmock_num_calls) {
+  return fake_lookup_elem(&ipv4_ct_tuple_map, key);
+}
+
+int ipv4_ct_tuple_map_update_elem_callback(const void* map, const void* key, const void* value, __u32 flags, int cmock_num_calls) {
+  return fake_update_elem(&ipv4_ct_tuple_map, key, value, flags, SNAT_MAPPING_IPV4_SIZE);
+}
+
+int ipv4_ct_tuple_map_delete_elem_callback(const void* map, const void* key, int cmock_num_calls) {
+  return fake_delete_elem(&ipv4_ct_tuple_map, key);
+}
+
+void test_snat_v4_new_mapping() {
+    struct __ctx_buff ctx;
+    struct ipv4_ct_tuple otuple;
+    struct ipv4_nat_entry ostate;
+    struct ipv4_nat_target target;
+
+    // Initiate the map.
+    fake_init_map(&ipv4_ct_tuple_map, bpf_hash_ipv4_ct_tuple, bpf_compare_ipv4_ct_tuple);
+
+    get_prandom_u32_ExpectAndReturn(0);
+    // Stub the map helpers with the callbacks defined above.
+    map_lookup_elem_Stub(ipv4_ct_tuple_map_lookup_elem_callback);
+    map_update_elem_Stub(ipv4_ct_tuple_map_update_elem_callback);
+    map_update_elem_Stub(ipv4_ct_tuple_map_update_elem_callback);
+    ktime_get_ns_ExpectAndReturn(0);
+    skb_event_output_IgnoreAndReturn(0);
+    // snat_v4_new_mapping will return 0 because rtuple will not be found by
+    // snat_v4_lookup and then snat_v4_update will update successfully.
+    assert(!snat_v4_new_mapping(&ctx, &otuple, &ostate, &target));
+}

test/bpf/Makefile

@@ -27,7 +27,7 @@ CLANG ?= $(QUIET) clang
 LLC ?= llc
 
 BPF_TARGETS := elf-demo.o
-ALL_TESTS := unit-test nat-test
+ALL_TESTS := unit-test nat-test drop-notify-test
 TARGETS := $(BPF_TARGETS) $(ALL_TESTS)
 
 all: $(TARGETS) unit-tests
@@ -45,8 +45,13 @@ nat-test: nat-test.c $(LIB)
 	make -C $(ROOT_DIR)/bpf/mock generate_helper_headers
 	make -C $(ROOT_DIR)/bpf/mock mock_helpers
 	make -C $(ROOT_DIR)/bpf/mock mock_customized filename=conntrack_stub.h
-	$(QUIET) $(DOCKER_RUN) $(CLANG) -I../../bpf/ -I../../bpf/include -I. -D__NR_CPUS__=$(shell nproc --all) -O2 -I $../../bpf/ -I../../../CMock/src -I../../../CMock/vendor/unity/src  $< ../../../CMock/vendor/unity/src/unity.c ../../../CMock/src/cmock.c ../../bpf/mock/mock_helpers.c ../../bpf/mock/mock_helpers_skb.c ../../bpf/mock/mock_conntrack_stub.c -o $@
-	rm ../../bpf/mock/mock_conntrack_stub.c ../../bpf/mock/mock_conntrack_stub.h ../../bpf/mock/mock_helpers.c ../../bpf/mock/mock_helpers.h ../../bpf/mock/mock_helpers_skb.c ../../bpf/mock/mock_helpers_skb.h ../../bpf/mock/mock_helpers_xdp.c ../../bpf/mock/mock_helpers_xdp.h ../../bpf/mock/helpers.h ../../bpf/mock/helpers_skb.h ../../bpf/mock/helpers_xdp.h;
+	$(QUIET) $(DOCKER_RUN) $(CLANG) $(FLAGS) -I../../bpf/mock -I $../../bpf/ -I../../../CMock/src -I../../../CMock/vendor/unity/src -I../../../hashmap/include  $< ../../../CMock/vendor/unity/src/unity.c ../../../CMock/src/cmock.c ../../../hashmap/src/hashmap.c ../../bpf/mock/mocks/mock_helpers.c ../../bpf/mock/mocks/mock_helpers_skb.c ../../bpf/mock/mocks/mock_conntrack_stub.c -o $@
+
+drop-notify-test: drop-notify-test.c $(LIB)
+	@$(ECHO_CC)
+	make -C $(ROOT_DIR)/bpf/mock generate_helper_headers
+	make -C $(ROOT_DIR)/bpf/mock mock_helpers
+	$(QUIET) $(DOCKER_RUN) $(CLANG) $(FLAGS) -I../../bpf/mock -I $../../bpf/ -I../../../CMock/src -I../../../CMock/vendor/unity/src  $< ../../../CMock/vendor/unity/src/unity.c ../../../CMock/src/cmock.c ../../bpf/mock/mocks/mock_helpers.c ../../bpf/mock/mocks/mock_helpers_skb.c -o $@
 
 unit-tests: $(ALL_TESTS)
 	@$(ECHO_CHECK)
@@ -58,4 +63,5 @@ unit-tests: $(ALL_TESTS)
 clean:
 	@$(ECHO_CLEAN)
 	-$(QUIET)rm -f $(TARGETS)
+	-$(QUIET)rm -r ../../bpf/mock/mocks
 

test/bpf/drop-notify-test.c

@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2021 Authors of Cilium */
+
+// source file for drop_notify_test.h.
+// It contains contains main functions to run test functions drop_notify_test.h.
+// It is used to perform unit test on functions in drop.h.
+
+#include "tests/drop_notify_test.h"
+
+void setUp(void) {
+    // set stuff up here
+}
+
+void tearDown(void) {
+    // clean stuff up here
+}
+
+
+int main(int argc, char *argv[])
+{
+  test_send_drop_notify();
+  return 0;
+}

test/bpf/nat-test.c

@@ -19,5 +19,6 @@ void tearDown(void) {
 int main(int argc, char *argv[])
 {
     test_snat_v4_track_local();
+    test_snat_v4_new_mapping();
     return 0;
 }