bpf: map pinning support

Signed-off-by: Maryam Tahhan <mtahhan@redhat.com>

Makefile

@@ -34,18 +34,22 @@ format:
 	@echo
 	@echo
 
+buildxdp:
+	@echo "******     Build xdp_pass     ******"
+	make -C ./internal/bpf/xdp-pass/
+	@echo "******     Build xdp_afxdp_redirect     ******"
+	make -C ./internal/bpf/xdp-afxdp-redirect/
+	@echo
+
 buildc:
 	@echo "******     Build BPF     ******"
 	@echo
 	gcc ./internal/bpf/bpfWrapper.c -lbpf -c -o ./internal/bpf/bpfWrapper.o
 	ar rs ./internal/bpf/libwrapper.a ./internal/bpf/bpfWrapper.o  &> /dev/null
-	@echo "******     Build xdp_pass     ******"
-	make -C ./internal/bpf/xdp-pass/
-	@echo
 	@echo
 	@echo
 
-builddp: buildc
+builddp: buildc buildxdp
 	@echo "******     Build DP      ******"
 	@echo
 	go build -o ./bin/afxdp-dp ./cmd/deviceplugin

README.md

@@ -376,6 +376,9 @@ EthtoolCmds is an array of strings. This is a setting that can be applied to dev
 
 UdsServerDisable is a Boolean configuration. If set to true, devices in this pool will not have the BPF app loaded onto the netdev. This means no UDS server is spun up when a device is allocated to a pod. By default, this is set to false.
 
+#### BpfMapPinningEnable
+BpfMapPinningEnable is a Boolean configuration. If set to true, will use BPF map pinning instead of a UDS to share an XSK map with a pod. By default, this is set to false. Should set UdsServerDisable to true when using this configuration.
+
 #### UdsTimeout
 
 UdsTimeout is an integer configuration. This value sets the amount of time, in seconds, that the UDS server will wait while there is no activity on the UDS. When this timeout limit is reached, the UDS server terminates and the UDS is deleted from the filesystem. This can be a useful setting, for example, in scenarios where large batches of pods are created together. Large batches of pods tend to take some time to spin up, so it might be beneficial to have the UDS server sit waiting a little longer for the pod to start. The maximum allowed value is 300 seconds (5 min). The minimum and default value is 30 seconds.

constants/constants.go

@@ -82,6 +82,10 @@ var (
 	udsSockDir    = "/tmp/afxdp_dp/"  // host location where we place our uds sockets. If changing location remember to update daemonset mount point
 	udsPodPath    = "/tmp/afxdp.sock" // the uds filepath as it will appear in the end user application pod
 
+	/* BPF*/
+	bpfMapPodPath = "/tmp/xsks_map"
+	xsk_map       = "/xsks_map"
+
 	udsDirFileMode = 0700 // permissions for the directory in which we create our uds sockets
 
 	/* Handshake*/
@@ -134,6 +138,8 @@ var (
 	DeviceFile deviceFile
 	/* DeviceFile contains constants related to the devicefile */
 	EthtoolFilter ethtoolFilter
+	/* Bpf contains constants related to the BPF Map pinning */
+	Bpf bpf
 )
 
 type cni struct {
@@ -218,6 +224,11 @@ type uds struct {
 	Handshake   handshake
 }
 
+type bpf struct {
+	BpfMapPodPath string
+	Xsk_map       string
+}
+
 type handshake struct {
 	Version             string
 	RequestVersion      string
@@ -339,6 +350,11 @@ func init() {
 		},
 	}
 
+	Bpf = bpf{
+		BpfMapPodPath: bpfMapPodPath,
+		Xsk_map:       xsk_map,
+	}
+
 	DeviceFile = deviceFile{
 		Name:            name,
 		FilePermissions: filePermissions,

deployments/daemonset-kind.yaml

@@ -13,7 +13,8 @@ data:
           {
             "name":"myPool",
             "mode":"primary",
-            "UdsServerDisable": false,
+            "UdsServerDisable": true,
+            "BpfMapPinningEnable": true,
             "drivers":[
               {
                 "name":"veth"
@@ -72,6 +73,8 @@ spec:
           volumeMounts:
             - name: unixsock
               mountPath: /tmp/afxdp_dp/
+            - name: bpfmappinning
+              mountPath: /var/run/afxdp_dp/
               mountPropagation: Bidirectional
             - name: devicesock
               mountPath: /var/lib/kubelet/device-plugins/
@@ -87,6 +90,9 @@ spec:
         - name: unixsock
           hostPath:
             path: /tmp/afxdp_dp/
+        - name: bpfmappinning
+          hostPath:
+            path: /var/run/afxdp_dp/
         - name: devicesock
           hostPath:
             path: /var/lib/kubelet/device-plugins/

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/google/gofuzz v1.1.0
 	github.com/google/uuid v1.3.0
 	github.com/intel/afxdp-plugins-for-kubernetes/pkg/subfunctions v0.0.0
+	github.com/moby/sys/mount v0.3.3 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1
 	github.com/sirupsen/logrus v1.9.0

go.sum

@@ -519,8 +519,12 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs=
+github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0=
 github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
+github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=

images/amd64.dockerfile

@@ -16,7 +16,6 @@ FROM golang:1.20@sha256:52921e63cc544c79c111db1d8461d8ab9070992d9c636e1573176642
 COPY . /usr/src/afxdp_k8s_plugins
 WORKDIR /usr/src/afxdp_k8s_plugins
 RUN apt-get update && apt-get -y install --no-install-recommends libbpf-dev=1:0.3-2 \
-       && apt-get -y install clang llvm gcc-multilib \
        && make buildcni
 
 FROM golang:1.20-alpine@sha256:87d0a3309b34e2ca732efd69fb899d3c420d3382370fd6e7e6d2cb5c930f27f9 as dpbuilder
@@ -34,4 +33,5 @@ COPY --from=cnibuilder /usr/src/afxdp_k8s_plugins/bin/afxdp /afxdp/afxdp
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/bin/afxdp-dp /afxdp/afxdp-dp
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/images/entrypoint.sh /afxdp/entrypoint.sh
 COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/internal/bpf/xdp-pass/xdp_pass.o /afxdp/xdp_pass.o
+COPY --from=dpbuilder /usr/src/afxdp_k8s_plugins/internal/bpf/xdp-afxdp-redirect/xdp_afxdp_redirect.o /afxdp/xdp_afxdp_redirect.o
 ENTRYPOINT ["/afxdp/entrypoint.sh"]

internal/bpf/bpfWrapper.c

@@ -5,7 +5,7 @@
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ *	 http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,6 +17,10 @@
 #include <bpf/xsk.h> // for xsk_setup_xdp_prog, bpf_set_link_xdp_fd
 #include <net/if.h>  // for if_nametoindex
 #include <linux/if_link.h> // for XDP_FLAGS_DRV_MODE
+#include <bpf/libbpf.h>
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
 
 #include "bpfWrapper.h"
 #include "log.h"
@@ -107,7 +111,7 @@ int Configure_busy_poll(int fd, int busy_timeout, int busy_budget) {
 	}
 
 	Log_Warning("%s: setsockopt failure, attempting to restore xsk to default state",
-		    __FUNCTION__);
+			__FUNCTION__);
 
 	Log_Warning("%s: unsetting SO_BUSY_POLL on file descriptor %d", __FUNCTION__, fd);
 
@@ -157,7 +161,7 @@ int Clean_bpf(char *ifname) {
 			// host libbpf versions. doesn't break functionality and this problem is
 			// being investigated.
 			Log_Warning("%s: Removal of xdp program is reporting error code: (%d)",
-				    __FUNCTION__, err);
+					__FUNCTION__, err);
 		} else {
 			Log_Error("%s: Removal of xdp program failed, returned: (%d)", __FUNCTION__,
 				  err);
@@ -182,9 +186,8 @@ int Load_attach_bpf_xdp_pass(char *ifname)
 	if (!ifindex) {
 		Log_Error("%s: if_index not valid: %s", __FUNCTION__, ifname);
 		return -1;
-	} else {
-		Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex);
 	}
+	Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex);
 
 	Log_Info("%s: starting setup of xdp-pass program on "
 		 "interface %s (%d)",
@@ -211,3 +214,69 @@ int Load_attach_bpf_xdp_pass(char *ifname)
 
 	return 0;
 }
+
+// TODO update logging to debug
+// TODO UPDATE TO RETURN MAP NAME
+// SEE struct bpf_map in https://elixir.bootlin.com/linux/latest/source/tools/lib/bpf/libbpf.c#L487
+int Load_bpf_pin_xsk_map(char *ifname, char *pin_path) {
+	struct bpf_object *obj;
+	struct bpf_program *prog;
+	struct bpf_link *link;
+	int ifindex, map_fd= -1;
+	int err;
+	const char *prog_name = "xdp_afxdp_redirect";
+	char *filename = "/afxdp/xdp_afxdp_redirect.o";
+    DECLARE_LIBBPF_OPTS(bpf_object_open_opts, bpf_opts, .pin_root_path = pin_path);
+
+	ifindex = if_nametoindex(ifname);
+	if (!ifindex) {
+		Log_Error("%s: if_index not valid: %s", __FUNCTION__, ifname);
+		return -1;
+	}
+	Log_Info("%s: if_index for interface %s is %d", __FUNCTION__, ifname, ifindex);
+
+	if (access(filename, O_RDONLY) < 0) {
+		Log_Error("%s:error accessing file %s: %s\n", __FUNCTION__, filename, strerror(errno));
+		return err;
+	}
+
+	obj = bpf_object__open_file(filename, &bpf_opts);
+	err = libbpf_get_error(obj);
+	if (err) {
+		Log_Error("%s: Couldn't open file(%s)\n",
+			__FUNCTION__, filename);
+		return err;
+	}
+
+	prog = bpf_object__find_program_by_name(obj, prog_name);
+	if (!prog) {
+		Log_Error("%s: Couldn't find xdp program in bpf object!\n",	__FUNCTION__);
+		err = -ENOENT;
+		return err;
+	}
+	bpf_program__set_type(prog, BPF_PROG_TYPE_XDP);
+
+	err = bpf_object__load(obj);
+	if (err) {
+	 	Log_Error("%s: Couldn't load BPF-OBJ file(%s) %s\n",
+	 		__FUNCTION__, filename, strerror(errno));
+	 	return err;
+	}
+
+	Log_Info("%s: bpf: Attach prog to ifindex %d\n", __FUNCTION__, ifindex);
+	link = bpf_program__attach_xdp(prog, ifindex);
+	if (!link) {
+		Log_Error("%s:ERROR: failed to attach program to %s\n", __FUNCTION__, ifname);
+		return err;
+	}
+
+	if(bpf_map__is_pinned(bpf_object__find_map_by_name(obj, "xsks_map"))){
+		Log_Info("%s: xsk map pinned to %s\n",  __FUNCTION__,
+		         bpf_map__get_pin_path(bpf_object__find_map_by_name(obj, "xsks_map")));
+	} else {
+		Log_Error("%s: xsk map is not pinned \n", __FUNCTION__);
+		return -1;
+	}
+
+	return 0;
+}

internal/bpf/bpfWrapper.go

@@ -18,6 +18,7 @@ package bpf
 
 //#include <bpf/libbpf.h>
 //#include <bpf/xsk.h>
+//#include <bpf/bpf.h>
 //#cgo CFLAGS: -I.
 //#cgo LDFLAGS: -L. -lbpf
 //#include "bpfWrapper.h"
@@ -39,6 +40,7 @@ type Handler interface {
 	LoadBpfSendXskMap(ifname string) (int, error)
 	LoadAttachBpfXdpPass(ifname string) error
 	ConfigureBusyPoll(fd int, busyTimeout int, busyBudget int) error
+	LoadBpfPinXskMap(ifname, pin_path string) error
 	Cleanbpf(ifname string) error
 }
 
@@ -80,6 +82,19 @@ func (r *handler) LoadAttachBpfXdpPass(ifname string) error {
 	return nil
 }
 
+/*
+LoadBpfPinXskMap is the GoLang wrapper for the C function Load_bpf_send_xsk_map
+*/
+func (r *handler) LoadBpfPinXskMap(ifname, pin_path string) error {
+	err := int(C.Load_bpf_pin_xsk_map(C.CString(ifname), C.CString(pin_path)))
+
+	if err < 0 {
+		return errors.New("error loading BPF program onto interface")
+	}
+
+	return nil
+}
+
 /*
 ConfigureBusyPoll is the GoLang wrapper for the C function Configure_busy_poll
 */

internal/bpf/bpfWrapper.h

@@ -19,6 +19,7 @@
 
 int Load_bpf_send_xsk_map(char *ifname);
 int Load_attach_bpf_xdp_pass(char *ifname);
+int Load_bpf_pin_xsk_map(char *ifname, char *pin_path);
 int Configure_busy_poll(int fd, int busy_timeout, int busy_budget);
 int Clean_bpf(char *ifname);
 

internal/bpf/bpfWrapper_fake.go

@@ -45,6 +45,14 @@ func (f *fakeHandler) LoadAttachBpfXdpPass(ifname string) error {
 	return nil
 }
 
+/*
+LoadBpfPinXskMap is the GoLang wrapper for the C function Load_bpf_pin_xsk_map
+In this fakeHandler it does nothing.
+*/
+func (f *fakeHandler) LoadBpfPinXskMap(ifname, pin_path string) error {
+	return nil
+}
+
 /*
 ConfigureBusyPoll is the GoLang wrapper for the C function Configure_busy_poll
 In this fakeHandler it does nothing.