Feat kind support map pinning

.gitignore

@@ -2,6 +2,7 @@ bin
 *.o
 *.a
 .idea
+*.ll
 test/e2e/udsTest
 test/fuzz/cni/cni-fuzz.zip
 test/fuzz/test/fuzz/deviceplugin/uds/fuzz_error.log

Makefile

@@ -1,16 +1,21 @@
 # Copyright(c) 2022 Intel Corporation.
+# Copyright(c) Red Hat Inc.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+.PHONY: help
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
 excluded_from_utests = "/test/e2e|/test/fuzz"
 
 .PHONY: all e2e
@@ -29,6 +34,13 @@ format:
 	@echo
 	@echo
 
+buildxdp:
+	@echo "******     Build xdp_pass     ******"
+	make -C ./internal/bpf/xdp-pass/
+	@echo "******     Build xdp_afxdp_redirect     ******"
+	make -C ./internal/bpf/xdp-afxdp-redirect/
+	@echo
+
 buildc:
 	@echo "******     Build BPF     ******"
 	@echo
@@ -37,7 +49,7 @@ buildc:
 	@echo
 	@echo
 
-builddp: buildc
+builddp: buildc buildxdp
 	@echo "******     Build DP      ******"
 	@echo
 	go build -o ./bin/afxdp-dp ./cmd/deviceplugin
@@ -53,15 +65,16 @@ buildcni: buildc
 
 build: builddp buildcni
 
-docker:
-	@echo "******   Docker Image    ******"
+##@ General Build - assumes K8s environment is already setup
+docker: ## Build docker image
+	@echo "******  Docker Image    ******"
 	@echo
 	docker build -t afxdp-device-plugin -f images/amd64.dockerfile .
 	@echo
 	@echo
 
-podman:
-	@echo "******   Podman Image    ******"
+podman: ## Build podman image
+	@echo "******  Podman Image    ******"
 	@echo
 	podman build -t afxdp-device-plugin -f images/amd64.dockerfile .
 	@echo
@@ -75,14 +88,14 @@ image:
 	 $(MAKE) docker; \
 	fi
 
-undeploy:
+undeploy: ## Undeploy the Deamonset
 	@echo "******  Stop Daemonset   ******"
 	@echo
 	kubectl delete -f ./deployments/daemonset.yml --ignore-not-found=true
 	@echo
 	@echo
 
-deploy: image undeploy
+deploy: image undeploy ## Deploy the Deamonset and CNI
 	@echo "****** Deploy Daemonset  ******"
 	@echo
 	kubectl create -f ./deployments/daemonset.yml
@@ -147,7 +160,7 @@ static-ci:
 	@echo
 
 # static: consists of static analysis tools required for internal CI repository workflows and locally
-# run tests. static includes static-ci test module. 
+# run tests. static includes static-ci test module.
 static: static-ci
 	@echo "******   GolangCI-Lint   ******"
 	@echo
@@ -188,3 +201,43 @@ clean:
 	rm -f ./internal/bpf/libwrapper.a
 	@echo
 	@echo
+
+##@ General setup
+
+.PHONY: setup-flannel
+setup-flannel: ## Setup flannel
+	kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml
+
+.PHONY: setup-multus
+setup-multus: ## Setup multus
+	kubectl apply -f https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml
+
+##@ Kind Deployment - sets up a kind cluster and deploys the plugin and CNI
+
+.PHONY: del-kind
+del-kind: ## Remove a kind cluster called af-xdp-deployment
+	kind delete cluster --name af-xdp-deployment
+
+.PHONY: setup-kind
+setup-kind: del-kind ## Setup a kind cluster called af-xdp-deployment
+	mkdir -p /tmp/afxdp_dp/
+	mkdir -p /tmp/afxdp_dp2/
+	kind create cluster --config hack/kind-config.yaml --name af-xdp-deployment
+
+.PHONY: label-kind-nodes
+label-kind-nodes: ## label the kind worker nodes with cndp="true"
+	kubectl label node af-xdp-deployment-worker cndp="true"
+	kubectl label node af-xdp-deployment-worker2 cndp="true"
+
+.PHONY: kind-deploy
+kind-deploy: image undeploy ## Deploy the Deamonset and CNI in Kind
+	@echo "****** Deploy Daemonset  ******"
+	@echo
+	kind load --name af-xdp-deployment docker-image afxdp-device-plugin
+	kubectl create -f ./deployments/daemonset-kind.yaml
+	@echo
+	@echo
+
+.PHONY: run-on-kind
+run-on-kind: del-kind setup-kind label-kind-nodes setup-multus kind-deploy ## Setup a kind cluster and deploy the device plugin
+	@echo "******       Kind Setup complete       ******"

cmd/deviceplugin/main.go

@@ -61,6 +61,15 @@ func main() {
 		exit(constants.Plugins.DevicePlugin.ExitLogError)
 	}
 
+	// configure a set of veths and a bridge as a secondary kind network.
+	if cfg.ClusterType == "kind" {
+		if err := configureKindSecondaryNetwork(); err != nil {
+			logging.Errorf("Error configuring Kind Secondary Network: %v", err)
+			exit(constants.Plugins.DevicePlugin.ExitKindError)
+		}
+
+	}
+
 	//device file
 	exists, err := tools.FilePathExists(deviceFile)
 	if err != nil {
@@ -166,6 +175,41 @@ func configureLogging(cfg deviceplugin.PluginConfig) error {
 	return nil
 }
 
+// On each Kind node
+// Create a bridge afxdp-kind-br
+// Create 4 vethpairs starting at veth6
+//  +===============+
+//  | afxdp-kind-br |
+//  |     +---------|         +---------+
+//  |     |  veth7  | <=====> |  veth6  |
+//  |     +---------|         +---------+
+//  |     +---------|         +---------+
+//  |     |  veth9  | <=====> |  veth8  |
+//  |     +---------|         +---------+
+//  |     +---------|         +---------+
+//  |     |  veth11 | <=====> |  veth10 |
+//  |     +---------|         +---------+
+//  |     +---------|         +---------+
+//  |     |  veth13 | <=====> |  veth12 |
+//  |     +---------|         +---------+
+//  +===============+
+// The "even" veth of the pair will be added to the device plugin resource pool.
+// and plumbed to the Pod.
+func configureKindSecondaryNetwork() error {
+	numVeths := 4
+	offset := 6
+
+	err := networking.CreateKindNetwork(numVeths, offset)
+	if err != nil {
+		logging.Errorf("Error Creating CreateKindNetwork %s", err.Error())
+		return err
+	}
+
+	logging.Infof("Created CreateKindNetwork")
+
+	return nil
+}
+
 func checkHost(host host.Handler) (bool, error) {
 	// kernel
 	logging.Debugf("Checking kernel version")

constants/constants.go

@@ -1,5 +1,6 @@
 /*
  * Copyright(c) 2022 Intel Corporation.
+ * Copyright(c) Red Hat Inc.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -25,6 +26,10 @@ var (
 	devicePluginExitLogError      = 2                          // device plugin logging error exit code, error creating log file, bad log level, etc.
 	devicePluginExitHostError     = 3                          // device plugin host check exit code, error occurred checking some attribute of the host
 	devicePluginExitPoolError     = 4                          // device plugin device pool exit code, error occurred while building a device pool
+	devicePluginExitKindError     = 5                          // device plugin Kind exit code, error occurred while creating a kind secondary network
+
+	/* Cluster Types */
+	clusterTypes = []string{"kind", "physical"} // accepted kind cluster types
 
 	/* Logging */
 	logLevels          = []string{"debug", "info", "warning", "error"} // accepted log levels
@@ -44,13 +49,13 @@ var (
 	deviceSecondaryMax   = 64                                                       // maximum number of secondary devices that can be created on top of a primary device
 
 	/* Drivers */
-	driversZeroCopy      = []string{"i40e", "E810", "ice"} // drivers that support zero copy AF_XDP
-	driversCdq           = []string{"ice"}                 // drivers that support CDQ subfunctions
-	driverValidNameRegex = `^[a-zA-Z0-9_-]+$`              // regex to check if a string is a valid driver name
-	driverValidNameMin   = 1                               // minimum length of a driver name
-	driverValidNameMax   = 50                              // maximum length of a deiver name
-	driverPrimaryMin     = 1                               // minimum number of primary devices a driver can take from a node
-	driverPrimaryMax     = 10                              // maximum number of primary devices a driver can take from a node
+	driversZeroCopy      = []string{"i40e", "E810", "ice", "veth"} // drivers that support zero copy AF_XDP
+	driversCdq           = []string{"ice"}                         // drivers that support CDQ subfunctions
+	driverValidNameRegex = `^[a-zA-Z0-9_-]+$`                      // regex to check if a string is a valid driver name
+	driverValidNameMin   = 1                                       // minimum length of a driver name
+	driverValidNameMax   = 50                                      // maximum length of a deiver name
+	driverPrimaryMin     = 1                                       // minimum number of primary devices a driver can take from a node
+	driverPrimaryMax     = 10                                      // maximum number of primary devices a driver can take from a node
 
 	/* Nodes */
 	nodeValidHostRegex = `^[a-zA-Z0-9-]+$` // regex to check if a string is a valid node name
@@ -77,6 +82,10 @@ var (
 	udsSockDir    = "/tmp/afxdp_dp/"  // host location where we place our uds sockets. If changing location remember to update daemonset mount point
 	udsPodPath    = "/tmp/afxdp.sock" // the uds filepath as it will appear in the end user application pod
 
+	/* BPF*/
+	bpfMapPodPath = "/tmp/xsks_map"
+	xsk_map       = "/xsks_map"
+
 	udsDirFileMode = 0700 // permissions for the directory in which we create our uds sockets
 
 	/* Handshake*/
@@ -129,6 +138,8 @@ var (
 	DeviceFile deviceFile
 	/* DeviceFile contains constants related to the devicefile */
 	EthtoolFilter ethtoolFilter
+	/* Bpf contains constants related to the BPF Map pinning */
+	Bpf bpf
 )
 
 type cni struct {
@@ -142,12 +153,14 @@ type devicePlugin struct {
 	ExitLogError      int
 	ExitHostError     int
 	ExitPoolError     int
+	ExitKindError     int
 }
 
 type plugins struct {
 	Modes        []string
 	Cni          cni
 	DevicePlugin devicePlugin
+	ClusterTypes []string
 }
 
 type afxdp struct {
@@ -211,6 +224,11 @@ type uds struct {
 	Handshake   handshake
 }
 
+type bpf struct {
+	BpfMapPodPath string
+	Xsk_map       string
+}
+
 type handshake struct {
 	Version             string
 	RequestVersion      string
@@ -241,7 +259,8 @@ type ethtoolFilter struct {
 
 func init() {
 	Plugins = plugins{
-		Modes: pluginModes,
+		Modes:        pluginModes,
+		ClusterTypes: clusterTypes,
 		DevicePlugin: devicePlugin{
 			DefaultConfigFile: devicePluginDefaultConfigFile,
 			DevicePrefix:      devicePluginDevicePrefix,
@@ -250,6 +269,7 @@ func init() {
 			ExitLogError:      devicePluginExitLogError,
 			ExitHostError:     devicePluginExitHostError,
 			ExitPoolError:     devicePluginExitPoolError,
+			ExitKindError:     devicePluginExitKindError,
 		},
 	}
 
@@ -330,6 +350,11 @@ func init() {
 		},
 	}
 
+	Bpf = bpf{
+		BpfMapPodPath: bpfMapPodPath,
+		Xsk_map:       xsk_map,
+	}
+
 	DeviceFile = deviceFile{
 		Name:            name,
 		FilePermissions: filePermissions,