Merge branch 'dev/k8s-deviceid-model' into master

examples/README.md

@@ -60,3 +60,34 @@ A sample `cni-configuration.conf` is provided, typically this file is placed in
 ## Other considerations
 
 Primarily in this setup one thing that one should consider are the aspects of the `macvlan-conf.yml`, which is likely specific to the configuration of the node on which this resides.
+
+## Passing down device information
+Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotaton in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on.
+
+In this exmaple (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/tree/dev/k8s-deviceid-model) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be pass-down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information.
+
+```yaml
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+spec:
+  config: '{
+  "type": "sriov",
+  "vlan": 1000,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.56.217.0/24",
+    "rangeStart": "10.56.217.171",
+    "rangeEnd": "10.56.217.181",
+    "routes": [{
+      "dst": "0.0.0.0/0"
+    }],
+    "gateway": "10.56.217.1"
+  }
+}'
+```
+
+>For further information on how to configure SRIOV Device Plugin and SRIOV-CNI please refer to the links given above.

examples/sriov-net.yaml

@@ -0,0 +1,21 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+spec:
+  config: '{
+  "type": "sriov",
+  "vlan": 1000,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.56.217.0/24",
+    "rangeStart": "10.56.217.171",
+    "rangeEnd": "10.56.217.181",
+    "routes": [{
+      "dst": "0.0.0.0/0"
+    }],
+    "gateway": "10.56.217.1"
+  }
+}'

k8sclient/k8sclient.go

@@ -35,11 +35,20 @@ import (
 	"github.com/intel/multus-cni/types"
 )
 
+const (
+	resourceNameAnnot = "k8s.v1.cni.cncf.io/resourceName"
+)
+
 // NoK8sNetworkError indicates error, no network in kubernetes
 type NoK8sNetworkError struct {
 	message string
 }
 
+type ResourceInfo struct {
+	Index     int
+	deviceIDs []string
+}
+
 type clientInfo struct {
 	Client       KubeClient
 	Podnamespace string
@@ -67,6 +76,26 @@ func (d *defaultKubeClient) UpdatePodStatus(pod *v1.Pod) (*v1.Pod, error) {
 	return d.client.Core().Pods(pod.Namespace).UpdateStatus(pod)
 }
 
+// GetComputeDeviceMap returns a map of resourceName to list of device IDs
+func GetComputeDeviceMap(pod *v1.Pod) map[string]*ResourceInfo {
+
+	resourceMap := make(map[string]*ResourceInfo)
+
+	for _, cntr := range pod.Spec.Containers {
+		for _, cd := range cntr.ComputeDevices {
+			entry, ok := resourceMap[cd.ResourceName]
+			if ok {
+				// already exists; append to it
+				entry.deviceIDs = append(entry.deviceIDs, cd.DeviceIDs...)
+			} else {
+				// new entry
+				resourceMap[cd.ResourceName] = &ResourceInfo{deviceIDs: cd.DeviceIDs}
+			}
+		}
+	}
+	return resourceMap
+}
+
 func setKubeClientInfo(c *clientInfo, client KubeClient, k8sArgs *types.K8sArgs) {
 	logging.Debugf("setKubeClientInfo: %v, %v, %v", c, client, k8sArgs)
 	c.Client = client
@@ -131,15 +160,8 @@ func setPodNetworkAnnotation(client KubeClient, namespace string, pod *v1.Pod, n
 	return pod, nil
 }
 
-func getPodNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, string, error) {
-	var err error
-
-	logging.Debugf("getPodNetworkAnnotation: %v, %v", client, k8sArgs)
-	pod, err := client.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
-	if err != nil {
-		return "", "", logging.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err)
-	}
-
+func getPodNetworkAnnotation(pod *v1.Pod) (string, string, error) {
+  logging.Debugf("getPodNetworkAnnotation: %v", pod)
 	return pod.Annotations["k8s.v1.cni.cncf.io/networks"], pod.ObjectMeta.Namespace, nil
 }
 
@@ -326,8 +348,9 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit
 	return config, nil
 }
 
-func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string) (*types.DelegateNetConf, error) {
-	logging.Debugf("getKubernetesDelegate: %v, %v, %s", client, net, confdir)
+func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string, resourceMap map[string]*ResourceInfo) (*types.DelegateNetConf, error) {
+
+  logging.Debugf("getKubernetesDelegate: %v, %v, %s", client, net, confdir)
 	rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", net.Namespace, net.Name)
 	netData, err := client.GetRawWithPath(rawPath)
 	if err != nil {
@@ -339,19 +362,57 @@ func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement
 		return nil, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err)
 	}
 
+	// DEVICE_ID
+	// Get resourceName annotation from NetDefinition
+	deviceID := ""
+	resourceName, ok := customResource.Metadata.Annotations[resourceNameAnnot]
+	if ok {
+		// ResourceName annotation is found; try to get device info from resourceMap
+		entry, ok := resourceMap[resourceName]
+		if ok {
+			if idCount := len(entry.deviceIDs); idCount > 0 && idCount > entry.Index {
+				deviceID = entry.deviceIDs[entry.Index]
+				entry.Index++ // increment Index for next delegate
+			}
+		}
+	}
+
 	configBytes, err := cniConfigFromNetworkResource(customResource, confdir)
 	if err != nil {
 		return nil, err
 	}
 
+	if deviceID != "" {
+		if configBytes, err = delegateAddDeviceID(configBytes, deviceID); err != nil {
+			return nil, err
+		}
+	}
+
 	delegate, err := types.LoadDelegateNetConf(configBytes, net.InterfaceRequest)
 	if err != nil {
 		return nil, err
 	}
-
+  
 	return delegate, nil
 }
 
+func delegateAddDeviceID(inBytes []byte, deviceID string) ([]byte, error) {
+	var rawConfig map[string]interface{}
+	var err error
+
+	err = json.Unmarshal(inBytes, &rawConfig)
+	if err != nil {
+		return nil, logging.Errorf("delegateAddDeviceID: failed to unmarshal inBytes: %v", err)
+	}
+	// Inject deviceID
+	rawConfig["deviceID"] = deviceID
+	configBytes, err := json.Marshal(rawConfig)
+	if err != nil {
+		return nil, logging.Errorf("delegateAddDeviceID: failed to re-marshal Spec.Config: %v", err)
+	}
+	return configBytes, nil
+}
+
 type KubeClient interface {
 	GetRawWithPath(path string) ([]byte, error)
 	GetPod(namespace, name string) (*v1.Pod, error)
@@ -447,11 +508,19 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error)
 func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) ([]*types.DelegateNetConf, error) {
 	logging.Debugf("GetK8sNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir)
 
-	netAnnot, defaultNamespace, err := getPodNetworkAnnotation(k8sclient, k8sArgs)
+	pod, err := k8sclient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME))
+	if err != nil {
+		return nil, logging.Errorf("GetK8sNetwork: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err)
+	}
+
+	netAnnot, defaultNamespace, err := getPodNetworkAnnotation(pod)
 	if err != nil {
 		return nil, err
 	}
 
+	// Get Pod ComputeDevices info
+	resourceMap := GetComputeDeviceMap(pod)
+
 	if len(netAnnot) == 0 {
 		return nil, &NoK8sNetworkError{"no kubernetes network found"}
 	}
@@ -464,7 +533,7 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string)
 	// Read all network objects referenced by 'networks'
 	var delegates []*types.DelegateNetConf
 	for _, net := range networks {
-		delegate, err := getKubernetesDelegate(k8sclient, net, confdir)
+		delegate, err := getKubernetesDelegate(k8sclient, net, confdir, resourceMap)
 		if err != nil {
 			return nil, logging.Errorf("GetK8sNetwork: failed getting the delegate: %v", err)
 		}