From dd1ab30d8e4d9762eb05e4f4a2659a7b0ee70145 Mon Sep 17 00:00:00 2001 From: Tomofumi Hayashi Date: Tue, 2 Oct 2018 22:49:59 +0900 Subject: [PATCH 01/16] Add bracket [] in Dockerfile's entrypoint to parse argument correctly. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 16f6cd236..91f66992a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,4 +24,4 @@ ADD ./images/entrypoint.sh / # does it require a root user? # USER 1001 -ENTRYPOINT /entrypoint.sh +ENTRYPOINT [/entrypoint.sh] From 660524c6f3853d8010b7f31ca2da3ca3c22b076f Mon Sep 17 00:00:00 2001 From: Louis Woods Date: Wed, 3 Oct 2018 11:28:15 -0700 Subject: [PATCH 02/16] Add the option to auto generate 00-multus.conf When `--multus-conf-file=auto` is used, 00-multus.conf will be automatically generated from the CNI configuration file of the master plugin (the first file in lexicographical order in cni-conf-dir). --- images/README.md | 15 +++++++----- images/entrypoint.sh | 54 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/images/README.md b/images/README.md index e16f4106f..706268365 100644 --- a/images/README.md +++ b/images/README.md @@ -31,10 +31,12 @@ You can get get help with the `--help` flag. ``` $ ./entrypoint.sh --help -This is an entrypoint script for Multus CNI to overlay its -binary and configuration into locations in a filesystem. -The configuration & binary file will be copied to the -corresponding configuration directory. +This is an entrypoint script for Multus CNI to overlay its binary and +configuration into locations in a filesystem. The configuration & binary file +will be copied to the corresponding configuration directory. When +`--multus-conf-file=auto` is used, 00-multus.conf will be automatically +generated from the CNI configuration file of the master plugin (the first file +in lexicographical order in cni-conf-dir). ./entrypoint.sh -h --help @@ -42,6 +44,7 @@ corresponding configuration directory. --cni-bin-dir=/host/opt/cni/bin --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf --multus-bin-file=/usr/src/multus-cni/bin/multus + --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig ``` You must use an `=` to delimit the parameter name and the value. For example you may set a custom `cni-conf-dir` like so: @@ -59,7 +62,7 @@ Note: You'll noticed that there's a `/host/...` directory from the root for the Example docker run command: ``` -$ docker run -it -v /opt/cni/bin/:/host/opt/cni/bin/ -v /etc/cni/net.d/:/host/etc/cni/net.d/ --entrypoint=/bin/bash dougbtv/multus +$ docker run -it -v /opt/cni/bin/:/host/opt/cni/bin/ -v /etc/cni/net.d/:/host/etc/cni/net.d/ --entrypoint=/bin/bash dougbtv/multus ``` -Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://github.com/projectcalico/calico/blob/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager/calico-daemonset.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153). \ No newline at end of file +Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://github.com/projectcalico/calico/blob/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager/calico-daemonset.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153). diff --git a/images/entrypoint.sh b/images/entrypoint.sh index 8b60bf032..542393825 100755 --- a/images/entrypoint.sh +++ b/images/entrypoint.sh @@ -8,14 +8,17 @@ CNI_CONF_DIR="/host/etc/cni/net.d" CNI_BIN_DIR="/host/opt/cni/bin" MULTUS_CONF_FILE="/usr/src/multus-cni/images/70-multus.conf" MULTUS_BIN_FILE="/usr/src/multus-cni/bin/multus" +MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig" # Give help text for parameters. function usage() { - echo -e "This is an entrypoint script for Multus CNI to overlay its" - echo -e "binary and configuration into locations in a filesystem." - echo -e "The configuration & binary file will be copied to the " - echo -e "corresponding configuration directory." + echo -e "This is an entrypoint script for Multus CNI to overlay its binary and " + echo -e "configuration into locations in a filesystem. The configuration & binary file " + echo -e "will be copied to the corresponding configuration directory. When " + echo -e "`--multus-conf-file=auto` is used, 00-multus.conf will be automatically " + echo -e "generated from the CNI configuration file of the master plugin (the first file " + echo -e "in lexicographical order in cni-conf-dir)." echo -e "" echo -e "./entrypoint.sh" echo -e "\t-h --help" @@ -23,6 +26,7 @@ function usage() echo -e "\t--cni-bin-dir=$CNI_BIN_DIR" echo -e "\t--multus-conf-file=$MULTUS_CONF_FILE" echo -e "\t--multus-bin-file=$MULTUS_BIN_FILE" + echo -e "\t--multus-kubeconfig-file-host=$MULTUS_KUBECONFIG_FILE_HOST" } # Parse parameters given as arguments to this script. @@ -46,6 +50,9 @@ while [ "$1" != "" ]; do --multus-bin-file) MULTUS_BIN_FILE=$VALUE ;; + --multus-kubeconfig-file-host) + MULTUS_KUBECONFIG_FILE_HOST=$VALUE + ;; *) echo "ERROR: unknown parameter \"$PARAM\"" usage @@ -57,7 +64,11 @@ done # Create array of known locations -declare -a arr=($CNI_CONF_DIR $CNI_BIN_DIR $MULTUS_CONF_FILE $MULTUS_BIN_FILE) +declare -a arr=($CNI_CONF_DIR $CNI_BIN_DIR $MULTUS_BIN_FILE) +if [ "$MULTUS_CONF_FILE" != "auto" ]; then + arr+=($MULTUS_BIN_FILE) +fi + # Loop through and verify each location each. for i in "${arr[@]}" @@ -69,8 +80,10 @@ do done # Copy files into proper places. -cp -f $MULTUS_CONF_FILE $CNI_CONF_DIR cp -f $MULTUS_BIN_FILE $CNI_BIN_DIR +if [ "$MULTUS_CONF_FILE" != "auto" ]; then + cp -f $MULTUS_CONF_FILE $CNI_CONF_DIR +fi # Make a multus.d directory (for our kubeconfig) @@ -134,6 +147,35 @@ fi # ---------------------- end Generate a "kube-config". +# ------------------------------- Generate "00-multus.conf" + +if [ "$MULTUS_CONF_FILE" == "auto" ]; then + echo "Generating Multus configuration file ..." + MASTER_PLUGIN="$(ls $CNI_CONF_DIR | grep .conf | head -1)" + if [ "$MASTER_PLUGIN" == "" ]; then + echo "Error: Multus could not be configured: no master plugin was found." + exit 1; + elif [ "$MASTER_PLUGIN" == "00-multus.conf" ]; then + echo "Warning: Multus is already configured: auto configuration skipped." + else + MASTER_PLUGIN_JSON="$(cat $CNI_CONF_DIR/$MASTER_PLUGIN)" + CONF=$(cat <<-EOF + { + "name": "multus-cni-network", + "type": "multus", + "kubeconfig": "$MULTUS_KUBECONFIG_FILE_HOST", + "delegates": [ + $MASTER_PLUGIN_JSON + ] + } + EOF + ) + echo $CONF > $CNI_CONF_DIR/00-multus.conf + fi +fi + +# ---------------------- end Generate "00-multus.conf". + echo "Entering sleep... (success)" # Sleep forever. From 9614a82349c69a46f98e83a13cb532eb7f997835 Mon Sep 17 00:00:00 2001 From: Louis Woods Date: Fri, 5 Oct 2018 14:15:52 -0700 Subject: [PATCH 03/16] Improve grep in entrypoint.sh to only find .conf and .conflist files --- images/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/entrypoint.sh b/images/entrypoint.sh index 542393825..679acaa0d 100755 --- a/images/entrypoint.sh +++ b/images/entrypoint.sh @@ -151,7 +151,7 @@ fi if [ "$MULTUS_CONF_FILE" == "auto" ]; then echo "Generating Multus configuration file ..." - MASTER_PLUGIN="$(ls $CNI_CONF_DIR | grep .conf | head -1)" + MASTER_PLUGIN="$(ls $CNI_CONF_DIR | grep -E '\.conf(list)?$' | head -1)" if [ "$MASTER_PLUGIN" == "" ]; then echo "Error: Multus could not be configured: no master plugin was found." exit 1; From ff091fe2b75dda0bdac6ed459053e2d8aa2a1c45 Mon Sep 17 00:00:00 2001 From: Abdul Halim Date: Mon, 10 Sep 2018 16:27:07 +0100 Subject: [PATCH 04/16] parse kubelet checkpoint file for pod devices Enabling kubelete checkpoint file parsing to get Pod device info so that these device information can be passed into CNI plugins that need specific device information to work on. Change-Id: I6630f56adc0a8307f575fc09ce9090c1ffca0337 --- checkpoint/checkpoint.go | 76 +++++++++++++++++++++++++++ examples/README.md | 32 +++++++++++ examples/net-resource-sample-pod.yaml | 21 ++++++++ examples/sriov-net.yaml | 21 ++++++++ k8sclient/k8sclient.go | 40 +++++++++++--- types/conf.go | 31 ++++++++++- types/types.go | 6 +++ 7 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 checkpoint/checkpoint.go create mode 100644 examples/net-resource-sample-pod.yaml create mode 100644 examples/sriov-net.yaml diff --git a/checkpoint/checkpoint.go b/checkpoint/checkpoint.go new file mode 100644 index 000000000..2180aebdd --- /dev/null +++ b/checkpoint/checkpoint.go @@ -0,0 +1,76 @@ +package checkpoint + +import ( + "encoding/json" + "fmt" + "io/ioutil" + + "github.com/intel/multus-cni/types" +) + +const ( + checkPointfile = "/var/lib/kubelet/device-plugins/kubelet_internal_checkpoint" +) + +type PodDevicesEntry struct { + PodUID string + ContainerName string + ResourceName string + DeviceIDs []string + AllocResp []byte +} + +type checkpointData struct { + PodDeviceEntries []PodDevicesEntry + RegisteredDevices map[string][]string +} + +type Data struct { + Data checkpointData + Checksum uint64 +} + +// getPodEntries gets all Pod device allocation entries from checkpoint file +func getPodEntries() ([]PodDevicesEntry, error) { + + podEntries := []PodDevicesEntry{} + + cpd := &Data{} + rawBytes, err := ioutil.ReadFile(checkPointfile) + if err != nil { + return podEntries, fmt.Errorf("getPodEntries(): error reading file %s\n%v\n", checkPointfile, err) + + } + + if err = json.Unmarshal(rawBytes, cpd); err != nil { + return podEntries, fmt.Errorf("getPodEntries(): error unmarshalling raw bytes %v", err) + } + + return cpd.Data.PodDeviceEntries, nil +} + +// GetComputeDeviceMap returns a map of resourceName to list of device IDs +func GetComputeDeviceMap(podID string) (map[string]*types.ResourceInfo, error) { + + resourceMap := make(map[string]*types.ResourceInfo) + podEntires, err := getPodEntries() + + if err != nil { + return nil, err + } + + for _, pod := range podEntires { + if pod.PodUID == podID { + entry, ok := resourceMap[pod.ResourceName] + if ok { + // already exists; append to it + entry.DeviceIDs = append(entry.DeviceIDs, pod.DeviceIDs...) + } else { + // new entry + resourceMap[pod.ResourceName] = &types.ResourceInfo{DeviceIDs: pod.DeviceIDs} + } + } + } + + return resourceMap, nil +} diff --git a/examples/README.md b/examples/README.md index ba5906e6b..fb9556fae 100644 --- a/examples/README.md +++ b/examples/README.md @@ -60,3 +60,35 @@ A sample `cni-configuration.conf` is provided, typically this file is placed in ## Other considerations Primarily in this setup one thing that one should consider are the aspects of the `macvlan-conf.yml`, which is likely specific to the configuration of the node on which this resides. + +## Passing down device information +Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotaton in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on. + +In this exmaple (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/tree/dev/k8s-deviceid-model) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information. + +```yaml +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net-a + annotations: + k8s.v1.cni.cncf.io/resourceName: intel.com/sriov +spec: + config: '{ + "type": "sriov", + "vlan": 1000, + "ipam": { + "type": "host-local", + "subnet": "10.56.217.0/24", + "rangeStart": "10.56.217.171", + "rangeEnd": "10.56.217.181", + "routes": [{ + "dst": "0.0.0.0/0" + }], + "gateway": "10.56.217.1" + } +}' +``` +The [net-resource-sample-pod.yaml](./net-resource-sample-pod.yaml) is an exmaple Pod manifest file that requesting a SRIOV device from a host which is then configured using the above network attachement definition. + +>For further information on how to configure SRIOV Device Plugin and SRIOV-CNI please refer to the links given above. \ No newline at end of file diff --git a/examples/net-resource-sample-pod.yaml b/examples/net-resource-sample-pod.yaml new file mode 100644 index 000000000..3f6eb0686 --- /dev/null +++ b/examples/net-resource-sample-pod.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: testpod1 + labels: + env: test + annotations: + k8s.v1.cni.cncf.io/networks: sriov-net-a +spec: + containers: + - name: appcntr1 + image: centos/tools + imagePullPolicy: IfNotPresent + command: [ "/bin/bash", "-c", "--" ] + args: [ "while true; do sleep 300000; done;" ] + resources: + requests: + intel.com/sriov: '1' + limits: + intel.com/sriov: '1' + restartPolicy: "Never" diff --git a/examples/sriov-net.yaml b/examples/sriov-net.yaml new file mode 100644 index 000000000..9b265d3e3 --- /dev/null +++ b/examples/sriov-net.yaml @@ -0,0 +1,21 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net-a + annotations: + k8s.v1.cni.cncf.io/resourceName: intel.com/sriov +spec: + config: '{ + "type": "sriov", + "vlan": 1000, + "ipam": { + "type": "host-local", + "subnet": "10.56.217.0/24", + "rangeStart": "10.56.217.171", + "rangeEnd": "10.56.217.181", + "routes": [{ + "dst": "0.0.0.0/0" + }], + "gateway": "10.56.217.1" + } +}' diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index 8e1ee6433..ba9435bf7 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -31,10 +31,15 @@ import ( "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/pkg/skel" cnitypes "github.com/containernetworking/cni/pkg/types" + "github.com/intel/multus-cni/checkpoint" "github.com/intel/multus-cni/logging" "github.com/intel/multus-cni/types" ) +const ( + resourceNameAnnot = "k8s.v1.cni.cncf.io/resourceName" +) + // NoK8sNetworkError indicates error, no network in kubernetes type NoK8sNetworkError struct { message string @@ -131,16 +136,16 @@ func setPodNetworkAnnotation(client KubeClient, namespace string, pod *v1.Pod, n return pod, nil } -func getPodNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, string, error) { +func getPodNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, string, string, error) { var err error logging.Debugf("getPodNetworkAnnotation: %v, %v", client, k8sArgs) pod, err := client.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) if err != nil { - return "", "", logging.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err) + return "", "", "", logging.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err) } - return pod.Annotations["k8s.v1.cni.cncf.io/networks"], pod.ObjectMeta.Namespace, nil + return pod.Annotations["k8s.v1.cni.cncf.io/networks"], pod.ObjectMeta.Namespace, string(pod.UID), nil } func parsePodNetworkObjectName(podnetwork string) (string, string, string, error) { @@ -326,7 +331,8 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit return config, nil } -func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string) (*types.DelegateNetConf, error) { +func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, error) { + logging.Debugf("getKubernetesDelegate: %v, %v, %s", client, net, confdir) rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", net.Namespace, net.Name) netData, err := client.GetRawWithPath(rawPath) @@ -339,12 +345,26 @@ func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement return nil, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err) } + // Get resourceName annotation from NetDefinition + deviceID := "" + resourceName, ok := customResource.Metadata.Annotations[resourceNameAnnot] + if ok { + // ResourceName annotation is found; try to get device info from resourceMap + entry, ok := resourceMap[resourceName] + if ok { + if idCount := len(entry.DeviceIDs); idCount > 0 && idCount > entry.Index { + deviceID = entry.DeviceIDs[entry.Index] + entry.Index++ // increment Index for next delegate + } + } + } + configBytes, err := cniConfigFromNetworkResource(customResource, confdir) if err != nil { return nil, err } - delegate, err := types.LoadDelegateNetConf(configBytes, net.InterfaceRequest) + delegate, err := types.LoadDelegateNetConf(configBytes, net.InterfaceRequest, deviceID) if err != nil { return nil, err } @@ -447,11 +467,17 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error) func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) ([]*types.DelegateNetConf, error) { logging.Debugf("GetK8sNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir) - netAnnot, defaultNamespace, err := getPodNetworkAnnotation(k8sclient, k8sArgs) + netAnnot, defaultNamespace, podID, err := getPodNetworkAnnotation(k8sclient, k8sArgs) if err != nil { return nil, err } + // Get Pod ComputeDevices info + resourceMap, err := checkpoint.GetComputeDeviceMap(podID) + if err != nil { + return nil, logging.Errorf("GetK8sNetwork: failed to get resourceMap for PodUID: %v %v", podID, err) + } + if len(netAnnot) == 0 { return nil, &NoK8sNetworkError{"no kubernetes network found"} } @@ -464,7 +490,7 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) // Read all network objects referenced by 'networks' var delegates []*types.DelegateNetConf for _, net := range networks { - delegate, err := getKubernetesDelegate(k8sclient, net, confdir) + delegate, err := getKubernetesDelegate(k8sclient, net, confdir, resourceMap) if err != nil { return nil, logging.Errorf("GetK8sNetwork: failed getting the delegate: %v", err) } diff --git a/types/conf.go b/types/conf.go index e95d9cee2..bfbc13bce 100644 --- a/types/conf.go +++ b/types/conf.go @@ -50,7 +50,16 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error } // Convert raw CNI JSON into a DelegateNetConf structure -func LoadDelegateNetConf(bytes []byte, ifnameRequest string) (*DelegateNetConf, error) { +func LoadDelegateNetConf(bytes []byte, ifnameRequest, deviceID string) (*DelegateNetConf, error) { + // If deviceID is present, inject this into delegate config + if deviceID != "" { + if updatedBytes, err := delegateAddDeviceID(bytes, deviceID); err != nil { + return nil, logging.Errorf("error in LoadDelegateNetConf - delegateAddDeviceID unable to update delegate config: %v", err) + } else { + bytes = updatedBytes + } + } + delegateConf := &DelegateNetConf{} logging.Debugf("LoadDelegateNetConf: %s, %s", string(bytes), ifnameRequest) if err := json.Unmarshal(bytes, &delegateConf.Conf); err != nil { @@ -190,7 +199,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { if err != nil { return nil, logging.Errorf("error marshalling delegate %d config: %v", idx, err) } - delegateConf, err := LoadDelegateNetConf(bytes, "") + delegateConf, err := LoadDelegateNetConf(bytes, "", "") if err != nil { return nil, logging.Errorf("failed to load delegate %d config: %v", idx, err) } @@ -210,3 +219,21 @@ func (n *NetConf) AddDelegates(newDelegates []*DelegateNetConf) error { n.Delegates = append(n.Delegates, newDelegates...) return nil } + +// delegateAddDeviceID injects deviceID information in delegate bytes +func delegateAddDeviceID(inBytes []byte, deviceID string) ([]byte, error) { + var rawConfig map[string]interface{} + var err error + + err = json.Unmarshal(inBytes, &rawConfig) + if err != nil { + return nil, logging.Errorf("delegateAddDeviceID: failed to unmarshal inBytes: %v", err) + } + // Inject deviceID + rawConfig["deviceID"] = deviceID + configBytes, err := json.Marshal(rawConfig) + if err != nil { + return nil, logging.Errorf("delegateAddDeviceID: failed to re-marshal Spec.Config: %v", err) + } + return configBytes, nil +} diff --git a/types/types.go b/types/types.go index 20ed055ea..b0760280b 100644 --- a/types/types.go +++ b/types/types.go @@ -119,3 +119,9 @@ type K8sArgs struct { K8S_POD_NAMESPACE types.UnmarshallableString K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString } + +// ResourceInfo is struct to hold Pod device allocation information +type ResourceInfo struct { + Index int + DeviceIDs []string +} From d40cba831cb1915df03223adf77ec5141a50d1c1 Mon Sep 17 00:00:00 2001 From: Abdul Halim Date: Tue, 18 Sep 2018 10:36:30 +0100 Subject: [PATCH 05/16] updated examples/README.md Change-Id: I650fec86659b3690e1dc4b15bf84b6574cb0baba --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index fb9556fae..17e0fb178 100644 --- a/examples/README.md +++ b/examples/README.md @@ -64,7 +64,7 @@ Primarily in this setup one thing that one should consider are the aspects of th ## Passing down device information Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotaton in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on. -In this exmaple (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/tree/dev/k8s-deviceid-model) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information. +In this exmaple (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information. ```yaml apiVersion: "k8s.cni.cncf.io/v1" From 80074d8a390eb49223e62620b899549e8b60410a Mon Sep 17 00:00:00 2001 From: Abdul Halim Date: Tue, 18 Sep 2018 13:14:58 +0100 Subject: [PATCH 06/16] only create resourceMap on demand making resourceMap a singleton object and only initialize it once if one or more CRDs have a resourceName annotation in them. Added copyright header for checkpoint/checkpoint.go. Replaced fmt.Errorf with logging. Change-Id: I54628d69324833e70a75dcf6533e6642dedde9b5 --- checkpoint/checkpoint.go | 42 +++++++++++++++++++++++++++++++++------- k8sclient/k8sclient.go | 30 ++++++++++++++++++---------- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/checkpoint/checkpoint.go b/checkpoint/checkpoint.go index 2180aebdd..a6577e175 100644 --- a/checkpoint/checkpoint.go +++ b/checkpoint/checkpoint.go @@ -1,10 +1,25 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + package checkpoint import ( "encoding/json" - "fmt" "io/ioutil" + "github.com/intel/multus-cni/logging" "github.com/intel/multus-cni/types" ) @@ -38,27 +53,41 @@ func getPodEntries() ([]PodDevicesEntry, error) { cpd := &Data{} rawBytes, err := ioutil.ReadFile(checkPointfile) if err != nil { - return podEntries, fmt.Errorf("getPodEntries(): error reading file %s\n%v\n", checkPointfile, err) + return podEntries, logging.Errorf("getPodEntries(): error reading file %s\n%v\n", checkPointfile, err) } if err = json.Unmarshal(rawBytes, cpd); err != nil { - return podEntries, fmt.Errorf("getPodEntries(): error unmarshalling raw bytes %v", err) + return podEntries, logging.Errorf("getPodEntries(): error unmarshalling raw bytes %v", err) } return cpd.Data.PodDeviceEntries, nil } -// GetComputeDeviceMap returns a map of resourceName to list of device IDs +var instance map[string]*types.ResourceInfo + +// GetComputeDeviceMap returns an instance of a map of ResourceInfo func GetComputeDeviceMap(podID string) (map[string]*types.ResourceInfo, error) { + if instance == nil { + if resourceMap, err := getResourceMapFromFile(podID); err == nil { + logging.Debugf("GetComputeDeviceMap(): created new instance of resourceMap for Pod: %s", podID) + instance = resourceMap + } else { + logging.Errorf("GetComputeDeviceMap(): error creating resourceMap instance %v", err) + return nil, err + } + } + logging.Debugf("GetComputeDeviceMap(): resourceMap instance: %+v", instance) + return instance, nil +} + +func getResourceMapFromFile(podID string) (map[string]*types.ResourceInfo, error) { resourceMap := make(map[string]*types.ResourceInfo) podEntires, err := getPodEntries() - if err != nil { return nil, err } - for _, pod := range podEntires { if pod.PodUID == podID { entry, ok := resourceMap[pod.ResourceName] @@ -71,6 +100,5 @@ func GetComputeDeviceMap(podID string) (map[string]*types.ResourceInfo, error) { } } } - return resourceMap, nil } diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index ba9435bf7..f239a78f4 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -331,29 +331,36 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit return config, nil } -func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, error) { +func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string, podID string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) { logging.Debugf("getKubernetesDelegate: %v, %v, %s", client, net, confdir) rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", net.Namespace, net.Name) netData, err := client.GetRawWithPath(rawPath) if err != nil { - return nil, logging.Errorf("getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err) + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err) } customResource := &types.NetworkAttachmentDefinition{} if err := json.Unmarshal(netData, customResource); err != nil { - return nil, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err) + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err) } // Get resourceName annotation from NetDefinition deviceID := "" resourceName, ok := customResource.Metadata.Annotations[resourceNameAnnot] - if ok { + if ok && podID != "" { + logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName) // ResourceName annotation is found; try to get device info from resourceMap + resourceMap, err := checkpoint.GetComputeDeviceMap(podID) + if err != nil { + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get resourceMap from kubelet checkpoint file: %v", err) + } + entry, ok := resourceMap[resourceName] if ok { if idCount := len(entry.DeviceIDs); idCount > 0 && idCount > entry.Index { deviceID = entry.DeviceIDs[entry.Index] + logging.Debugf("getKubernetesDelegate: podID: %s deviceID: %s", podID, deviceID) entry.Index++ // increment Index for next delegate } } @@ -361,15 +368,15 @@ func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement configBytes, err := cniConfigFromNetworkResource(customResource, confdir) if err != nil { - return nil, err + return nil, resourceMap, err } delegate, err := types.LoadDelegateNetConf(configBytes, net.InterfaceRequest, deviceID) if err != nil { - return nil, err + return nil, resourceMap, err } - return delegate, nil + return delegate, resourceMap, nil } type KubeClient interface { @@ -472,8 +479,6 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) return nil, err } - // Get Pod ComputeDevices info - resourceMap, err := checkpoint.GetComputeDeviceMap(podID) if err != nil { return nil, logging.Errorf("GetK8sNetwork: failed to get resourceMap for PodUID: %v %v", podID, err) } @@ -487,14 +492,19 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) return nil, err } + // resourceMap holds Pod device allocation information; only initizized if CRD contains 'resourceName' annotation. + // This is only initialized once and all delegate objects can reference this to look up device info. + var resourceMap map[string]*types.ResourceInfo + // Read all network objects referenced by 'networks' var delegates []*types.DelegateNetConf for _, net := range networks { - delegate, err := getKubernetesDelegate(k8sclient, net, confdir, resourceMap) + delegate, resourceMap, err := getKubernetesDelegate(k8sclient, net, confdir, podID, resourceMap) if err != nil { return nil, logging.Errorf("GetK8sNetwork: failed getting the delegate: %v", err) } delegates = append(delegates, delegate) + _ = resourceMap // workaround for 'Go' error: 'resourceMap' declared and not used. } return delegates, nil From 5d9a3750fb6291d2164c7664e4cee5f8792a9ce6 Mon Sep 17 00:00:00 2001 From: Abdul Halim Date: Wed, 19 Sep 2018 16:41:08 +0100 Subject: [PATCH 07/16] refactoring checkpoint.go code to be testable this changes will allow mocking checkpoint instance for unit tests Change-Id: I72fb25d15d5c9f28577a0fcbfcd385df523a5e57 --- checkpoint/checkpoint.go | 67 +++++++++++++++++++++++----------------- k8sclient/k8sclient.go | 22 ++++++++----- 2 files changed, 53 insertions(+), 36 deletions(-) diff --git a/checkpoint/checkpoint.go b/checkpoint/checkpoint.go index a6577e175..be7dd7385 100644 --- a/checkpoint/checkpoint.go +++ b/checkpoint/checkpoint.go @@ -45,50 +45,59 @@ type Data struct { Checksum uint64 } -// getPodEntries gets all Pod device allocation entries from checkpoint file -func getPodEntries() ([]PodDevicesEntry, error) { +type Checkpoint interface { + // GetComputeDeviceMap returns an instance of a map of ResourceInfo for a PodID + GetComputeDeviceMap(string) (map[string]*types.ResourceInfo, error) +} +type checkpoint struct { + fileName string + podEntires []PodDevicesEntry +} - podEntries := []PodDevicesEntry{} +// GetCheckpoint returns an instance of Checkpoint +func GetCheckpoint() (Checkpoint, error) { + logging.Debugf("GetCheckpoint(): invoked") + return getCheckpoint(checkPointfile) +} - cpd := &Data{} - rawBytes, err := ioutil.ReadFile(checkPointfile) +func getCheckpoint(filePath string) (Checkpoint, error) { + cp := &checkpoint{fileName: filePath} + err := cp.getPodEntries() if err != nil { - return podEntries, logging.Errorf("getPodEntries(): error reading file %s\n%v\n", checkPointfile, err) + return nil, err + } + logging.Debugf("getCheckpoint(): created checkpoint instance with file: %s", filePath) + return cp, nil +} + +// getPodEntries gets all Pod device allocation entries from checkpoint file +func (cp *checkpoint) getPodEntries() error { + cpd := &Data{} + rawBytes, err := ioutil.ReadFile(cp.fileName) + if err != nil { + return logging.Errorf("getPodEntries(): error reading file %s\n%v\n", checkPointfile, err) } if err = json.Unmarshal(rawBytes, cpd); err != nil { - return podEntries, logging.Errorf("getPodEntries(): error unmarshalling raw bytes %v", err) + return logging.Errorf("getPodEntries(): error unmarshalling raw bytes %v", err) } - return cpd.Data.PodDeviceEntries, nil + cp.podEntires = cpd.Data.PodDeviceEntries + logging.Debugf("getPodEntries(): podEntires %+v", cp.podEntires) + return nil } -var instance map[string]*types.ResourceInfo - // GetComputeDeviceMap returns an instance of a map of ResourceInfo -func GetComputeDeviceMap(podID string) (map[string]*types.ResourceInfo, error) { - - if instance == nil { - if resourceMap, err := getResourceMapFromFile(podID); err == nil { - logging.Debugf("GetComputeDeviceMap(): created new instance of resourceMap for Pod: %s", podID) - instance = resourceMap - } else { - logging.Errorf("GetComputeDeviceMap(): error creating resourceMap instance %v", err) - return nil, err - } - } - logging.Debugf("GetComputeDeviceMap(): resourceMap instance: %+v", instance) - return instance, nil -} +func (cp *checkpoint) GetComputeDeviceMap(podID string) (map[string]*types.ResourceInfo, error) { -func getResourceMapFromFile(podID string) (map[string]*types.ResourceInfo, error) { resourceMap := make(map[string]*types.ResourceInfo) - podEntires, err := getPodEntries() - if err != nil { - return nil, err + + if podID == "" { + return nil, logging.Errorf("GetComputeDeviceMap(): invalid Pod cannot be empty") } - for _, pod := range podEntires { + + for _, pod := range cp.podEntires { if pod.PodUID == podID { entry, ok := resourceMap[pod.ResourceName] if ok { diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index f239a78f4..c07da2ba6 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -349,11 +349,19 @@ func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement deviceID := "" resourceName, ok := customResource.Metadata.Annotations[resourceNameAnnot] if ok && podID != "" { - logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName) // ResourceName annotation is found; try to get device info from resourceMap - resourceMap, err := checkpoint.GetComputeDeviceMap(podID) - if err != nil { - return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get resourceMap from kubelet checkpoint file: %v", err) + logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName) + + if resourceMap == nil { + checkpoint, err := checkpoint.GetCheckpoint() + if err != nil { + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get a checkpoint instance: %v", err) + } + resourceMap, err = checkpoint.GetComputeDeviceMap(podID) + if err != nil { + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get resourceMap from kubelet checkpoint file: %v", err) + } + logging.Debugf("getKubernetesDelegate(): resourceMap instance: %+v", resourceMap) } entry, ok := resourceMap[resourceName] @@ -493,18 +501,18 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) } // resourceMap holds Pod device allocation information; only initizized if CRD contains 'resourceName' annotation. - // This is only initialized once and all delegate objects can reference this to look up device info. + // This will only be initialized once and all delegate objects can reference this to look up device info. var resourceMap map[string]*types.ResourceInfo // Read all network objects referenced by 'networks' var delegates []*types.DelegateNetConf for _, net := range networks { - delegate, resourceMap, err := getKubernetesDelegate(k8sclient, net, confdir, podID, resourceMap) + delegate, updatedResourceMap, err := getKubernetesDelegate(k8sclient, net, confdir, podID, resourceMap) if err != nil { return nil, logging.Errorf("GetK8sNetwork: failed getting the delegate: %v", err) } delegates = append(delegates, delegate) - _ = resourceMap // workaround for 'Go' error: 'resourceMap' declared and not used. + resourceMap = updatedResourceMap } return delegates, nil From c03eb3d21d17b089bdf215c2573298ea9102f971 Mon Sep 17 00:00:00 2001 From: Abdul Halim Date: Wed, 19 Sep 2018 18:14:32 +0100 Subject: [PATCH 08/16] added checkpoint tests file Change-Id: I53551660ffd017fe170de58abdf7a96e29178000 --- checkpoint/checkpoint_test.go | 120 ++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 checkpoint/checkpoint_test.go diff --git a/checkpoint/checkpoint_test.go b/checkpoint/checkpoint_test.go new file mode 100644 index 000000000..7834a382e --- /dev/null +++ b/checkpoint/checkpoint_test.go @@ -0,0 +1,120 @@ +package checkpoint + +import ( + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "io/ioutil" + "testing" + + "github.com/intel/multus-cni/types" +) + +const ( + fakeTempFile = "/tmp/kubelet_internal_checkpoint" +) + +type fakeCheckpoint struct { + fileName string +} + +func (fc *fakeCheckpoint) WriteToFile(inBytes []byte) error { + return ioutil.WriteFile(fc.fileName, inBytes, 0600) +} + +func (fc *fakeCheckpoint) DeleteFile() error { + return os.Remove(fc.fileName) +} + +func TestCheckpoint(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Checkpoint") +} + +var _ = BeforeSuite(func() { + sampleData := `{ + "Data": { + "PodDeviceEntries": [ + { + "PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23", + "ContainerName": "appcntr1", + "ResourceName": "intel.com/sriov_net_A", + "DeviceIDs": [ + "0000:03:02.3", + "0000:03:02.0" + ], + "AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA==" + } + ], + "RegisteredDevices": { + "intel.com/sriov_net_A": [ + "0000:03:02.1", + "0000:03:02.2", + "0000:03:02.3", + "0000:03:02.0" + ], + "intel.com/sriov_net_B": [ + "0000:03:06.3", + "0000:03:06.0", + "0000:03:06.1", + "0000:03:06.2" + ] + } + }, + "Checksum": 229855270 + }` + + fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile} + err := fakeCheckpoint.WriteToFile([]byte(sampleData)) + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = Describe("Kubelet checkpoint data read operations", func() { + Context("Using /tmp/kubelet_internal_checkpoint file", func() { + var ( + cp Checkpoint + err error + resourceMap map[string]*types.ResourceInfo + resourceInfo *types.ResourceInfo + resourceAnnot = "intel.com/sriov_net_A" + ) + + It("should get a Checkpoint instance from file", func() { + cp, err = getCheckpoint(fakeTempFile) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should return a ResourceMap instance", func() { + rmap, err := cp.GetComputeDeviceMap("970a395d-bb3b-11e8-89df-408d5c537d23") + Expect(err).NotTo(HaveOccurred()) + Expect(rmap).NotTo(BeEmpty()) + resourceMap = rmap + }) + + It("resourceMap should have value for \"intel.com/sriov_net_A\"", func() { + rInfo, ok := resourceMap[resourceAnnot] + Expect(ok).To(BeTrue()) + resourceInfo = rInfo + }) + + It("should have 2 deviceIDs", func() { + Expect(len(resourceInfo.DeviceIDs)).To(BeEquivalentTo(2)) + }) + + It("should have \"0000:03:02.3\" in deviceIDs[0]", func() { + Expect(resourceInfo.DeviceIDs[0]).To(BeEquivalentTo("0000:03:02.3")) + }) + + It("should have \"0000:03:02.0\" in deviceIDs[1]", func() { + Expect(resourceInfo.DeviceIDs[1]).To(BeEquivalentTo("0000:03:02.0")) + }) + }) +}) + +var _ = AfterSuite(func() { + fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile} + err := fakeCheckpoint.DeleteFile() + Expect(err).NotTo(HaveOccurred()) +}) From e21ddf6cdd18885802dda2e65c89417dcb79714e Mon Sep 17 00:00:00 2001 From: Abdul Halim Date: Mon, 1 Oct 2018 10:38:56 +0100 Subject: [PATCH 09/16] fixed some typos in comments Change-Id: Ieb650479b6b0fef1a4ecaeb2c3c1a7c15fff43d5 --- k8sclient/k8sclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index c07da2ba6..fe5b548af 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -345,7 +345,7 @@ func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err) } - // Get resourceName annotation from NetDefinition + // Get resourceName annotation from NetworkAttachmentDefinition deviceID := "" resourceName, ok := customResource.Metadata.Annotations[resourceNameAnnot] if ok && podID != "" { From 925f919324f689c31b965399c45d3c3052927f2c Mon Sep 17 00:00:00 2001 From: Kuralamudhan Ramakrishnan Date: Wed, 10 Oct 2018 15:14:28 +0100 Subject: [PATCH 10/16] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 1c9bdf273..f51618e2f 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ * [Verifying Pod network interfaces](#verifying-pod-network-interfaces) * [Using with Multus conf file](#using-with-multus-conf-file) * [Logging Options](#logging-options) + * [How to use with Network Device plugins?](#cni-running-with-network-device-plugin) * [Testing Multus CNI](#testing-multus-cni) * [Multiple flannel networks](#multiple-flannel-networks) * [Configure Kubernetes with CNI](#configure-kubernetes-with-cni) @@ -491,7 +492,13 @@ You may configure the logging level by using the `LogLevel` option in your CNI c ``` "LogLevel": "debug", ``` +## CNI running with Network device plugin +Allocation of the Network device(such as SRIOV VFs) are done by Device plugins(Eg.SRIOV Network device plugin), Multus developed to work in the co-existence enviroment to work with device plugin by passing down the allocated device information to the CNI plugins. + +* [Device plugin & CNI, NUMA Manager alignment - technical architecture document](https://docs.google.com/document/d/1Ewe9Of84GkP0b2Q2PC0y9RVZNkN2WeVEagX9m99Nrzc/edit) +* Reference implementation : [SRIOV Network devie plugin](https://github.com/intel/sriov-network-device-plugin) +* Example: [How to make Multus work with device plugin?](https://github.com/intel/multus-cni/tree/master/examples#passing-down-device-information) ## Testing Multus CNI ### Multiple flannel networks From 6c49dc8a6e9fc97dc9e0c9ff30da4b95908c042e Mon Sep 17 00:00:00 2001 From: Kuralamudhan Ramakrishnan Date: Wed, 10 Oct 2018 15:29:58 +0100 Subject: [PATCH 11/16] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f51618e2f..8adf5a1a3 100644 --- a/README.md +++ b/README.md @@ -497,7 +497,7 @@ You may configure the logging level by using the `LogLevel` option in your CNI c Allocation of the Network device(such as SRIOV VFs) are done by Device plugins(Eg.SRIOV Network device plugin), Multus developed to work in the co-existence enviroment to work with device plugin by passing down the allocated device information to the CNI plugins. * [Device plugin & CNI, NUMA Manager alignment - technical architecture document](https://docs.google.com/document/d/1Ewe9Of84GkP0b2Q2PC0y9RVZNkN2WeVEagX9m99Nrzc/edit) -* Reference implementation : [SRIOV Network devie plugin](https://github.com/intel/sriov-network-device-plugin) +* Reference implementation : [SRIOV Network device plugin](https://github.com/intel/sriov-network-device-plugin) * Example: [How to make Multus work with device plugin?](https://github.com/intel/multus-cni/tree/master/examples#passing-down-device-information) ## Testing Multus CNI From 2a02eef418144712e6d5a89eaf29e00000f823e0 Mon Sep 17 00:00:00 2001 From: Doug Smith Date: Wed, 10 Oct 2018 14:09:38 -0400 Subject: [PATCH 12/16] Default network readiness [NOTE: Conflict due to omitted commit during cherry pick for stable] --- README.md | 16 ++++++++++++++++ multus/multus.go | 19 +++++++++++++++++++ multus/multus_test.go | 13 +++++++++++++ types/conf.go | 11 ++++++++--- types/conf_test.go | 36 ++++++++++++++++++++++++++++++++++++ types/types.go | 13 +++++++------ 6 files changed, 99 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8adf5a1a3..556eee517 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ * [Using with Multus conf file](#using-with-multus-conf-file) * [Logging Options](#logging-options) * [How to use with Network Device plugins?](#cni-running-with-network-device-plugin) + * [Default Network Readiness Checks](#default-network-readiness-checks) * [Testing Multus CNI](#testing-multus-cni) * [Multiple flannel networks](#multiple-flannel-networks) * [Configure Kubernetes with CNI](#configure-kubernetes-with-cni) @@ -499,6 +500,21 @@ Allocation of the Network device(such as SRIOV VFs) are done by Device plugins(E * [Device plugin & CNI, NUMA Manager alignment - technical architecture document](https://docs.google.com/document/d/1Ewe9Of84GkP0b2Q2PC0y9RVZNkN2WeVEagX9m99Nrzc/edit) * Reference implementation : [SRIOV Network device plugin](https://github.com/intel/sriov-network-device-plugin) * Example: [How to make Multus work with device plugin?](https://github.com/intel/multus-cni/tree/master/examples#passing-down-device-information) + +## Default Network Readiness Checks + +You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file. + +For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists. + +In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready. + +Only one option is necessary to configure this functionality: + +* `readinessindicatorfile`: The path to a file whose existance denotes that the default network is ready. + +*NOTE*: If `readinessindicatorfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default. + ## Testing Multus CNI ### Multiple flannel networks diff --git a/multus/multus.go b/multus/multus.go index 73f150159..f461674bf 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "path/filepath" + "time" "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/pkg/invoke" @@ -35,8 +36,16 @@ import ( "github.com/intel/multus-cni/logging" "github.com/intel/multus-cni/types" "github.com/vishvananda/netlink" + "k8s.io/apimachinery/pkg/util/wait" ) +var defaultReadinessBackoff = wait.Backoff{ + Steps: 4, + Duration: 250 * time.Millisecond, + Factor: 4.0, + Jitter: 0.1, +} + func saveScratchNetConf(containerID, dataDir string, netconf []byte) error { logging.Debugf("saveScratchNetConf: %s, %s, %s", containerID, dataDir, string(netconf)) if err := os.MkdirAll(dataDir, 0700); err != nil { @@ -227,6 +236,16 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn return nil, logging.Errorf("Multus: Err in getting k8s args: %v", err) } + wait.ExponentialBackoff(defaultReadinessBackoff, func() (bool, error) { + _, err := os.Stat(n.ReadinessIndicatorFile) + switch { + case err == nil: + return true, nil + default: + return false, nil + } + }) + numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, n, kubeClient) if err != nil { return nil, logging.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err) diff --git a/multus/multus_test.go b/multus/multus_test.go index 35cad0255..d05a0dbb1 100644 --- a/multus/multus_test.go +++ b/multus/multus_test.go @@ -173,6 +173,8 @@ var _ = Describe("multus operations", func() { StdinData: []byte(`{ "name": "node-cni-network", "type": "multus", + "defaultnetworkfile": "/tmp/foo.multus.conf", + "defaultnetworkwaitseconds": 3, "delegates": [{ "name": "weave1", "cniVersion": "0.2.0", @@ -185,6 +187,10 @@ var _ = Describe("multus operations", func() { }`), } + // Touch the default network file. + configPath := "/tmp/foo.multus.conf" + os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755) + fExec := &fakeExec{} expectedResult1 := &types020.Result{ CNIVersion: "0.2.0", @@ -226,6 +232,13 @@ var _ = Describe("multus operations", func() { err = cmdDel(args, fExec, nil) Expect(err).NotTo(HaveOccurred()) Expect(fExec.delIndex).To(Equal(len(fExec.plugins))) + + // Cleanup default network file. + if _, errStat := os.Stat(configPath); errStat == nil { + errRemove := os.Remove(configPath) + Expect(errRemove).NotTo(HaveOccurred()) + } + }) It("executes delegates and kubernetes networks", func() { diff --git a/types/conf.go b/types/conf.go index bfbc13bce..ffc330ec4 100644 --- a/types/conf.go +++ b/types/conf.go @@ -27,9 +27,10 @@ import ( ) const ( - defaultCNIDir = "/var/lib/cni/multus" - defaultConfDir = "/etc/cni/multus/net.d" - defaultBinDir = "/opt/cni/bin" + defaultCNIDir = "/var/lib/cni/multus" + defaultConfDir = "/etc/cni/multus/net.d" + defaultBinDir = "/opt/cni/bin" + defaultReadinessIndicatorFile = "" ) func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error { @@ -194,6 +195,10 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { netconf.BinDir = defaultBinDir } + if netconf.ReadinessIndicatorFile == "" { + netconf.ReadinessIndicatorFile = defaultReadinessIndicatorFile + } + for idx, rawConf := range netconf.RawDelegates { bytes, err := json.Marshal(rawConf) if err != nil { diff --git a/types/conf_test.go b/types/conf_test.go index fa64a3ddb..b6775eef1 100644 --- a/types/conf_test.go +++ b/types/conf_test.go @@ -81,4 +81,40 @@ var _ = Describe("config operations", func() { _, err := LoadNetConf([]byte(conf)) Expect(err).To(HaveOccurred()) }) + + It("has defaults set for network readiness", func() { + conf := `{ + "name": "defaultnetwork", + "type": "multus", + "kubeconfig": "/etc/kubernetes/kubelet.conf", + "delegates": [{ + "cniVersion": "0.3.0", + "name": "defaultnetwork", + "type": "flannel", + "isDefaultGateway": true + }] +}` + netConf, err := LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + Expect(netConf.ReadinessIndicatorFile).To(Equal("")) + }) + + It("honors overrides for network readiness", func() { + conf := `{ + "name": "defaultnetwork", + "type": "multus", + "readinessindicatorfile": "/etc/cni/net.d/foo", + "kubeconfig": "/etc/kubernetes/kubelet.conf", + "delegates": [{ + "cniVersion": "0.3.0", + "name": "defaultnetwork", + "type": "flannel", + "isDefaultGateway": true + }] +}` + netConf, err := LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + Expect(netConf.ReadinessIndicatorFile).To(Equal("/etc/cni/net.d/foo")) + }) + }) diff --git a/types/types.go b/types/types.go index b0760280b..efe7d3cfd 100644 --- a/types/types.go +++ b/types/types.go @@ -36,12 +36,13 @@ type NetConf struct { CNIDir string `json:"cniDir"` BinDir string `json:"binDir"` // RawDelegates is private to the NetConf class; use Delegates instead - RawDelegates []map[string]interface{} `json:"delegates"` - Delegates []*DelegateNetConf `json:"-"` - NetStatus []*NetworkStatus `json:"-"` - Kubeconfig string `json:"kubeconfig"` - LogFile string `json:"logFile"` - LogLevel string `json:"logLevel"` + RawDelegates []map[string]interface{} `json:"delegates"` + Delegates []*DelegateNetConf `json:"-"` + NetStatus []*NetworkStatus `json:"-"` + Kubeconfig string `json:"kubeconfig"` + LogFile string `json:"logFile"` + LogLevel string `json:"logLevel"` + ReadinessIndicatorFile string `json:readinessindicatorfile` } type NetworkStatus struct { From 7050fc5aa750bff7a2f2199315978388e86d3045 Mon Sep 17 00:00:00 2001 From: Tomofumi Hayashi Date: Sat, 6 Oct 2018 20:06:20 +0900 Subject: [PATCH 13/16] TravisCI yaml parameterized This change fixes #143, to make some specific TravisCI args parameter. --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f117fff72..cebcd10b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,8 @@ language: go dist: trusty env: global: - - REGISTRY_USER=nfvperobot - - secure: "LnQV09sy5nfrJd0PKAbxYPdKJ5QtLECofsunYfVk7tFp+ivKyZBXHwi4V4aGFuB2SqCnpauXBRTLet8hrfm5kN9ZZQRqy0WNs/fJHdFC6YKOKwyCQwczFb1by/iTX68dxWc2nK9+Opi6s/81Bh5yb3Oquqzdk+OEgaQHz2KP7BwI4yDrobinBR5laJ4KdxZJYgYx4mP6uUPxj7UZww+HaWqyiGy8cAeK3L81sGjxXJIYTRRfG1J4pifI5A3c3IOJRID0pvifgUIsQXp5MHpx+nxmhRJ7KMBLeNkUKruLTEsufgGCvhY5eWpdBhVN2YefGTqlKBCtKEqRUPlLbP5eJGUdY1PlUMUnQsr+FRWAZz90A1TESOZXZqDs4xR1ox1wX7mBUeelViXvUfLQB9sOD8G86FkXqNTqx/thp3x0Dqgy44pL+12Y3k5xVZmIsWDSpGmmIe1jOCsoL26Fdic+dTO/l3mx3KP1+gPNqbScuJsccLyPsr96uFCBCPJ2mSy7nCqb01KZTbbkIvv6oOCQ+Mfq8MT9lkxf6FJ+K+7vVbcgshOGhqA/l1UO3rKxnGt8Rkj/5XoHkcjXjM6YzT5LvljVWszJGXeTQxGjcsPrK2AscyX7JvNp/AMElII/Hxm6P0NESfV0whrZHyVOaqIRrbhUsK9j4YP8IMFoI4qYp4g=" + - REGISTRY_USER=${REGISTRY_USER} + - secure: "${REGISTRY_SECURE}" before_install: - sudo apt-get update -qq @@ -42,7 +42,7 @@ before_deploy: deploy: - provider: releases api_key: - secure: "iy7eqzXNvb/juc+5eVPQ/pFYDTCqDt8Zjt63n+zEK856Qzr2aEZwwOguMWs78XFDMFXagCs5PRTvtvZz8apoTfHX7Wkss3kRyEziAkuldQbH5yGDvpGyHsGBw78N95hauMoogefE7NuuLG3qRSWPeVz8RAKGhP7ADwEVyyfQKKYdum3Bqrz0D89HqKbCQqs3eZae7ppDIler3lab9WAQGuKNJ2HL6mqREVe48kb8sdsuSr+yV4qwVrBDNhXxQDxAT6LYuMXbknE7qTde2vViP13ZHpptbuZqiZG2ytzReIIs/iC9AWoIQXr3XTXl9z8fqlC3VljPCikBWVcmxDFA2aANYzx3M/7fMOO/DniwNhlZc9+pYfAkUrpoQPfPOWNqf45Qz0jP3wk49xy5hxEqe/rfmo5lipSsqeUsk+j3pT8kjVIAnDLrQpxSx7xwnijPLgtm34UwROVowfwLlOhE/7mUOFCbYlzEo3CKvjDN3Kmn35yHEueuu//Gv5jesVYvgcNPBHqaTKb5AXVTqymNBtA43PchLJ8gCC1mNukzSZifQP996vzbV5c9AxzBLjWbiDJ3lOFIpNhF8Sed0m0C0RylrTXHTX5TSrlMdXXffzYwbjJ96J+cFPBTpJNfSn+3N7hiart1r1k1bSXoPqYW4+94M8E1eZ5LjszoeiZbRrI=" + secure: "${DEPLOY_SECURE}" file_glob: true file: "$TRAVIS_BUILD_DIR/dist/*/*.gz" skip_cleanup: true From fa99eab7ed07cab25bd765996f525aba7c7080b7 Mon Sep 17 00:00:00 2001 From: Tomofumi Hayashi Date: Mon, 15 Oct 2018 16:04:16 +0900 Subject: [PATCH 14/16] Fix TravisCI for the failure of 'go get golint' --- .travis.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index cebcd10b2..34d241d20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,12 +13,7 @@ before_install: - go get github.com/mattn/goveralls install: - # workaround golint install error in https://github.com/golang/lint/issues/288 - - mkdir -p $GOPATH/src/golang.org/x - - pushd $GOPATH/src/golang.org/x - - git clone https://github.com/golang/lint.git - - go get github.com/golang/lint/golint - - popd + - go get -u golang.org/x/lint/golint before_script: - golint ./multus/... | grep -v ALL_CAPS | xargs -r false From 22a0efc0c4175bed4834aece2386020bd9f48184 Mon Sep 17 00:00:00 2001 From: Shahar Klein Date: Thu, 11 Oct 2018 10:01:41 -0700 Subject: [PATCH 15/16] Seems like the ENTRYPOINT value must be quoted Signed-off-by: Shahar Klein --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 91f66992a..dbecd6869 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,4 +24,4 @@ ADD ./images/entrypoint.sh / # does it require a root user? # USER 1001 -ENTRYPOINT [/entrypoint.sh] +ENTRYPOINT ["/entrypoint.sh"] From 55ee3385f4fff8dd36629353d1f1e4360dd8b900 Mon Sep 17 00:00:00 2001 From: dougbtv Date: Thu, 18 Oct 2018 11:15:58 -0400 Subject: [PATCH 16/16] [bugfix][divergence] Diverges from master in requiring change to logging.Debugf value changed from string --- types/conf.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/types/conf.go b/types/conf.go index ffc330ec4..5073aa580 100644 --- a/types/conf.go +++ b/types/conf.go @@ -27,9 +27,9 @@ import ( ) const ( - defaultCNIDir = "/var/lib/cni/multus" - defaultConfDir = "/etc/cni/multus/net.d" - defaultBinDir = "/opt/cni/bin" + defaultCNIDir = "/var/lib/cni/multus" + defaultConfDir = "/etc/cni/multus/net.d" + defaultBinDir = "/opt/cni/bin" defaultReadinessIndicatorFile = "" ) @@ -104,7 +104,7 @@ func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string) (*l } func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*NetworkStatus, error) { - logging.Debugf("LoadNetworkStatus: %v, %s, %s", r, netName, defaultNet) + logging.Debugf("LoadNetworkStatus: %v, %s, %v", r, netName, defaultNet) // Convert whatever the IPAM result was into the current Result type result, err := current.NewResultFromResult(r)