Updates to release-v3 stable branch

.travis.yml

@@ -5,20 +5,15 @@ language: go
 dist: trusty
 env:
   global:
-    - REGISTRY_USER=nfvperobot
-    - secure: "LnQV09sy5nfrJd0PKAbxYPdKJ5QtLECofsunYfVk7tFp+ivKyZBXHwi4V4aGFuB2SqCnpauXBRTLet8hrfm5kN9ZZQRqy0WNs/fJHdFC6YKOKwyCQwczFb1by/iTX68dxWc2nK9+Opi6s/81Bh5yb3Oquqzdk+OEgaQHz2KP7BwI4yDrobinBR5laJ4KdxZJYgYx4mP6uUPxj7UZww+HaWqyiGy8cAeK3L81sGjxXJIYTRRfG1J4pifI5A3c3IOJRID0pvifgUIsQXp5MHpx+nxmhRJ7KMBLeNkUKruLTEsufgGCvhY5eWpdBhVN2YefGTqlKBCtKEqRUPlLbP5eJGUdY1PlUMUnQsr+FRWAZz90A1TESOZXZqDs4xR1ox1wX7mBUeelViXvUfLQB9sOD8G86FkXqNTqx/thp3x0Dqgy44pL+12Y3k5xVZmIsWDSpGmmIe1jOCsoL26Fdic+dTO/l3mx3KP1+gPNqbScuJsccLyPsr96uFCBCPJ2mSy7nCqb01KZTbbkIvv6oOCQ+Mfq8MT9lkxf6FJ+K+7vVbcgshOGhqA/l1UO3rKxnGt8Rkj/5XoHkcjXjM6YzT5LvljVWszJGXeTQxGjcsPrK2AscyX7JvNp/AMElII/Hxm6P0NESfV0whrZHyVOaqIRrbhUsK9j4YP8IMFoI4qYp4g="
+    - REGISTRY_USER=${REGISTRY_USER}
+    - secure: "${REGISTRY_SECURE}"
 
 before_install:
   - sudo apt-get update -qq
   - go get github.com/mattn/goveralls
 
 install:
-  # workaround golint install error in https://github.com/golang/lint/issues/288
-  - mkdir -p $GOPATH/src/golang.org/x
-  - pushd $GOPATH/src/golang.org/x
-  - git clone https://github.com/golang/lint.git
-  - go get github.com/golang/lint/golint
-  - popd
+  - go get -u golang.org/x/lint/golint
 
 before_script:
   - golint ./multus/... | grep -v ALL_CAPS | xargs -r false
@@ -42,7 +37,7 @@ before_deploy:
 deploy:
   - provider: releases
     api_key:
-      secure: "iy7eqzXNvb/juc+5eVPQ/pFYDTCqDt8Zjt63n+zEK856Qzr2aEZwwOguMWs78XFDMFXagCs5PRTvtvZz8apoTfHX7Wkss3kRyEziAkuldQbH5yGDvpGyHsGBw78N95hauMoogefE7NuuLG3qRSWPeVz8RAKGhP7ADwEVyyfQKKYdum3Bqrz0D89HqKbCQqs3eZae7ppDIler3lab9WAQGuKNJ2HL6mqREVe48kb8sdsuSr+yV4qwVrBDNhXxQDxAT6LYuMXbknE7qTde2vViP13ZHpptbuZqiZG2ytzReIIs/iC9AWoIQXr3XTXl9z8fqlC3VljPCikBWVcmxDFA2aANYzx3M/7fMOO/DniwNhlZc9+pYfAkUrpoQPfPOWNqf45Qz0jP3wk49xy5hxEqe/rfmo5lipSsqeUsk+j3pT8kjVIAnDLrQpxSx7xwnijPLgtm34UwROVowfwLlOhE/7mUOFCbYlzEo3CKvjDN3Kmn35yHEueuu//Gv5jesVYvgcNPBHqaTKb5AXVTqymNBtA43PchLJ8gCC1mNukzSZifQP996vzbV5c9AxzBLjWbiDJ3lOFIpNhF8Sed0m0C0RylrTXHTX5TSrlMdXXffzYwbjJ96J+cFPBTpJNfSn+3N7hiart1r1k1bSXoPqYW4+94M8E1eZ5LjszoeiZbRrI="
+      secure: "${DEPLOY_SECURE}"
     file_glob: true
     file: "$TRAVIS_BUILD_DIR/dist/*/*.gz"
     skip_cleanup: true

Dockerfile

@@ -24,4 +24,4 @@ ADD ./images/entrypoint.sh /
 # does it require a root user?
 # USER 1001
 
-ENTRYPOINT /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

README.md

@@ -16,6 +16,8 @@
          * [Verifying Pod network interfaces](#verifying-pod-network-interfaces)
       * [Using with Multus conf file](#using-with-multus-conf-file)
       * [Logging Options](#logging-options)
+      * [How to use with Network Device plugins?](#cni-running-with-network-device-plugin)
+      * [Default Network Readiness Checks](#default-network-readiness-checks)
       * [Testing Multus CNI](#testing-multus-cni)
          * [Multiple flannel networks](#multiple-flannel-networks)
             * [Configure Kubernetes with CNI](#configure-kubernetes-with-cni)
@@ -491,6 +493,27 @@ You may configure the logging level by using the `LogLevel` option in your CNI c
 ```
     "LogLevel": "debug",
 ```
+## CNI running with Network device plugin
+
+Allocation of the Network device(such as SRIOV VFs) are done by Device plugins(Eg.SRIOV Network device plugin), Multus developed to work in the co-existence enviroment to work with device plugin by passing down the allocated device information to the CNI plugins.
+
+* [Device plugin & CNI, NUMA Manager alignment - technical architecture document](https://docs.google.com/document/d/1Ewe9Of84GkP0b2Q2PC0y9RVZNkN2WeVEagX9m99Nrzc/edit)
+* Reference implementation : [SRIOV Network device plugin](https://github.com/intel/sriov-network-device-plugin)
+* Example: [How to make Multus work with device plugin?](https://github.com/intel/multus-cni/tree/master/examples#passing-down-device-information)
+
+## Default Network Readiness Checks
+
+You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file.
+
+For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists.
+
+In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready.
+
+Only one option is necessary to configure this functionality:
+
+* `readinessindicatorfile`: The path to a file whose existance denotes that the default network is ready.
+
+*NOTE*: If `readinessindicatorfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default.
 
 ## Testing Multus CNI
 

checkpoint/checkpoint.go

@@ -0,0 +1,113 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package checkpoint
+
+import (
+	"encoding/json"
+	"io/ioutil"
+
+	"github.com/intel/multus-cni/logging"
+	"github.com/intel/multus-cni/types"
+)
+
+const (
+	checkPointfile = "/var/lib/kubelet/device-plugins/kubelet_internal_checkpoint"
+)
+
+type PodDevicesEntry struct {
+	PodUID        string
+	ContainerName string
+	ResourceName  string
+	DeviceIDs     []string
+	AllocResp     []byte
+}
+
+type checkpointData struct {
+	PodDeviceEntries  []PodDevicesEntry
+	RegisteredDevices map[string][]string
+}
+
+type Data struct {
+	Data     checkpointData
+	Checksum uint64
+}
+
+type Checkpoint interface {
+	// GetComputeDeviceMap returns an instance of a map of ResourceInfo for a PodID
+	GetComputeDeviceMap(string) (map[string]*types.ResourceInfo, error)
+}
+type checkpoint struct {
+	fileName   string
+	podEntires []PodDevicesEntry
+}
+
+// GetCheckpoint returns an instance of Checkpoint
+func GetCheckpoint() (Checkpoint, error) {
+	logging.Debugf("GetCheckpoint(): invoked")
+	return getCheckpoint(checkPointfile)
+}
+
+func getCheckpoint(filePath string) (Checkpoint, error) {
+	cp := &checkpoint{fileName: filePath}
+	err := cp.getPodEntries()
+	if err != nil {
+		return nil, err
+	}
+	logging.Debugf("getCheckpoint(): created checkpoint instance with file: %s", filePath)
+	return cp, nil
+}
+
+// getPodEntries gets all Pod device allocation entries from checkpoint file
+func (cp *checkpoint) getPodEntries() error {
+
+	cpd := &Data{}
+	rawBytes, err := ioutil.ReadFile(cp.fileName)
+	if err != nil {
+		return logging.Errorf("getPodEntries(): error reading file %s\n%v\n", checkPointfile, err)
+	}
+
+	if err = json.Unmarshal(rawBytes, cpd); err != nil {
+		return logging.Errorf("getPodEntries(): error unmarshalling raw bytes %v", err)
+	}
+
+	cp.podEntires = cpd.Data.PodDeviceEntries
+	logging.Debugf("getPodEntries(): podEntires %+v", cp.podEntires)
+	return nil
+}
+
+// GetComputeDeviceMap returns an instance of a map of ResourceInfo
+func (cp *checkpoint) GetComputeDeviceMap(podID string) (map[string]*types.ResourceInfo, error) {
+
+	resourceMap := make(map[string]*types.ResourceInfo)
+
+	if podID == "" {
+		return nil, logging.Errorf("GetComputeDeviceMap(): invalid Pod cannot be empty")
+	}
+
+	for _, pod := range cp.podEntires {
+		if pod.PodUID == podID {
+			entry, ok := resourceMap[pod.ResourceName]
+			if ok {
+				// already exists; append to it
+				entry.DeviceIDs = append(entry.DeviceIDs, pod.DeviceIDs...)
+			} else {
+				// new entry
+				resourceMap[pod.ResourceName] = &types.ResourceInfo{DeviceIDs: pod.DeviceIDs}
+			}
+		}
+	}
+	return resourceMap, nil
+}

checkpoint/checkpoint_test.go

@@ -0,0 +1,120 @@
+package checkpoint
+
+import (
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"io/ioutil"
+	"testing"
+
+	"github.com/intel/multus-cni/types"
+)
+
+const (
+	fakeTempFile = "/tmp/kubelet_internal_checkpoint"
+)
+
+type fakeCheckpoint struct {
+	fileName string
+}
+
+func (fc *fakeCheckpoint) WriteToFile(inBytes []byte) error {
+	return ioutil.WriteFile(fc.fileName, inBytes, 0600)
+}
+
+func (fc *fakeCheckpoint) DeleteFile() error {
+	return os.Remove(fc.fileName)
+}
+
+func TestCheckpoint(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Checkpoint")
+}
+
+var _ = BeforeSuite(func() {
+	sampleData := `{
+		"Data": {
+			"PodDeviceEntries": [
+			{
+				"PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23",
+				"ContainerName": "appcntr1",
+				"ResourceName": "intel.com/sriov_net_A",
+				"DeviceIDs": [
+				"0000:03:02.3",
+				"0000:03:02.0"
+				],
+				"AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA=="
+			}
+			],
+			"RegisteredDevices": {
+			"intel.com/sriov_net_A": [
+				"0000:03:02.1",
+				"0000:03:02.2",
+				"0000:03:02.3",
+				"0000:03:02.0"
+			],
+			"intel.com/sriov_net_B": [
+				"0000:03:06.3",
+				"0000:03:06.0",
+				"0000:03:06.1",
+				"0000:03:06.2"
+			]
+			}
+		},
+		"Checksum": 229855270
+		}`
+
+	fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile}
+	err := fakeCheckpoint.WriteToFile([]byte(sampleData))
+	Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = Describe("Kubelet checkpoint data read operations", func() {
+	Context("Using /tmp/kubelet_internal_checkpoint file", func() {
+		var (
+			cp            Checkpoint
+			err           error
+			resourceMap   map[string]*types.ResourceInfo
+			resourceInfo  *types.ResourceInfo
+			resourceAnnot = "intel.com/sriov_net_A"
+		)
+
+		It("should get a Checkpoint instance from file", func() {
+			cp, err = getCheckpoint(fakeTempFile)
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("should return a ResourceMap instance", func() {
+			rmap, err := cp.GetComputeDeviceMap("970a395d-bb3b-11e8-89df-408d5c537d23")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(rmap).NotTo(BeEmpty())
+			resourceMap = rmap
+		})
+
+		It("resourceMap should have value for \"intel.com/sriov_net_A\"", func() {
+			rInfo, ok := resourceMap[resourceAnnot]
+			Expect(ok).To(BeTrue())
+			resourceInfo = rInfo
+		})
+
+		It("should have 2 deviceIDs", func() {
+			Expect(len(resourceInfo.DeviceIDs)).To(BeEquivalentTo(2))
+		})
+
+		It("should have \"0000:03:02.3\" in deviceIDs[0]", func() {
+			Expect(resourceInfo.DeviceIDs[0]).To(BeEquivalentTo("0000:03:02.3"))
+		})
+
+		It("should have \"0000:03:02.0\" in deviceIDs[1]", func() {
+			Expect(resourceInfo.DeviceIDs[1]).To(BeEquivalentTo("0000:03:02.0"))
+		})
+	})
+})
+
+var _ = AfterSuite(func() {
+	fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile}
+	err := fakeCheckpoint.DeleteFile()
+	Expect(err).NotTo(HaveOccurred())
+})

examples/README.md

@@ -60,3 +60,35 @@ A sample `cni-configuration.conf` is provided, typically this file is placed in
 ## Other considerations
 
 Primarily in this setup one thing that one should consider are the aspects of the `macvlan-conf.yml`, which is likely specific to the configuration of the node on which this resides.
+
+## Passing down device information
+Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotaton in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on.
+
+In this exmaple (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information.
+
+```yaml
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+spec:
+  config: '{
+  "type": "sriov",
+  "vlan": 1000,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.56.217.0/24",
+    "rangeStart": "10.56.217.171",
+    "rangeEnd": "10.56.217.181",
+    "routes": [{
+      "dst": "0.0.0.0/0"
+    }],
+    "gateway": "10.56.217.1"
+  }
+}'
+```
+The [net-resource-sample-pod.yaml](./net-resource-sample-pod.yaml) is an exmaple Pod manifest file that requesting a SRIOV device from a host which is then configured using the above network attachement definition.
+
+>For further information on how to configure SRIOV Device Plugin and SRIOV-CNI please refer to the links given above.

examples/net-resource-sample-pod.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: testpod1
+  labels:
+    env: test
+  annotations:
+    k8s.v1.cni.cncf.io/networks: sriov-net-a
+spec:
+  containers:
+  - name: appcntr1
+    image: centos/tools
+    imagePullPolicy: IfNotPresent
+    command: [ "/bin/bash", "-c", "--" ]
+    args: [ "while true; do sleep 300000; done;" ]
+    resources:
+      requests:
+        intel.com/sriov: '1'
+      limits:
+        intel.com/sriov: '1'
+  restartPolicy: "Never"

examples/sriov-net.yaml

@@ -0,0 +1,21 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: sriov-net-a
+  annotations:
+    k8s.v1.cni.cncf.io/resourceName: intel.com/sriov
+spec:
+  config: '{
+  "type": "sriov",
+  "vlan": 1000,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "10.56.217.0/24",
+    "rangeStart": "10.56.217.171",
+    "rangeEnd": "10.56.217.181",
+    "routes": [{
+      "dst": "0.0.0.0/0"
+    }],
+    "gateway": "10.56.217.1"
+  }
+}'