diff --git a/.travis.yml b/.travis.yml index f117fff72..34d241d20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,20 +5,15 @@ language: go dist: trusty env: global: - - REGISTRY_USER=nfvperobot - - secure: "LnQV09sy5nfrJd0PKAbxYPdKJ5QtLECofsunYfVk7tFp+ivKyZBXHwi4V4aGFuB2SqCnpauXBRTLet8hrfm5kN9ZZQRqy0WNs/fJHdFC6YKOKwyCQwczFb1by/iTX68dxWc2nK9+Opi6s/81Bh5yb3Oquqzdk+OEgaQHz2KP7BwI4yDrobinBR5laJ4KdxZJYgYx4mP6uUPxj7UZww+HaWqyiGy8cAeK3L81sGjxXJIYTRRfG1J4pifI5A3c3IOJRID0pvifgUIsQXp5MHpx+nxmhRJ7KMBLeNkUKruLTEsufgGCvhY5eWpdBhVN2YefGTqlKBCtKEqRUPlLbP5eJGUdY1PlUMUnQsr+FRWAZz90A1TESOZXZqDs4xR1ox1wX7mBUeelViXvUfLQB9sOD8G86FkXqNTqx/thp3x0Dqgy44pL+12Y3k5xVZmIsWDSpGmmIe1jOCsoL26Fdic+dTO/l3mx3KP1+gPNqbScuJsccLyPsr96uFCBCPJ2mSy7nCqb01KZTbbkIvv6oOCQ+Mfq8MT9lkxf6FJ+K+7vVbcgshOGhqA/l1UO3rKxnGt8Rkj/5XoHkcjXjM6YzT5LvljVWszJGXeTQxGjcsPrK2AscyX7JvNp/AMElII/Hxm6P0NESfV0whrZHyVOaqIRrbhUsK9j4YP8IMFoI4qYp4g=" + - REGISTRY_USER=${REGISTRY_USER} + - secure: "${REGISTRY_SECURE}" before_install: - sudo apt-get update -qq - go get github.com/mattn/goveralls install: - # workaround golint install error in https://github.com/golang/lint/issues/288 - - mkdir -p $GOPATH/src/golang.org/x - - pushd $GOPATH/src/golang.org/x - - git clone https://github.com/golang/lint.git - - go get github.com/golang/lint/golint - - popd + - go get -u golang.org/x/lint/golint before_script: - golint ./multus/... | grep -v ALL_CAPS | xargs -r false @@ -42,7 +37,7 @@ before_deploy: deploy: - provider: releases api_key: - secure: "iy7eqzXNvb/juc+5eVPQ/pFYDTCqDt8Zjt63n+zEK856Qzr2aEZwwOguMWs78XFDMFXagCs5PRTvtvZz8apoTfHX7Wkss3kRyEziAkuldQbH5yGDvpGyHsGBw78N95hauMoogefE7NuuLG3qRSWPeVz8RAKGhP7ADwEVyyfQKKYdum3Bqrz0D89HqKbCQqs3eZae7ppDIler3lab9WAQGuKNJ2HL6mqREVe48kb8sdsuSr+yV4qwVrBDNhXxQDxAT6LYuMXbknE7qTde2vViP13ZHpptbuZqiZG2ytzReIIs/iC9AWoIQXr3XTXl9z8fqlC3VljPCikBWVcmxDFA2aANYzx3M/7fMOO/DniwNhlZc9+pYfAkUrpoQPfPOWNqf45Qz0jP3wk49xy5hxEqe/rfmo5lipSsqeUsk+j3pT8kjVIAnDLrQpxSx7xwnijPLgtm34UwROVowfwLlOhE/7mUOFCbYlzEo3CKvjDN3Kmn35yHEueuu//Gv5jesVYvgcNPBHqaTKb5AXVTqymNBtA43PchLJ8gCC1mNukzSZifQP996vzbV5c9AxzBLjWbiDJ3lOFIpNhF8Sed0m0C0RylrTXHTX5TSrlMdXXffzYwbjJ96J+cFPBTpJNfSn+3N7hiart1r1k1bSXoPqYW4+94M8E1eZ5LjszoeiZbRrI=" + secure: "${DEPLOY_SECURE}" file_glob: true file: "$TRAVIS_BUILD_DIR/dist/*/*.gz" skip_cleanup: true diff --git a/Dockerfile b/Dockerfile index 16f6cd236..dbecd6869 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,4 +24,4 @@ ADD ./images/entrypoint.sh / # does it require a root user? # USER 1001 -ENTRYPOINT /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/README.md b/README.md index 1c9bdf273..556eee517 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ * [Verifying Pod network interfaces](#verifying-pod-network-interfaces) * [Using with Multus conf file](#using-with-multus-conf-file) * [Logging Options](#logging-options) + * [How to use with Network Device plugins?](#cni-running-with-network-device-plugin) + * [Default Network Readiness Checks](#default-network-readiness-checks) * [Testing Multus CNI](#testing-multus-cni) * [Multiple flannel networks](#multiple-flannel-networks) * [Configure Kubernetes with CNI](#configure-kubernetes-with-cni) @@ -491,6 +493,27 @@ You may configure the logging level by using the `LogLevel` option in your CNI c ``` "LogLevel": "debug", ``` +## CNI running with Network device plugin + +Allocation of the Network device(such as SRIOV VFs) are done by Device plugins(Eg.SRIOV Network device plugin), Multus developed to work in the co-existence enviroment to work with device plugin by passing down the allocated device information to the CNI plugins. + +* [Device plugin & CNI, NUMA Manager alignment - technical architecture document](https://docs.google.com/document/d/1Ewe9Of84GkP0b2Q2PC0y9RVZNkN2WeVEagX9m99Nrzc/edit) +* Reference implementation : [SRIOV Network device plugin](https://github.com/intel/sriov-network-device-plugin) +* Example: [How to make Multus work with device plugin?](https://github.com/intel/multus-cni/tree/master/examples#passing-down-device-information) + +## Default Network Readiness Checks + +You may wish for your "default network" (that is, the CNI plugin & its configuration you specify as your default delegate) to become ready before you attach networks with Multus. This is disabled by default and not used unless you add the readiness check option(s) to your CNI configuration file. + +For example, if you use Flannel as a default network, the recommended method for Flannel to be installed is via a daemonset that also drops a configuration file in `/etc/cni/net.d/`. This may apply to other plugins that place that configuration file upon their readiness, hence, Multus uses their configuration filename as a semaphore and optionally waits to attach networks to pods until that file exists. + +In this manner, you may prevent pods from crash looping, and instead wait for that default network to be ready. + +Only one option is necessary to configure this functionality: + +* `readinessindicatorfile`: The path to a file whose existance denotes that the default network is ready. + +*NOTE*: If `readinessindicatorfile` is unset, or is an empty string, this functionality will be disabled, and is disabled by default. ## Testing Multus CNI diff --git a/checkpoint/checkpoint.go b/checkpoint/checkpoint.go new file mode 100644 index 000000000..be7dd7385 --- /dev/null +++ b/checkpoint/checkpoint.go @@ -0,0 +1,113 @@ +// Copyright (c) 2018 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package checkpoint + +import ( + "encoding/json" + "io/ioutil" + + "github.com/intel/multus-cni/logging" + "github.com/intel/multus-cni/types" +) + +const ( + checkPointfile = "/var/lib/kubelet/device-plugins/kubelet_internal_checkpoint" +) + +type PodDevicesEntry struct { + PodUID string + ContainerName string + ResourceName string + DeviceIDs []string + AllocResp []byte +} + +type checkpointData struct { + PodDeviceEntries []PodDevicesEntry + RegisteredDevices map[string][]string +} + +type Data struct { + Data checkpointData + Checksum uint64 +} + +type Checkpoint interface { + // GetComputeDeviceMap returns an instance of a map of ResourceInfo for a PodID + GetComputeDeviceMap(string) (map[string]*types.ResourceInfo, error) +} +type checkpoint struct { + fileName string + podEntires []PodDevicesEntry +} + +// GetCheckpoint returns an instance of Checkpoint +func GetCheckpoint() (Checkpoint, error) { + logging.Debugf("GetCheckpoint(): invoked") + return getCheckpoint(checkPointfile) +} + +func getCheckpoint(filePath string) (Checkpoint, error) { + cp := &checkpoint{fileName: filePath} + err := cp.getPodEntries() + if err != nil { + return nil, err + } + logging.Debugf("getCheckpoint(): created checkpoint instance with file: %s", filePath) + return cp, nil +} + +// getPodEntries gets all Pod device allocation entries from checkpoint file +func (cp *checkpoint) getPodEntries() error { + + cpd := &Data{} + rawBytes, err := ioutil.ReadFile(cp.fileName) + if err != nil { + return logging.Errorf("getPodEntries(): error reading file %s\n%v\n", checkPointfile, err) + } + + if err = json.Unmarshal(rawBytes, cpd); err != nil { + return logging.Errorf("getPodEntries(): error unmarshalling raw bytes %v", err) + } + + cp.podEntires = cpd.Data.PodDeviceEntries + logging.Debugf("getPodEntries(): podEntires %+v", cp.podEntires) + return nil +} + +// GetComputeDeviceMap returns an instance of a map of ResourceInfo +func (cp *checkpoint) GetComputeDeviceMap(podID string) (map[string]*types.ResourceInfo, error) { + + resourceMap := make(map[string]*types.ResourceInfo) + + if podID == "" { + return nil, logging.Errorf("GetComputeDeviceMap(): invalid Pod cannot be empty") + } + + for _, pod := range cp.podEntires { + if pod.PodUID == podID { + entry, ok := resourceMap[pod.ResourceName] + if ok { + // already exists; append to it + entry.DeviceIDs = append(entry.DeviceIDs, pod.DeviceIDs...) + } else { + // new entry + resourceMap[pod.ResourceName] = &types.ResourceInfo{DeviceIDs: pod.DeviceIDs} + } + } + } + return resourceMap, nil +} diff --git a/checkpoint/checkpoint_test.go b/checkpoint/checkpoint_test.go new file mode 100644 index 000000000..7834a382e --- /dev/null +++ b/checkpoint/checkpoint_test.go @@ -0,0 +1,120 @@ +package checkpoint + +import ( + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "io/ioutil" + "testing" + + "github.com/intel/multus-cni/types" +) + +const ( + fakeTempFile = "/tmp/kubelet_internal_checkpoint" +) + +type fakeCheckpoint struct { + fileName string +} + +func (fc *fakeCheckpoint) WriteToFile(inBytes []byte) error { + return ioutil.WriteFile(fc.fileName, inBytes, 0600) +} + +func (fc *fakeCheckpoint) DeleteFile() error { + return os.Remove(fc.fileName) +} + +func TestCheckpoint(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Checkpoint") +} + +var _ = BeforeSuite(func() { + sampleData := `{ + "Data": { + "PodDeviceEntries": [ + { + "PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23", + "ContainerName": "appcntr1", + "ResourceName": "intel.com/sriov_net_A", + "DeviceIDs": [ + "0000:03:02.3", + "0000:03:02.0" + ], + "AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA==" + } + ], + "RegisteredDevices": { + "intel.com/sriov_net_A": [ + "0000:03:02.1", + "0000:03:02.2", + "0000:03:02.3", + "0000:03:02.0" + ], + "intel.com/sriov_net_B": [ + "0000:03:06.3", + "0000:03:06.0", + "0000:03:06.1", + "0000:03:06.2" + ] + } + }, + "Checksum": 229855270 + }` + + fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile} + err := fakeCheckpoint.WriteToFile([]byte(sampleData)) + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = Describe("Kubelet checkpoint data read operations", func() { + Context("Using /tmp/kubelet_internal_checkpoint file", func() { + var ( + cp Checkpoint + err error + resourceMap map[string]*types.ResourceInfo + resourceInfo *types.ResourceInfo + resourceAnnot = "intel.com/sriov_net_A" + ) + + It("should get a Checkpoint instance from file", func() { + cp, err = getCheckpoint(fakeTempFile) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should return a ResourceMap instance", func() { + rmap, err := cp.GetComputeDeviceMap("970a395d-bb3b-11e8-89df-408d5c537d23") + Expect(err).NotTo(HaveOccurred()) + Expect(rmap).NotTo(BeEmpty()) + resourceMap = rmap + }) + + It("resourceMap should have value for \"intel.com/sriov_net_A\"", func() { + rInfo, ok := resourceMap[resourceAnnot] + Expect(ok).To(BeTrue()) + resourceInfo = rInfo + }) + + It("should have 2 deviceIDs", func() { + Expect(len(resourceInfo.DeviceIDs)).To(BeEquivalentTo(2)) + }) + + It("should have \"0000:03:02.3\" in deviceIDs[0]", func() { + Expect(resourceInfo.DeviceIDs[0]).To(BeEquivalentTo("0000:03:02.3")) + }) + + It("should have \"0000:03:02.0\" in deviceIDs[1]", func() { + Expect(resourceInfo.DeviceIDs[1]).To(BeEquivalentTo("0000:03:02.0")) + }) + }) +}) + +var _ = AfterSuite(func() { + fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile} + err := fakeCheckpoint.DeleteFile() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/examples/README.md b/examples/README.md index ba5906e6b..17e0fb178 100644 --- a/examples/README.md +++ b/examples/README.md @@ -60,3 +60,35 @@ A sample `cni-configuration.conf` is provided, typically this file is placed in ## Other considerations Primarily in this setup one thing that one should consider are the aspects of the `macvlan-conf.yml`, which is likely specific to the configuration of the node on which this resides. + +## Passing down device information +Some CNI plugins require specific device information which maybe pre-allocated by K8s device plugin. This could be indicated by providing `k8s.v1.cni.cncf.io/resourceName` annotaton in its network attachment definition CRD. The file [`examples/sriov-net.yaml`](./sriov-net.yaml) shows an example on how to define a Network attachment definition with specific device allocation information. Multus will get allocated device information and make them available for CNI plugin to work on. + +In this exmaple (shown below), it is expected that an [SRIOV Device Plugin](https://github.com/intel/sriov-network-device-plugin/) making a pool of SRIOV VFs available to the K8s with `intel.com/sriov` as their resourceName. Any device allocated from this resource pool will be passed down by Multus to the [sriov-cni](https://github.com/intel/sriov-cni/tree/dev/k8s-deviceid-model) plugin in `deviceID` field. This is up to the sriov-cni plugin to capture this information and work with this specific device information. + +```yaml +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net-a + annotations: + k8s.v1.cni.cncf.io/resourceName: intel.com/sriov +spec: + config: '{ + "type": "sriov", + "vlan": 1000, + "ipam": { + "type": "host-local", + "subnet": "10.56.217.0/24", + "rangeStart": "10.56.217.171", + "rangeEnd": "10.56.217.181", + "routes": [{ + "dst": "0.0.0.0/0" + }], + "gateway": "10.56.217.1" + } +}' +``` +The [net-resource-sample-pod.yaml](./net-resource-sample-pod.yaml) is an exmaple Pod manifest file that requesting a SRIOV device from a host which is then configured using the above network attachement definition. + +>For further information on how to configure SRIOV Device Plugin and SRIOV-CNI please refer to the links given above. \ No newline at end of file diff --git a/examples/net-resource-sample-pod.yaml b/examples/net-resource-sample-pod.yaml new file mode 100644 index 000000000..3f6eb0686 --- /dev/null +++ b/examples/net-resource-sample-pod.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: testpod1 + labels: + env: test + annotations: + k8s.v1.cni.cncf.io/networks: sriov-net-a +spec: + containers: + - name: appcntr1 + image: centos/tools + imagePullPolicy: IfNotPresent + command: [ "/bin/bash", "-c", "--" ] + args: [ "while true; do sleep 300000; done;" ] + resources: + requests: + intel.com/sriov: '1' + limits: + intel.com/sriov: '1' + restartPolicy: "Never" diff --git a/examples/sriov-net.yaml b/examples/sriov-net.yaml new file mode 100644 index 000000000..9b265d3e3 --- /dev/null +++ b/examples/sriov-net.yaml @@ -0,0 +1,21 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: sriov-net-a + annotations: + k8s.v1.cni.cncf.io/resourceName: intel.com/sriov +spec: + config: '{ + "type": "sriov", + "vlan": 1000, + "ipam": { + "type": "host-local", + "subnet": "10.56.217.0/24", + "rangeStart": "10.56.217.171", + "rangeEnd": "10.56.217.181", + "routes": [{ + "dst": "0.0.0.0/0" + }], + "gateway": "10.56.217.1" + } +}' diff --git a/images/README.md b/images/README.md index e16f4106f..706268365 100644 --- a/images/README.md +++ b/images/README.md @@ -31,10 +31,12 @@ You can get get help with the `--help` flag. ``` $ ./entrypoint.sh --help -This is an entrypoint script for Multus CNI to overlay its -binary and configuration into locations in a filesystem. -The configuration & binary file will be copied to the -corresponding configuration directory. +This is an entrypoint script for Multus CNI to overlay its binary and +configuration into locations in a filesystem. The configuration & binary file +will be copied to the corresponding configuration directory. When +`--multus-conf-file=auto` is used, 00-multus.conf will be automatically +generated from the CNI configuration file of the master plugin (the first file +in lexicographical order in cni-conf-dir). ./entrypoint.sh -h --help @@ -42,6 +44,7 @@ corresponding configuration directory. --cni-bin-dir=/host/opt/cni/bin --multus-conf-file=/usr/src/multus-cni/images/70-multus.conf --multus-bin-file=/usr/src/multus-cni/bin/multus + --multus-kubeconfig-file-host=/etc/cni/net.d/multus.d/multus.kubeconfig ``` You must use an `=` to delimit the parameter name and the value. For example you may set a custom `cni-conf-dir` like so: @@ -59,7 +62,7 @@ Note: You'll noticed that there's a `/host/...` directory from the root for the Example docker run command: ``` -$ docker run -it -v /opt/cni/bin/:/host/opt/cni/bin/ -v /etc/cni/net.d/:/host/etc/cni/net.d/ --entrypoint=/bin/bash dougbtv/multus +$ docker run -it -v /opt/cni/bin/:/host/opt/cni/bin/ -v /etc/cni/net.d/:/host/etc/cni/net.d/ --entrypoint=/bin/bash dougbtv/multus ``` -Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://github.com/projectcalico/calico/blob/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager/calico-daemonset.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153). \ No newline at end of file +Originally inspired by and is a portmanteau of the [Flannel daemonset](https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel.yml), the [Calico Daemonset](https://github.com/projectcalico/calico/blob/master/v2.0/getting-started/kubernetes/installation/hosted/k8s-backend-addon-manager/calico-daemonset.yaml), and the [Calico CNI install bash script](https://github.com/projectcalico/cni-plugin/blob/be4df4db2e47aa7378b1bdf6933724bac1f348d0/k8s-install/scripts/install-cni.sh#L104-L153). diff --git a/images/entrypoint.sh b/images/entrypoint.sh index 8b60bf032..679acaa0d 100755 --- a/images/entrypoint.sh +++ b/images/entrypoint.sh @@ -8,14 +8,17 @@ CNI_CONF_DIR="/host/etc/cni/net.d" CNI_BIN_DIR="/host/opt/cni/bin" MULTUS_CONF_FILE="/usr/src/multus-cni/images/70-multus.conf" MULTUS_BIN_FILE="/usr/src/multus-cni/bin/multus" +MULTUS_KUBECONFIG_FILE_HOST="/etc/cni/net.d/multus.d/multus.kubeconfig" # Give help text for parameters. function usage() { - echo -e "This is an entrypoint script for Multus CNI to overlay its" - echo -e "binary and configuration into locations in a filesystem." - echo -e "The configuration & binary file will be copied to the " - echo -e "corresponding configuration directory." + echo -e "This is an entrypoint script for Multus CNI to overlay its binary and " + echo -e "configuration into locations in a filesystem. The configuration & binary file " + echo -e "will be copied to the corresponding configuration directory. When " + echo -e "`--multus-conf-file=auto` is used, 00-multus.conf will be automatically " + echo -e "generated from the CNI configuration file of the master plugin (the first file " + echo -e "in lexicographical order in cni-conf-dir)." echo -e "" echo -e "./entrypoint.sh" echo -e "\t-h --help" @@ -23,6 +26,7 @@ function usage() echo -e "\t--cni-bin-dir=$CNI_BIN_DIR" echo -e "\t--multus-conf-file=$MULTUS_CONF_FILE" echo -e "\t--multus-bin-file=$MULTUS_BIN_FILE" + echo -e "\t--multus-kubeconfig-file-host=$MULTUS_KUBECONFIG_FILE_HOST" } # Parse parameters given as arguments to this script. @@ -46,6 +50,9 @@ while [ "$1" != "" ]; do --multus-bin-file) MULTUS_BIN_FILE=$VALUE ;; + --multus-kubeconfig-file-host) + MULTUS_KUBECONFIG_FILE_HOST=$VALUE + ;; *) echo "ERROR: unknown parameter \"$PARAM\"" usage @@ -57,7 +64,11 @@ done # Create array of known locations -declare -a arr=($CNI_CONF_DIR $CNI_BIN_DIR $MULTUS_CONF_FILE $MULTUS_BIN_FILE) +declare -a arr=($CNI_CONF_DIR $CNI_BIN_DIR $MULTUS_BIN_FILE) +if [ "$MULTUS_CONF_FILE" != "auto" ]; then + arr+=($MULTUS_BIN_FILE) +fi + # Loop through and verify each location each. for i in "${arr[@]}" @@ -69,8 +80,10 @@ do done # Copy files into proper places. -cp -f $MULTUS_CONF_FILE $CNI_CONF_DIR cp -f $MULTUS_BIN_FILE $CNI_BIN_DIR +if [ "$MULTUS_CONF_FILE" != "auto" ]; then + cp -f $MULTUS_CONF_FILE $CNI_CONF_DIR +fi # Make a multus.d directory (for our kubeconfig) @@ -134,6 +147,35 @@ fi # ---------------------- end Generate a "kube-config". +# ------------------------------- Generate "00-multus.conf" + +if [ "$MULTUS_CONF_FILE" == "auto" ]; then + echo "Generating Multus configuration file ..." + MASTER_PLUGIN="$(ls $CNI_CONF_DIR | grep -E '\.conf(list)?$' | head -1)" + if [ "$MASTER_PLUGIN" == "" ]; then + echo "Error: Multus could not be configured: no master plugin was found." + exit 1; + elif [ "$MASTER_PLUGIN" == "00-multus.conf" ]; then + echo "Warning: Multus is already configured: auto configuration skipped." + else + MASTER_PLUGIN_JSON="$(cat $CNI_CONF_DIR/$MASTER_PLUGIN)" + CONF=$(cat <<-EOF + { + "name": "multus-cni-network", + "type": "multus", + "kubeconfig": "$MULTUS_KUBECONFIG_FILE_HOST", + "delegates": [ + $MASTER_PLUGIN_JSON + ] + } + EOF + ) + echo $CONF > $CNI_CONF_DIR/00-multus.conf + fi +fi + +# ---------------------- end Generate "00-multus.conf". + echo "Entering sleep... (success)" # Sleep forever. diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index 8e1ee6433..fe5b548af 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -31,10 +31,15 @@ import ( "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/pkg/skel" cnitypes "github.com/containernetworking/cni/pkg/types" + "github.com/intel/multus-cni/checkpoint" "github.com/intel/multus-cni/logging" "github.com/intel/multus-cni/types" ) +const ( + resourceNameAnnot = "k8s.v1.cni.cncf.io/resourceName" +) + // NoK8sNetworkError indicates error, no network in kubernetes type NoK8sNetworkError struct { message string @@ -131,16 +136,16 @@ func setPodNetworkAnnotation(client KubeClient, namespace string, pod *v1.Pod, n return pod, nil } -func getPodNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, string, error) { +func getPodNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, string, string, error) { var err error logging.Debugf("getPodNetworkAnnotation: %v, %v", client, k8sArgs) pod, err := client.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) if err != nil { - return "", "", logging.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err) + return "", "", "", logging.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err) } - return pod.Annotations["k8s.v1.cni.cncf.io/networks"], pod.ObjectMeta.Namespace, nil + return pod.Annotations["k8s.v1.cni.cncf.io/networks"], pod.ObjectMeta.Namespace, string(pod.UID), nil } func parsePodNetworkObjectName(podnetwork string) (string, string, string, error) { @@ -326,30 +331,60 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit return config, nil } -func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string) (*types.DelegateNetConf, error) { +func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string, podID string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) { + logging.Debugf("getKubernetesDelegate: %v, %v, %s", client, net, confdir) rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", net.Namespace, net.Name) netData, err := client.GetRawWithPath(rawPath) if err != nil { - return nil, logging.Errorf("getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err) + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err) } customResource := &types.NetworkAttachmentDefinition{} if err := json.Unmarshal(netData, customResource); err != nil { - return nil, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err) + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err) + } + + // Get resourceName annotation from NetworkAttachmentDefinition + deviceID := "" + resourceName, ok := customResource.Metadata.Annotations[resourceNameAnnot] + if ok && podID != "" { + // ResourceName annotation is found; try to get device info from resourceMap + logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName) + + if resourceMap == nil { + checkpoint, err := checkpoint.GetCheckpoint() + if err != nil { + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get a checkpoint instance: %v", err) + } + resourceMap, err = checkpoint.GetComputeDeviceMap(podID) + if err != nil { + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get resourceMap from kubelet checkpoint file: %v", err) + } + logging.Debugf("getKubernetesDelegate(): resourceMap instance: %+v", resourceMap) + } + + entry, ok := resourceMap[resourceName] + if ok { + if idCount := len(entry.DeviceIDs); idCount > 0 && idCount > entry.Index { + deviceID = entry.DeviceIDs[entry.Index] + logging.Debugf("getKubernetesDelegate: podID: %s deviceID: %s", podID, deviceID) + entry.Index++ // increment Index for next delegate + } + } } configBytes, err := cniConfigFromNetworkResource(customResource, confdir) if err != nil { - return nil, err + return nil, resourceMap, err } - delegate, err := types.LoadDelegateNetConf(configBytes, net.InterfaceRequest) + delegate, err := types.LoadDelegateNetConf(configBytes, net.InterfaceRequest, deviceID) if err != nil { - return nil, err + return nil, resourceMap, err } - return delegate, nil + return delegate, resourceMap, nil } type KubeClient interface { @@ -447,11 +482,15 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error) func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) ([]*types.DelegateNetConf, error) { logging.Debugf("GetK8sNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir) - netAnnot, defaultNamespace, err := getPodNetworkAnnotation(k8sclient, k8sArgs) + netAnnot, defaultNamespace, podID, err := getPodNetworkAnnotation(k8sclient, k8sArgs) if err != nil { return nil, err } + if err != nil { + return nil, logging.Errorf("GetK8sNetwork: failed to get resourceMap for PodUID: %v %v", podID, err) + } + if len(netAnnot) == 0 { return nil, &NoK8sNetworkError{"no kubernetes network found"} } @@ -461,14 +500,19 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) return nil, err } + // resourceMap holds Pod device allocation information; only initizized if CRD contains 'resourceName' annotation. + // This will only be initialized once and all delegate objects can reference this to look up device info. + var resourceMap map[string]*types.ResourceInfo + // Read all network objects referenced by 'networks' var delegates []*types.DelegateNetConf for _, net := range networks { - delegate, err := getKubernetesDelegate(k8sclient, net, confdir) + delegate, updatedResourceMap, err := getKubernetesDelegate(k8sclient, net, confdir, podID, resourceMap) if err != nil { return nil, logging.Errorf("GetK8sNetwork: failed getting the delegate: %v", err) } delegates = append(delegates, delegate) + resourceMap = updatedResourceMap } return delegates, nil diff --git a/multus/multus.go b/multus/multus.go index 73f150159..f461674bf 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "path/filepath" + "time" "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/pkg/invoke" @@ -35,8 +36,16 @@ import ( "github.com/intel/multus-cni/logging" "github.com/intel/multus-cni/types" "github.com/vishvananda/netlink" + "k8s.io/apimachinery/pkg/util/wait" ) +var defaultReadinessBackoff = wait.Backoff{ + Steps: 4, + Duration: 250 * time.Millisecond, + Factor: 4.0, + Jitter: 0.1, +} + func saveScratchNetConf(containerID, dataDir string, netconf []byte) error { logging.Debugf("saveScratchNetConf: %s, %s, %s", containerID, dataDir, string(netconf)) if err := os.MkdirAll(dataDir, 0700); err != nil { @@ -227,6 +236,16 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn return nil, logging.Errorf("Multus: Err in getting k8s args: %v", err) } + wait.ExponentialBackoff(defaultReadinessBackoff, func() (bool, error) { + _, err := os.Stat(n.ReadinessIndicatorFile) + switch { + case err == nil: + return true, nil + default: + return false, nil + } + }) + numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, n, kubeClient) if err != nil { return nil, logging.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err) diff --git a/multus/multus_test.go b/multus/multus_test.go index 35cad0255..d05a0dbb1 100644 --- a/multus/multus_test.go +++ b/multus/multus_test.go @@ -173,6 +173,8 @@ var _ = Describe("multus operations", func() { StdinData: []byte(`{ "name": "node-cni-network", "type": "multus", + "defaultnetworkfile": "/tmp/foo.multus.conf", + "defaultnetworkwaitseconds": 3, "delegates": [{ "name": "weave1", "cniVersion": "0.2.0", @@ -185,6 +187,10 @@ var _ = Describe("multus operations", func() { }`), } + // Touch the default network file. + configPath := "/tmp/foo.multus.conf" + os.OpenFile(configPath, os.O_RDONLY|os.O_CREATE, 0755) + fExec := &fakeExec{} expectedResult1 := &types020.Result{ CNIVersion: "0.2.0", @@ -226,6 +232,13 @@ var _ = Describe("multus operations", func() { err = cmdDel(args, fExec, nil) Expect(err).NotTo(HaveOccurred()) Expect(fExec.delIndex).To(Equal(len(fExec.plugins))) + + // Cleanup default network file. + if _, errStat := os.Stat(configPath); errStat == nil { + errRemove := os.Remove(configPath) + Expect(errRemove).NotTo(HaveOccurred()) + } + }) It("executes delegates and kubernetes networks", func() { diff --git a/types/conf.go b/types/conf.go index e95d9cee2..5073aa580 100644 --- a/types/conf.go +++ b/types/conf.go @@ -27,9 +27,10 @@ import ( ) const ( - defaultCNIDir = "/var/lib/cni/multus" - defaultConfDir = "/etc/cni/multus/net.d" - defaultBinDir = "/opt/cni/bin" + defaultCNIDir = "/var/lib/cni/multus" + defaultConfDir = "/etc/cni/multus/net.d" + defaultBinDir = "/opt/cni/bin" + defaultReadinessIndicatorFile = "" ) func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error { @@ -50,7 +51,16 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error } // Convert raw CNI JSON into a DelegateNetConf structure -func LoadDelegateNetConf(bytes []byte, ifnameRequest string) (*DelegateNetConf, error) { +func LoadDelegateNetConf(bytes []byte, ifnameRequest, deviceID string) (*DelegateNetConf, error) { + // If deviceID is present, inject this into delegate config + if deviceID != "" { + if updatedBytes, err := delegateAddDeviceID(bytes, deviceID); err != nil { + return nil, logging.Errorf("error in LoadDelegateNetConf - delegateAddDeviceID unable to update delegate config: %v", err) + } else { + bytes = updatedBytes + } + } + delegateConf := &DelegateNetConf{} logging.Debugf("LoadDelegateNetConf: %s, %s", string(bytes), ifnameRequest) if err := json.Unmarshal(bytes, &delegateConf.Conf); err != nil { @@ -94,7 +104,7 @@ func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string) (*l } func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*NetworkStatus, error) { - logging.Debugf("LoadNetworkStatus: %v, %s, %s", r, netName, defaultNet) + logging.Debugf("LoadNetworkStatus: %v, %s, %v", r, netName, defaultNet) // Convert whatever the IPAM result was into the current Result type result, err := current.NewResultFromResult(r) @@ -185,12 +195,16 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { netconf.BinDir = defaultBinDir } + if netconf.ReadinessIndicatorFile == "" { + netconf.ReadinessIndicatorFile = defaultReadinessIndicatorFile + } + for idx, rawConf := range netconf.RawDelegates { bytes, err := json.Marshal(rawConf) if err != nil { return nil, logging.Errorf("error marshalling delegate %d config: %v", idx, err) } - delegateConf, err := LoadDelegateNetConf(bytes, "") + delegateConf, err := LoadDelegateNetConf(bytes, "", "") if err != nil { return nil, logging.Errorf("failed to load delegate %d config: %v", idx, err) } @@ -210,3 +224,21 @@ func (n *NetConf) AddDelegates(newDelegates []*DelegateNetConf) error { n.Delegates = append(n.Delegates, newDelegates...) return nil } + +// delegateAddDeviceID injects deviceID information in delegate bytes +func delegateAddDeviceID(inBytes []byte, deviceID string) ([]byte, error) { + var rawConfig map[string]interface{} + var err error + + err = json.Unmarshal(inBytes, &rawConfig) + if err != nil { + return nil, logging.Errorf("delegateAddDeviceID: failed to unmarshal inBytes: %v", err) + } + // Inject deviceID + rawConfig["deviceID"] = deviceID + configBytes, err := json.Marshal(rawConfig) + if err != nil { + return nil, logging.Errorf("delegateAddDeviceID: failed to re-marshal Spec.Config: %v", err) + } + return configBytes, nil +} diff --git a/types/conf_test.go b/types/conf_test.go index fa64a3ddb..b6775eef1 100644 --- a/types/conf_test.go +++ b/types/conf_test.go @@ -81,4 +81,40 @@ var _ = Describe("config operations", func() { _, err := LoadNetConf([]byte(conf)) Expect(err).To(HaveOccurred()) }) + + It("has defaults set for network readiness", func() { + conf := `{ + "name": "defaultnetwork", + "type": "multus", + "kubeconfig": "/etc/kubernetes/kubelet.conf", + "delegates": [{ + "cniVersion": "0.3.0", + "name": "defaultnetwork", + "type": "flannel", + "isDefaultGateway": true + }] +}` + netConf, err := LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + Expect(netConf.ReadinessIndicatorFile).To(Equal("")) + }) + + It("honors overrides for network readiness", func() { + conf := `{ + "name": "defaultnetwork", + "type": "multus", + "readinessindicatorfile": "/etc/cni/net.d/foo", + "kubeconfig": "/etc/kubernetes/kubelet.conf", + "delegates": [{ + "cniVersion": "0.3.0", + "name": "defaultnetwork", + "type": "flannel", + "isDefaultGateway": true + }] +}` + netConf, err := LoadNetConf([]byte(conf)) + Expect(err).NotTo(HaveOccurred()) + Expect(netConf.ReadinessIndicatorFile).To(Equal("/etc/cni/net.d/foo")) + }) + }) diff --git a/types/types.go b/types/types.go index 20ed055ea..efe7d3cfd 100644 --- a/types/types.go +++ b/types/types.go @@ -36,12 +36,13 @@ type NetConf struct { CNIDir string `json:"cniDir"` BinDir string `json:"binDir"` // RawDelegates is private to the NetConf class; use Delegates instead - RawDelegates []map[string]interface{} `json:"delegates"` - Delegates []*DelegateNetConf `json:"-"` - NetStatus []*NetworkStatus `json:"-"` - Kubeconfig string `json:"kubeconfig"` - LogFile string `json:"logFile"` - LogLevel string `json:"logLevel"` + RawDelegates []map[string]interface{} `json:"delegates"` + Delegates []*DelegateNetConf `json:"-"` + NetStatus []*NetworkStatus `json:"-"` + Kubeconfig string `json:"kubeconfig"` + LogFile string `json:"logFile"` + LogLevel string `json:"logLevel"` + ReadinessIndicatorFile string `json:readinessindicatorfile` } type NetworkStatus struct { @@ -119,3 +120,9 @@ type K8sArgs struct { K8S_POD_NAMESPACE types.UnmarshallableString K8S_POD_INFRA_CONTAINER_ID types.UnmarshallableString } + +// ResourceInfo is struct to hold Pod device allocation information +type ResourceInfo struct { + Index int + DeviceIDs []string +}