diff --git a/.travis.yml b/.travis.yml index d5592a5da..f117fff72 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,12 +10,12 @@ env: before_install: - sudo apt-get update -qq + - go get github.com/mattn/goveralls install: # workaround golint install error in https://github.com/golang/lint/issues/288 - mkdir -p $GOPATH/src/golang.org/x - pushd $GOPATH/src/golang.org/x - - git clone https://github.com/golang/tools.git - git clone https://github.com/golang/lint.git - go get github.com/golang/lint/golint - popd @@ -29,9 +29,10 @@ before_script: script: - ./build - sudo ./test.sh + - $GOPATH/bin/goveralls -coverprofile=coverage.out -service=travis-ci - mkdir -p ${TRAVIS_BUILD_DIR}/dist - tar cvfz ${TRAVIS_BUILD_DIR}/dist/multus-cni_amd64.tar.gz --warning=no-file-changed --exclude="dist" --exclude="vendor" . - - docker build -t nfvpe/multus -f ./images/Dockerfile . + - docker build -t nfvpe/multus . before_deploy: - go get -u github.com/laher/goxc diff --git a/images/Dockerfile b/Dockerfile similarity index 100% rename from images/Dockerfile rename to Dockerfile diff --git a/README.md b/README.md index e9d773069..1a7fabbe5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ ![multus-cni Logo](https://github.com/intel/multus-cni/blob/master/doc/images/Multus.png) +[![Travis CI](https://travis-ci.org/intel/multus-cni.svg?branch=master)](https://travis-ci.org/intel/multus-cni/builds)[![Go Report Card](https://goreportcard.com/badge/github.com/intel/multus-cni)](https://goreportcard.com/report/github.com/intel/multus-cni) + * [MULTUS CNI plugin](#multus-cni-plugin) * [Quickstart Guide](#quickstart-guide) * [Multi-Homed pod](#multi-homed-pod) diff --git a/examples/npwg-demo-1/01_crd.yml b/examples/npwg-demo-1/01_crd.yml index cb1486560..7b74a5f04 100644 --- a/examples/npwg-demo-1/01_crd.yml +++ b/examples/npwg-demo-1/01_crd.yml @@ -2,21 +2,29 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: # name must match the spec fields below, and be in the form: . - name: networks.kubernetes.cni.cncf.io + name: network-attachment-definitions.k8s.cni.cncf.io spec: # group name to use for REST API: /apis// - group: kubernetes.cni.cncf.io + group: k8s.cni.cncf.io # version name to use for REST API: /apis// version: v1 # either Namespaced or Cluster scope: Namespaced names: # plural name to be used in the URL: /apis/// - plural: networks + plural: network-attachment-definitions # singular name to be used as an alias on the CLI and for display - singular: network + singular: network-attachment-definition # kind is normally the CamelCased singular type. Your resource manifests use this. - kind: Network + kind: NetworkAttachmentDefinition # shortNames allow shorter string to match your resource on the CLI shortNames: - - net + - net-attach-def + validation: + openAPIV3Schema: + properties: + spec: + properties: + config: + type: string + diff --git a/examples/npwg-demo-1/02_clusterrole.yml b/examples/npwg-demo-1/02_clusterrole.yml index 635ba0c1e..d374cca51 100644 --- a/examples/npwg-demo-1/02_clusterrole.yml +++ b/examples/npwg-demo-1/02_clusterrole.yml @@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: multus-crd-overpowered + name: multus rules: - apiGroups: - '*' diff --git a/glide.yaml b/glide.yaml index 92213fb75..8fbf50306 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,16 +1,18 @@ package: github.com/intel/multus-cni +ignore: +- bytes import: - package: github.com/containernetworking/cni version: 07c1a6da47b7fbf8b357f4949ecce2113e598491 subpackages: - - pkg/ip - - pkg/ipam - pkg/skel - pkg/types - pkg/version - package: github.com/containernetworking/plugins version: 2b8b1ac0af4568e928d96ccc5f47b075416eeabd subpackages: + - pkg/ip + - pkg/ipam - pkg/ns - package: github.com/onsi/ginkgo version: 7f8ab55aaf3b86885aa55b762e803744d1674700 diff --git a/images/70-multus.conf b/images/70-multus.conf index 6f91a0ca2..591c174af 100644 --- a/images/70-multus.conf +++ b/images/70-multus.conf @@ -6,7 +6,8 @@ "type": "flannel", "name": "flannel.1", "delegate": { - "isDefaultGateway": true + "isDefaultGateway": true, + "hairpinMode": true } } ], diff --git a/images/README.md b/images/README.md index ec647d7c7..e16f4106f 100644 --- a/images/README.md +++ b/images/README.md @@ -2,10 +2,10 @@ This is used for distribution of Multus in a Docker image. -Typically you'd build this from the root of your Multus clone, and you'd set the `-f` flag to specify the Dockerfile during build time. This allows the addition of the entirety of the Multus git clone as part of the Docker context. Use the `-f` flag with the root of the clone as the context (e.g. your current work directory would be root of git clone), such as: +Typically you'd build this from the root of your Multus clone, as such: ``` -$ docker build -t dougbtv/multus -f ./images/Dockerfile . +$ docker build -t dougbtv/multus . ``` --- diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index d3d7744b6..49b74ab39 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -31,6 +31,7 @@ import ( "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/pkg/skel" cnitypes "github.com/containernetworking/cni/pkg/types" + "github.com/intel/multus-cni/logging" "github.com/intel/multus-cni/types" ) @@ -96,6 +97,7 @@ func GetComputeDeviceMap(pod *v1.Pod) map[string]*ResourceInfo { } func setKubeClientInfo(c *clientInfo, client KubeClient, k8sArgs *types.K8sArgs) { + logging.Debugf("setKubeClientInfo: %v, %v, %v", c, client, k8sArgs) c.Client = client c.Podnamespace = string(k8sArgs.K8S_POD_NAMESPACE) c.Podname = string(k8sArgs.K8S_POD_NAME) @@ -103,9 +105,10 @@ func setKubeClientInfo(c *clientInfo, client KubeClient, k8sArgs *types.K8sArgs) func SetNetworkStatus(k *clientInfo, netStatus []*types.NetworkStatus) error { + logging.Debugf("SetNetworkStatus: %v, %v", k, netStatus) pod, err := k.Client.GetPod(k.Podnamespace, k.Podname) if err != nil { - return fmt.Errorf("SetNetworkStatus: failed to query the pod %v in out of cluster comm: %v", k.Podname, err) + return logging.Errorf("SetNetworkStatus: failed to query the pod %v in out of cluster comm: %v", k.Podname, err) } var ns string @@ -114,7 +117,7 @@ func SetNetworkStatus(k *clientInfo, netStatus []*types.NetworkStatus) error { for _, nets := range netStatus { data, err := json.MarshalIndent(nets, "", " ") if err != nil { - return fmt.Errorf("SetNetworkStatus: error with Marshal Indent: %v", err) + return logging.Errorf("SetNetworkStatus: error with Marshal Indent: %v", err) } networkStatus = append(networkStatus, string(data)) } @@ -123,13 +126,14 @@ func SetNetworkStatus(k *clientInfo, netStatus []*types.NetworkStatus) error { } _, err = setPodNetworkAnnotation(k.Client, k.Podnamespace, pod, ns) if err != nil { - return fmt.Errorf("SetNetworkStatus: failed to update the pod %v in out of cluster comm: %v", k.Podname, err) + return logging.Errorf("SetNetworkStatus: failed to update the pod %v in out of cluster comm: %v", k.Podname, err) } return nil } func setPodNetworkAnnotation(client KubeClient, namespace string, pod *v1.Pod, networkstatus string) (*v1.Pod, error) { + logging.Debugf("setPodNetworkAnnotation: %v, %s, %v, %s", client, namespace, pod, networkstatus) //if pod annotations is empty, make sure it allocatable if len(pod.Annotations) == 0 { pod.Annotations = make(map[string]string) @@ -151,12 +155,13 @@ func setPodNetworkAnnotation(client KubeClient, namespace string, pod *v1.Pod, n pod, err = client.UpdatePodStatus(pod) return err }); resultErr != nil { - return nil, fmt.Errorf("status update failed for pod %s/%s: %v", pod.Namespace, pod.Name, resultErr) + return nil, logging.Errorf("status update failed for pod %s/%s: %v", pod.Namespace, pod.Name, resultErr) } return pod, nil } func getPodNetworkAnnotation(pod *v1.Pod) (string, string, error) { + logging.Debugf("getPodNetworkAnnotation: %v", pod) return pod.Annotations["k8s.v1.cni.cncf.io/networks"], pod.ObjectMeta.Namespace, nil } @@ -165,6 +170,7 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error var netIfName string var networkName string + logging.Debugf("parsePodNetworkObjectName: %s", podnetwork) slashItems := strings.Split(podnetwork, "/") if len(slashItems) == 2 { netNsName = strings.TrimSpace(slashItems[0]) @@ -172,7 +178,7 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error } else if len(slashItems) == 1 { networkName = slashItems[0] } else { - return "", "", "", fmt.Errorf("Invalid network object (failed at '/')") + return "", "", "", logging.Errorf("Invalid network object (failed at '/')") } atItems := strings.Split(networkName, "@") @@ -180,7 +186,7 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error if len(atItems) == 2 { netIfName = strings.TrimSpace(atItems[1]) } else if len(atItems) != 1 { - return "", "", "", fmt.Errorf("Invalid network object (failed at '@')") + return "", "", "", logging.Errorf("Invalid network object (failed at '@')") } // Check and see if each item matches the specification for valid attachment name. @@ -192,23 +198,25 @@ func parsePodNetworkObjectName(podnetwork string) (string, string, string, error for i := range allItems { matched, _ := regexp.MatchString("^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", allItems[i]) if !matched && len([]rune(allItems[i])) > 0 { - return "", "", "", fmt.Errorf(fmt.Sprintf("Failed to parse: one or more items did not match comma-delimited format (must consist of lower case alphanumeric characters). Must start and end with an alphanumeric character), mismatch @ '%v'", allItems[i])) + return "", "", "", logging.Errorf(fmt.Sprintf("Failed to parse: one or more items did not match comma-delimited format (must consist of lower case alphanumeric characters). Must start and end with an alphanumeric character), mismatch @ '%v'", allItems[i])) } } + logging.Debugf("parsePodNetworkObjectName: parsed: %s, %s, %s", netNsName, networkName, netIfName) return netNsName, networkName, netIfName, nil } func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.NetworkSelectionElement, error) { var networks []*types.NetworkSelectionElement + logging.Debugf("parsePodNetworkAnnotation: %s, %s", podNetworks, defaultNamespace) if podNetworks == "" { - return nil, fmt.Errorf("parsePodNetworkAnnotation: pod annotation not having \"network\" as key, refer Multus README.md for the usage guide") + return nil, logging.Errorf("parsePodNetworkAnnotation: pod annotation not having \"network\" as key, refer Multus README.md for the usage guide") } if strings.IndexAny(podNetworks, "[{\"") >= 0 { if err := json.Unmarshal([]byte(podNetworks), &networks); err != nil { - return nil, fmt.Errorf("parsePodNetworkAnnotation: failed to parse pod Network Attachment Selection Annotation JSON format: %v", err) + return nil, logging.Errorf("parsePodNetworkAnnotation: failed to parse pod Network Attachment Selection Annotation JSON format: %v", err) } } else { // Comma-delimited list of network attachment object names @@ -219,7 +227,7 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N // Parse network name (i.e. /@) netNsName, networkName, netIfName, err := parsePodNetworkObjectName(item) if err != nil { - return nil, fmt.Errorf("parsePodNetworkAnnotation: %v", err) + return nil, logging.Errorf("parsePodNetworkAnnotation: %v", err) } networks = append(networks, &types.NetworkSelectionElement{ @@ -240,6 +248,7 @@ func parsePodNetworkAnnotation(podNetworks, defaultNamespace string) ([]*types.N } func getCNIConfigFromFile(name string, confdir string) ([]byte, error) { + logging.Debugf("getCNIConfigFromFile: %s, %s", name, confdir) // In the absence of valid keys in a Spec, the runtime (or // meta-plugin) should load and execute a CNI .configlist @@ -250,9 +259,9 @@ func getCNIConfigFromFile(name string, confdir string) ([]byte, error) { files, err := libcni.ConfFiles(confdir, []string{".conf", ".json", ".conflist"}) switch { case err != nil: - return nil, fmt.Errorf("No networks found in %s", confdir) + return nil, logging.Errorf("No networks found in %s", confdir) case len(files) == 0: - return nil, fmt.Errorf("No networks found in %s", confdir) + return nil, logging.Errorf("No networks found in %s", confdir) } for _, confFile := range files { @@ -260,7 +269,7 @@ func getCNIConfigFromFile(name string, confdir string) ([]byte, error) { if strings.HasSuffix(confFile, ".conflist") { confList, err = libcni.ConfListFromFile(confFile) if err != nil { - return nil, fmt.Errorf("Error loading CNI conflist file %s: %v", confFile, err) + return nil, logging.Errorf("Error loading CNI conflist file %s: %v", confFile, err) } if confList.Name == name { @@ -270,21 +279,21 @@ func getCNIConfigFromFile(name string, confdir string) ([]byte, error) { } else { conf, err := libcni.ConfFromFile(confFile) if err != nil { - return nil, fmt.Errorf("Error loading CNI config file %s: %v", confFile, err) + return nil, logging.Errorf("Error loading CNI config file %s: %v", confFile, err) } if conf.Network.Name == name { // Ensure the config has a "type" so we know what plugin to run. // Also catches the case where somebody put a conflist into a conf file. if conf.Network.Type == "" { - return nil, fmt.Errorf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile) + return nil, logging.Errorf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile) } return conf.Bytes, nil } } } - return nil, fmt.Errorf("no network available in the name %s in cni dir %s", name, confdir) + return nil, logging.Errorf("no network available in the name %s in cni dir %s", name, confdir) } // getCNIConfigFromSpec reads a CNI JSON configuration from the NetworkAttachmentDefinition @@ -293,10 +302,11 @@ func getCNIConfigFromSpec(configData, netName string) ([]byte, error) { var rawConfig map[string]interface{} var err error + logging.Debugf("getCNIConfigFromSpec: %s, %s", configData, netName) configBytes := []byte(configData) err = json.Unmarshal(configBytes, &rawConfig) if err != nil { - return nil, fmt.Errorf("getCNIConfigFromSpec: failed to unmarshal Spec.Config: %v", err) + return nil, logging.Errorf("getCNIConfigFromSpec: failed to unmarshal Spec.Config: %v", err) } // Inject network name if missing from Config for the thick plugin case @@ -304,7 +314,7 @@ func getCNIConfigFromSpec(configData, netName string) ([]byte, error) { rawConfig["name"] = netName configBytes, err = json.Marshal(rawConfig) if err != nil { - return nil, fmt.Errorf("getCNIConfigFromSpec: failed to re-marshal Spec.Config: %v", err) + return nil, logging.Errorf("getCNIConfigFromSpec: failed to re-marshal Spec.Config: %v", err) } } @@ -315,6 +325,7 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit var config []byte var err error + logging.Debugf("cniConfigFromNetworkResource: %v, %s", customResource, confdir) emptySpec := types.NetworkAttachmentDefinitionSpec{} if customResource.Spec == emptySpec { // Network Spec empty; generate delegate from CNI JSON config @@ -322,7 +333,7 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit // name as the custom resource config, err = getCNIConfigFromFile(customResource.Metadata.Name, confdir) if err != nil { - return nil, fmt.Errorf("cniConfigFromNetworkResource: err in getCNIConfigFromFile: %v", err) + return nil, logging.Errorf("cniConfigFromNetworkResource: err in getCNIConfigFromFile: %v", err) } } else { // Config contains a standard JSON-encoded CNI configuration @@ -330,7 +341,7 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit // execute. config, err = getCNIConfigFromSpec(customResource.Spec.Config, customResource.Metadata.Name) if err != nil { - return nil, fmt.Errorf("cniConfigFromNetworkResource: err in getCNIConfigFromSpec: %v", err) + return nil, logging.Errorf("cniConfigFromNetworkResource: err in getCNIConfigFromSpec: %v", err) } } @@ -338,15 +349,17 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit } func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string, resourceMap map[string]*ResourceInfo) (*types.DelegateNetConf, error) { + + logging.Debugf("getKubernetesDelegate: %v, %v, %s", client, net, confdir) rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", net.Namespace, net.Name) netData, err := client.GetRawWithPath(rawPath) if err != nil { - return nil, fmt.Errorf("getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err) + return nil, logging.Errorf("getKubernetesDelegate: failed to get network resource, refer Multus README.md for the usage guide: %v", err) } customResource := &types.NetworkAttachmentDefinition{} if err := json.Unmarshal(netData, customResource); err != nil { - return nil, fmt.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err) + return nil, logging.Errorf("getKubernetesDelegate: failed to get the netplugin data: %v", err) } // DEVICE_ID @@ -379,7 +392,7 @@ func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement if err != nil { return nil, err } - // return delegate, fmt.Errorf("Debug: delegate object: %+v", string(delegate.Bytes)) + return delegate, nil } @@ -389,13 +402,13 @@ func delegateAddDeviceID(inBytes []byte, deviceID string) ([]byte, error) { err = json.Unmarshal(inBytes, &rawConfig) if err != nil { - return nil, fmt.Errorf("delegateAddDeviceID: failed to unmarshal inBytes: %v", err) + return nil, logging.Errorf("delegateAddDeviceID: failed to unmarshal inBytes: %v", err) } // Inject deviceID rawConfig["deviceID"] = deviceID configBytes, err := json.Marshal(rawConfig) if err != nil { - return nil, fmt.Errorf("delegateAddDeviceID: failed to re-marshal Spec.Config: %v", err) + return nil, logging.Errorf("delegateAddDeviceID: failed to re-marshal Spec.Config: %v", err) } return configBytes, nil } @@ -409,6 +422,7 @@ type KubeClient interface { func GetK8sArgs(args *skel.CmdArgs) (*types.K8sArgs, error) { k8sArgs := &types.K8sArgs{} + logging.Debugf("GetK8sNetwork: %v", args) err := cnitypes.LoadArgs(args.Args, k8sArgs) if err != nil { return nil, err @@ -423,6 +437,7 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient var err error clientInfo := &clientInfo{} + logging.Debugf("TryLoadK8sDelegates: %v, %v, %v", k8sArgs, conf, kubeClient) kubeClient, err = GetK8sClient(conf.Kubeconfig, kubeClient) if err != nil { return 0, nil, err @@ -431,7 +446,7 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient if kubeClient == nil { if len(conf.Delegates) == 0 { // No available kube client and no delegates, we can't do anything - return 0, nil, fmt.Errorf("must have either Kubernetes config or delegates, refer Multus README.md for the usage guide") + return 0, nil, logging.Errorf("must have either Kubernetes config or delegates, refer Multus README.md for the usage guide") } return 0, nil, nil } @@ -442,11 +457,9 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient if _, ok := err.(*NoK8sNetworkError); ok { return 0, clientInfo, nil } - return 0, nil, fmt.Errorf("Multus: Err in getting k8s network from pod: %v", err) + return 0, nil, logging.Errorf("Multus: Err in getting k8s network from pod: %v", err) } - // TODO: add Device information into delegates from resourceMap - if err = conf.AddDelegates(delegates); err != nil { return 0, nil, err } @@ -455,6 +468,7 @@ func TryLoadK8sDelegates(k8sArgs *types.K8sArgs, conf *types.NetConf, kubeClient } func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error) { + logging.Debugf("GetK8sClient: %s, %v", kubeconfig, kubeClient) // If we get a valid kubeClient (eg from testcases) just return that // one. if kubeClient != nil { @@ -469,13 +483,13 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error) // uses the current context in kubeconfig config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) if err != nil { - return nil, fmt.Errorf("GetK8sClient: failed to get context for the kubeconfig %v, refer Multus README.md for the usage guide: %v", kubeconfig, err) + return nil, logging.Errorf("GetK8sClient: failed to get context for the kubeconfig %v, refer Multus README.md for the usage guide: %v", kubeconfig, err) } } else if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" { // Try in-cluster config where multus might be running in a kubernetes pod config, err = rest.InClusterConfig() if err != nil { - return nil, fmt.Errorf("createK8sClient: failed to get context for in-cluster kube config, refer Multus README.md for the usage guide: %v", err) + return nil, logging.Errorf("createK8sClient: failed to get context for in-cluster kube config, refer Multus README.md for the usage guide: %v", err) } } else { // No kubernetes config; assume we shouldn't talk to Kube at all @@ -492,10 +506,11 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error) } func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) ([]*types.DelegateNetConf, error) { + logging.Debugf("GetK8sNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir) pod, err := k8sclient.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) if err != nil { - return nil, fmt.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err) + return nil, logging.Errorf("GetK8sNetwork: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err) } netAnnot, defaultNamespace, err := getPodNetworkAnnotation(pod) @@ -520,7 +535,7 @@ func GetK8sNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string) for _, net := range networks { delegate, err := getKubernetesDelegate(k8sclient, net, confdir, resourceMap) if err != nil { - return nil, fmt.Errorf("GetK8sNetwork: failed getting the delegate: %v", err) + return nil, logging.Errorf("GetK8sNetwork: failed getting the delegate: %v", err) } delegates = append(delegates, delegate) } diff --git a/logging/logging.go b/logging/logging.go index 3e1fbdb1d..866646066 100644 --- a/logging/logging.go +++ b/logging/logging.go @@ -35,8 +35,9 @@ const ( ) var loggingStderr bool -var loggingFp *os.File +var loggingFp *os.File var loggingLevel Level + const defaultTimestampFormat = time.RFC3339 func (l Level) String() string { @@ -75,8 +76,9 @@ func Debugf(format string, a ...interface{}) { Printf(DebugLevel, format, a...) } -func Errorf(format string, a ...interface{}) { +func Errorf(format string, a ...interface{}) error { Printf(ErrorLevel, format, a...) + return fmt.Errorf(format, a...) } func Panicf(format string, a ...interface{}) { @@ -86,7 +88,7 @@ func Panicf(format string, a ...interface{}) { Printf(PanicLevel, "========= Stack trace output end ========") } -func GetLoggingLevel (levelStr string) Level { +func GetLoggingLevel(levelStr string) Level { switch strings.ToLower(levelStr) { case "debug": return DebugLevel @@ -99,18 +101,18 @@ func GetLoggingLevel (levelStr string) Level { return UnknownLevel } -func SetLogLevel (levelStr string) { +func SetLogLevel(levelStr string) { level := GetLoggingLevel(levelStr) if level < MaxLevel { loggingLevel = level } } -func SetLogStderr (enable bool) { +func SetLogStderr(enable bool) { loggingStderr = enable } -func SetLogFile (filename string) { +func SetLogFile(filename string) { if filename == "" { return } diff --git a/logging/logging_test.go b/logging/logging_test.go index 335d03ecd..797e8fc0a 100644 --- a/logging/logging_test.go +++ b/logging/logging_test.go @@ -17,8 +17,8 @@ package logging import ( "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) func TestLogging(t *testing.T) { diff --git a/multus/multus.go b/multus/multus.go index 591c9ffac..73f150159 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -32,26 +32,29 @@ import ( "github.com/containernetworking/cni/pkg/version" "github.com/containernetworking/plugins/pkg/ns" k8s "github.com/intel/multus-cni/k8sclient" + "github.com/intel/multus-cni/logging" "github.com/intel/multus-cni/types" "github.com/vishvananda/netlink" ) func saveScratchNetConf(containerID, dataDir string, netconf []byte) error { + logging.Debugf("saveScratchNetConf: %s, %s, %s", containerID, dataDir, string(netconf)) if err := os.MkdirAll(dataDir, 0700); err != nil { - return fmt.Errorf("failed to create the multus data directory(%q): %v", dataDir, err) + return logging.Errorf("failed to create the multus data directory(%q): %v", dataDir, err) } path := filepath.Join(dataDir, containerID) err := ioutil.WriteFile(path, netconf, 0600) if err != nil { - return fmt.Errorf("failed to write container data in the path(%q): %v", path, err) + return logging.Errorf("failed to write container data in the path(%q): %v", path, err) } return err } func consumeScratchNetConf(containerID, dataDir string) ([]byte, error) { + logging.Debugf("consumeScratchNetConf: %s, %s", containerID, dataDir) path := filepath.Join(dataDir, containerID) defer os.Remove(path) @@ -59,6 +62,7 @@ func consumeScratchNetConf(containerID, dataDir string) ([]byte, error) { } func getIfname(delegate *types.DelegateNetConf, argif string, idx int) string { + logging.Debugf("getIfname: %v, %s, %d", delegate, argif, idx) if delegate.IfnameRequest != "" { return delegate.IfnameRequest } @@ -73,22 +77,24 @@ func getIfname(delegate *types.DelegateNetConf, argif string, idx int) string { } func saveDelegates(containerID, dataDir string, delegates []*types.DelegateNetConf) error { + logging.Debugf("saveDelegates: %s, %s, %v", containerID, dataDir, delegates) delegatesBytes, err := json.Marshal(delegates) if err != nil { - return fmt.Errorf("error serializing delegate netconf: %v", err) + return logging.Errorf("error serializing delegate netconf: %v", err) } if err = saveScratchNetConf(containerID, dataDir, delegatesBytes); err != nil { - return fmt.Errorf("error in saving the delegates : %v", err) + return logging.Errorf("error in saving the delegates : %v", err) } return err } func validateIfName(nsname string, ifname string) error { + logging.Debugf("validateIfName: %s, %s", nsname, ifname) podNs, err := ns.GetNS(nsname) if err != nil { - return fmt.Errorf("no netns: %v", err) + return logging.Errorf("no netns: %v", err) } err = podNs.Do(func(_ ns.NetNS) error { @@ -99,61 +105,64 @@ func validateIfName(nsname string, ifname string) error { } return err } - return fmt.Errorf("ifname %s is already exist", ifname) + return logging.Errorf("ifname %s is already exist", ifname) }) return err } func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string) (cnitypes.Result, error) { + logging.Debugf("conflistAdd: %v, %s, %s", rt, string(rawnetconflist), binDir) // In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go binDirs := []string{binDir} cniNet := libcni.CNIConfig{Path: binDirs} confList, err := libcni.ConfListFromBytes(rawnetconflist) if err != nil { - return nil, fmt.Errorf("error in converting the raw bytes to conflist: %v", err) + return nil, logging.Errorf("error in converting the raw bytes to conflist: %v", err) } result, err := cniNet.AddNetworkList(confList, rt) if err != nil { - return nil, fmt.Errorf("error in getting result from AddNetworkList: %v", err) + return nil, logging.Errorf("error in getting result from AddNetworkList: %v", err) } return result, nil } func conflistDel(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string) error { + logging.Debugf("conflistDel: %v, %s, %s", rt, string(rawnetconflist), binDir) // In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go binDirs := []string{binDir} cniNet := libcni.CNIConfig{Path: binDirs} confList, err := libcni.ConfListFromBytes(rawnetconflist) if err != nil { - return fmt.Errorf("error in converting the raw bytes to conflist: %v", err) + return logging.Errorf("error in converting the raw bytes to conflist: %v", err) } err = cniNet.DelNetworkList(confList, rt) if err != nil { - return fmt.Errorf("error in getting result from DelNetworkList: %v", err) + return logging.Errorf("error in getting result from DelNetworkList: %v", err) } return err } func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetConf, rt *libcni.RuntimeConf, binDir string) (cnitypes.Result, error) { + logging.Debugf("delegateAdd: %v, %s, %v, %v, %s", exec, ifName, delegate, rt, binDir) if os.Setenv("CNI_IFNAME", ifName) != nil { - return nil, fmt.Errorf("Multus: error in setting CNI_IFNAME") + return nil, logging.Errorf("Multus: error in setting CNI_IFNAME") } if err := validateIfName(os.Getenv("CNI_NETNS"), ifName); err != nil { - return nil, fmt.Errorf("cannot set %q ifname to %q: %v", delegate.Conf.Type, ifName, err) + return nil, logging.Errorf("cannot set %q ifname to %q: %v", delegate.Conf.Type, ifName, err) } if delegate.ConfListPlugin != false { result, err := conflistAdd(rt, delegate.Bytes, binDir) if err != nil { - return nil, fmt.Errorf("Multus: error in invoke Conflist add - %q: %v", delegate.ConfList.Name, err) + return nil, logging.Errorf("Multus: error in invoke Conflist add - %q: %v", delegate.ConfList.Name, err) } return result, nil @@ -161,36 +170,38 @@ func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetCon result, err := invoke.DelegateAdd(delegate.Conf.Type, delegate.Bytes, exec) if err != nil { - return nil, fmt.Errorf("Multus: error in invoke Delegate add - %q: %v", delegate.Conf.Type, err) + return nil, logging.Errorf("Multus: error in invoke Delegate add - %q: %v", delegate.Conf.Type, err) } return result, nil } func delegateDel(exec invoke.Exec, ifName string, delegateConf *types.DelegateNetConf, rt *libcni.RuntimeConf, binDir string) error { + logging.Debugf("delegateDel: %v, %s, %v, %v, %s", exec, ifName, delegateConf, rt, binDir) if os.Setenv("CNI_IFNAME", ifName) != nil { - return fmt.Errorf("Multus: error in setting CNI_IFNAME") + return logging.Errorf("Multus: error in setting CNI_IFNAME") } if delegateConf.ConfListPlugin != false { err := conflistDel(rt, delegateConf.Bytes, binDir) if err != nil { - return fmt.Errorf("Multus: error in invoke Conflist Del - %q: %v", delegateConf.ConfList.Name, err) + return logging.Errorf("Multus: error in invoke Conflist Del - %q: %v", delegateConf.ConfList.Name, err) } return err } if err := invoke.DelegateDel(delegateConf.Conf.Type, delegateConf.Bytes, exec); err != nil { - return fmt.Errorf("Multus: error in invoke Delegate del - %q: %v", delegateConf.Conf.Type, err) + return logging.Errorf("Multus: error in invoke Delegate del - %q: %v", delegateConf.Conf.Type, err) } return nil } func delPlugins(exec invoke.Exec, argIfname string, delegates []*types.DelegateNetConf, lastIdx int, rt *libcni.RuntimeConf, binDir string) error { + logging.Debugf("delPlugins: %v, %s, %v, %d, %v, %s", exec, argIfname, delegates, lastIdx, rt, binDir) if os.Setenv("CNI_COMMAND", "DEL") != nil { - return fmt.Errorf("Multus: error in setting CNI_COMMAND to DEL") + return logging.Errorf("Multus: error in setting CNI_COMMAND to DEL") } for idx := lastIdx; idx >= 0; idx-- { @@ -205,25 +216,26 @@ func delPlugins(exec invoke.Exec, argIfname string, delegates []*types.DelegateN } func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cnitypes.Result, error) { + logging.Debugf("cmdAdd: %v, %v, %v", args, exec, kubeClient) n, err := types.LoadNetConf(args.StdinData) if err != nil { - return nil, fmt.Errorf("err in loading netconf: %v", err) + return nil, logging.Errorf("err in loading netconf: %v", err) } k8sArgs, err := k8s.GetK8sArgs(args) if err != nil { - return nil, fmt.Errorf("Multus: Err in getting k8s args: %v", err) + return nil, logging.Errorf("Multus: Err in getting k8s args: %v", err) } numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, n, kubeClient) if err != nil { - return nil, fmt.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err) + return nil, logging.Errorf("Multus: Err in loading K8s Delegates k8s args: %v", err) } if numK8sDelegates == 0 { // cache the multus config if we have only Multus delegates if err := saveDelegates(args.ContainerID, n.CNIDir, n.Delegates); err != nil { - return nil, fmt.Errorf("Multus: Err in saving the delegates: %v", err) + return nil, logging.Errorf("Multus: Err in saving the delegates: %v", err) } } @@ -250,7 +262,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn if kc.Podnamespace != "kube-system" { delegateNetStatus, err := types.LoadNetworkStatus(tmpResult, delegate.Conf.Name, delegate.MasterPlugin) if err != nil { - return nil, fmt.Errorf("Multus: Err in setting networks status: %v", err) + return nil, logging.Errorf("Multus: Err in setting networks status: %v", err) } netStatus = append(netStatus, delegateNetStatus) @@ -261,7 +273,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn if err != nil { // Ignore errors; DEL must be idempotent anyway _ = delPlugins(exec, args.IfName, n.Delegates, lastIdx, rt, n.BinDir) - return nil, fmt.Errorf("Multus: Err in tearing down failed plugins: %v", err) + return nil, logging.Errorf("Multus: Err in tearing down failed plugins: %v", err) } //set the network status annotation in apiserver, only in case Multus as kubeconfig @@ -269,7 +281,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn if kc.Podnamespace != "kube-system" { err = k8s.SetNetworkStatus(kc, netStatus) if err != nil { - return nil, fmt.Errorf("Multus: Err set the networks status: %v", err) + return nil, logging.Errorf("Multus: Err set the networks status: %v", err) } } } @@ -278,6 +290,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn } func cmdGet(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cnitypes.Result, error) { + logging.Debugf("cmdGet: %v, %v, %v", args, exec, kubeClient) in, err := types.LoadNetConf(args.StdinData) if err != nil { return nil, err @@ -289,14 +302,31 @@ func cmdGet(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn } func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) error { + logging.Debugf("cmdDel: %v, %v, %v", args, exec, kubeClient) in, err := types.LoadNetConf(args.StdinData) if err != nil { return err } + if args.Netns == "" { + return nil + } + netns, err := ns.GetNS(args.Netns) + if err != nil { + // if NetNs is passed down by the Cloud Orchestration Engine, or if it called multiple times + // so don't return an error if the device is already removed. + // https://github.com/kubernetes/kubernetes/issues/43014#issuecomment-287164444 + _, ok := err.(ns.NSPathNotExistErr) + if ok { + return nil + } + return fmt.Errorf("failed to open netns %q: %v", netns, err) + } + defer netns.Close() + k8sArgs, err := k8s.GetK8sArgs(args) if err != nil { - return fmt.Errorf("Multus: Err in getting k8s args: %v", err) + return logging.Errorf("Multus: Err in getting k8s args: %v", err) } numK8sDelegates, kc, err := k8s.TryLoadK8sDelegates(k8sArgs, in, kubeClient) @@ -312,11 +342,11 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err // Per spec should ignore error if resources are missing / already removed return nil } - return fmt.Errorf("Multus: Err in reading the delegates: %v", err) + return logging.Errorf("Multus: Err in reading the delegates: %v", err) } if err := json.Unmarshal(netconfBytes, &in.Delegates); err != nil { - return fmt.Errorf("Multus: failed to load netconf: %v", err) + return logging.Errorf("Multus: failed to load netconf: %v", err) } } @@ -325,7 +355,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err if kc.Podnamespace != "kube-system" { err := k8s.SetNetworkStatus(kc, nil) if err != nil { - return fmt.Errorf("Multus: Err unset the networks status: %v", err) + return logging.Errorf("Multus: Err unset the networks status: %v", err) } } } diff --git a/test.sh b/test.sh index 334203c70..ad17ae8d2 100755 --- a/test.sh +++ b/test.sh @@ -13,5 +13,5 @@ export GO15VENDOREXPERIMENT=1 export GOBIN=${PWD}/bin export GOPATH=${PWD}/gopath -bash -c "umask 0; cd ${GOPATH}/src/${REPO_PATH}; PATH=${GOROOT}/bin:$(pwd)/bin:${PATH} go test ./..." +bash -c "umask 0; cd ${GOPATH}/src/${REPO_PATH}; PATH=${GOROOT}/bin:$(pwd)/bin:${PATH} go test -v -covermode=count -coverprofile=coverage.out ./..." diff --git a/types/conf.go b/types/conf.go index 6fcc92124..e95d9cee2 100644 --- a/types/conf.go +++ b/types/conf.go @@ -17,7 +17,6 @@ package types import ( "encoding/json" - "fmt" "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/pkg/skel" @@ -35,15 +34,16 @@ const ( func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error { + logging.Debugf("LoadDelegateNetConfList: %s, %v", string(bytes), delegateConf) if err := json.Unmarshal(bytes, &delegateConf.ConfList); err != nil { - return fmt.Errorf("err in unmarshalling delegate conflist: %v", err) + return logging.Errorf("err in unmarshalling delegate conflist: %v", err) } if delegateConf.ConfList.Plugins == nil { - return fmt.Errorf("delegate must have the 'type'or 'Plugin' field") + return logging.Errorf("delegate must have the 'type'or 'Plugin' field") } if delegateConf.ConfList.Plugins[0].Type == "" { - return fmt.Errorf("a plugin delegate must have the 'type' field") + return logging.Errorf("a plugin delegate must have the 'type' field") } delegateConf.ConfListPlugin = true return nil @@ -52,14 +52,15 @@ func LoadDelegateNetConfList(bytes []byte, delegateConf *DelegateNetConf) error // Convert raw CNI JSON into a DelegateNetConf structure func LoadDelegateNetConf(bytes []byte, ifnameRequest string) (*DelegateNetConf, error) { delegateConf := &DelegateNetConf{} + logging.Debugf("LoadDelegateNetConf: %s, %s", string(bytes), ifnameRequest) if err := json.Unmarshal(bytes, &delegateConf.Conf); err != nil { - return nil, fmt.Errorf("error in LoadDelegateNetConf - unmarshalling delegate config: %v", err) + return nil, logging.Errorf("error in LoadDelegateNetConf - unmarshalling delegate config: %v", err) } // Do some minimal validation if delegateConf.Conf.Type == "" { if err := LoadDelegateNetConfList(bytes, delegateConf); err != nil { - return nil, fmt.Errorf("error in LoadDelegateNetConf: %v", err) + return nil, logging.Errorf("error in LoadDelegateNetConf: %v", err) } } @@ -74,6 +75,7 @@ func LoadDelegateNetConf(bytes []byte, ifnameRequest string) (*DelegateNetConf, func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string) (*libcni.RuntimeConf, error) { + logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s", args, k8sArgs, ifName) // In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#buildCNIRuntimeConf // Todo // ingress, egress and bandwidth capability features as same as kubelet. @@ -92,10 +94,12 @@ func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string) (*l } func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*NetworkStatus, error) { + logging.Debugf("LoadNetworkStatus: %v, %s, %s", r, netName, defaultNet) + // Convert whatever the IPAM result was into the current Result type result, err := current.NewResultFromResult(r) if err != nil { - return nil, fmt.Errorf("error convert the type.Result to current.Result: %v", err) + return nil, logging.Errorf("error convert the type.Result to current.Result: %v", err) } netstatus := &NetworkStatus{} @@ -128,8 +132,10 @@ func LoadNetworkStatus(r types.Result, netName string, defaultNet bool) (*Networ func LoadNetConf(bytes []byte) (*NetConf, error) { netconf := &NetConf{} + + logging.Debugf("LoadNetConf: %s", string(bytes)) if err := json.Unmarshal(bytes, netconf); err != nil { - return nil, fmt.Errorf("failed to load netconf: %v", err) + return nil, logging.Errorf("failed to load netconf: %v", err) } // Logging @@ -144,16 +150,16 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { if netconf.RawPrevResult != nil { resultBytes, err := json.Marshal(netconf.RawPrevResult) if err != nil { - return nil, fmt.Errorf("could not serialize prevResult: %v", err) + return nil, logging.Errorf("could not serialize prevResult: %v", err) } res, err := version.NewResult(netconf.CNIVersion, resultBytes) if err != nil { - return nil, fmt.Errorf("could not parse prevResult: %v", err) + return nil, logging.Errorf("could not parse prevResult: %v", err) } netconf.RawPrevResult = nil netconf.PrevResult, err = current.NewResultFromResult(res) if err != nil { - return nil, fmt.Errorf("could not convert result to current version: %v", err) + return nil, logging.Errorf("could not convert result to current version: %v", err) } } @@ -164,7 +170,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { // the existing delegate list and all delegates executed in-order. if len(netconf.RawDelegates) == 0 { - return nil, fmt.Errorf("at least one delegate must be specified") + return nil, logging.Errorf("at least one delegate must be specified") } if netconf.CNIDir == "" { @@ -182,11 +188,11 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { for idx, rawConf := range netconf.RawDelegates { bytes, err := json.Marshal(rawConf) if err != nil { - return nil, fmt.Errorf("error marshalling delegate %d config: %v", idx, err) + return nil, logging.Errorf("error marshalling delegate %d config: %v", idx, err) } delegateConf, err := LoadDelegateNetConf(bytes, "") if err != nil { - return nil, fmt.Errorf("failed to load delegate %d config: %v", idx, err) + return nil, logging.Errorf("failed to load delegate %d config: %v", idx, err) } netconf.Delegates = append(netconf.Delegates, delegateConf) } @@ -200,6 +206,7 @@ func LoadNetConf(bytes []byte) (*NetConf, error) { // AddDelegates appends the new delegates to the delegates list func (n *NetConf) AddDelegates(newDelegates []*DelegateNetConf) error { + logging.Debugf("AddDelegates: %v", newDelegates) n.Delegates = append(n.Delegates, newDelegates...) return nil }