From b18c5c4d537d91b3a3301a6b7026d02f15bd9c7d Mon Sep 17 00:00:00 2001 From: Abdul Halim Date: Thu, 7 Feb 2019 15:48:40 +0000 Subject: [PATCH] add kubelet client for Pod resource info This change introduces kubelet client to get allocated device information of a Pod from newly added Kubelet grpc service. For more information please see: [kubernetes/kubernetes#70508](https://github.com/kubernetes/kubernetes/pull/70508) With this kubelet service we no longer need to rely on kubelet checkpoint file to get device information. Change-Id: I11e58ccdd52662601f445fa24c7d55c225441efc Signed-off-by: Abdul Halim --- checkpoint/checkpoint.go | 113 --------------------- checkpoint/checkpoint_test.go | 120 ---------------------- glide.lock | 64 ++++++++++-- glide.yaml | 6 ++ k8sclient/k8sclient.go | 34 +++---- kubeletclient/kubeletclient.go | 90 +++++++++++++++++ kubeletclient/kubeletclient_test.go | 150 ++++++++++++++++++++++++++++ 7 files changed, 319 insertions(+), 258 deletions(-) delete mode 100644 checkpoint/checkpoint.go delete mode 100644 checkpoint/checkpoint_test.go create mode 100644 kubeletclient/kubeletclient.go create mode 100644 kubeletclient/kubeletclient_test.go diff --git a/checkpoint/checkpoint.go b/checkpoint/checkpoint.go deleted file mode 100644 index be7dd7385..000000000 --- a/checkpoint/checkpoint.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) 2018 Intel Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package checkpoint - -import ( - "encoding/json" - "io/ioutil" - - "github.com/intel/multus-cni/logging" - "github.com/intel/multus-cni/types" -) - -const ( - checkPointfile = "/var/lib/kubelet/device-plugins/kubelet_internal_checkpoint" -) - -type PodDevicesEntry struct { - PodUID string - ContainerName string - ResourceName string - DeviceIDs []string - AllocResp []byte -} - -type checkpointData struct { - PodDeviceEntries []PodDevicesEntry - RegisteredDevices map[string][]string -} - -type Data struct { - Data checkpointData - Checksum uint64 -} - -type Checkpoint interface { - // GetComputeDeviceMap returns an instance of a map of ResourceInfo for a PodID - GetComputeDeviceMap(string) (map[string]*types.ResourceInfo, error) -} -type checkpoint struct { - fileName string - podEntires []PodDevicesEntry -} - -// GetCheckpoint returns an instance of Checkpoint -func GetCheckpoint() (Checkpoint, error) { - logging.Debugf("GetCheckpoint(): invoked") - return getCheckpoint(checkPointfile) -} - -func getCheckpoint(filePath string) (Checkpoint, error) { - cp := &checkpoint{fileName: filePath} - err := cp.getPodEntries() - if err != nil { - return nil, err - } - logging.Debugf("getCheckpoint(): created checkpoint instance with file: %s", filePath) - return cp, nil -} - -// getPodEntries gets all Pod device allocation entries from checkpoint file -func (cp *checkpoint) getPodEntries() error { - - cpd := &Data{} - rawBytes, err := ioutil.ReadFile(cp.fileName) - if err != nil { - return logging.Errorf("getPodEntries(): error reading file %s\n%v\n", checkPointfile, err) - } - - if err = json.Unmarshal(rawBytes, cpd); err != nil { - return logging.Errorf("getPodEntries(): error unmarshalling raw bytes %v", err) - } - - cp.podEntires = cpd.Data.PodDeviceEntries - logging.Debugf("getPodEntries(): podEntires %+v", cp.podEntires) - return nil -} - -// GetComputeDeviceMap returns an instance of a map of ResourceInfo -func (cp *checkpoint) GetComputeDeviceMap(podID string) (map[string]*types.ResourceInfo, error) { - - resourceMap := make(map[string]*types.ResourceInfo) - - if podID == "" { - return nil, logging.Errorf("GetComputeDeviceMap(): invalid Pod cannot be empty") - } - - for _, pod := range cp.podEntires { - if pod.PodUID == podID { - entry, ok := resourceMap[pod.ResourceName] - if ok { - // already exists; append to it - entry.DeviceIDs = append(entry.DeviceIDs, pod.DeviceIDs...) - } else { - // new entry - resourceMap[pod.ResourceName] = &types.ResourceInfo{DeviceIDs: pod.DeviceIDs} - } - } - } - return resourceMap, nil -} diff --git a/checkpoint/checkpoint_test.go b/checkpoint/checkpoint_test.go deleted file mode 100644 index 7834a382e..000000000 --- a/checkpoint/checkpoint_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package checkpoint - -import ( - "os" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "io/ioutil" - "testing" - - "github.com/intel/multus-cni/types" -) - -const ( - fakeTempFile = "/tmp/kubelet_internal_checkpoint" -) - -type fakeCheckpoint struct { - fileName string -} - -func (fc *fakeCheckpoint) WriteToFile(inBytes []byte) error { - return ioutil.WriteFile(fc.fileName, inBytes, 0600) -} - -func (fc *fakeCheckpoint) DeleteFile() error { - return os.Remove(fc.fileName) -} - -func TestCheckpoint(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Checkpoint") -} - -var _ = BeforeSuite(func() { - sampleData := `{ - "Data": { - "PodDeviceEntries": [ - { - "PodUID": "970a395d-bb3b-11e8-89df-408d5c537d23", - "ContainerName": "appcntr1", - "ResourceName": "intel.com/sriov_net_A", - "DeviceIDs": [ - "0000:03:02.3", - "0000:03:02.0" - ], - "AllocResp": "CikKC3NyaW92X25ldF9BEhogMDAwMDowMzowMi4zIDAwMDA6MDM6MDIuMA==" - } - ], - "RegisteredDevices": { - "intel.com/sriov_net_A": [ - "0000:03:02.1", - "0000:03:02.2", - "0000:03:02.3", - "0000:03:02.0" - ], - "intel.com/sriov_net_B": [ - "0000:03:06.3", - "0000:03:06.0", - "0000:03:06.1", - "0000:03:06.2" - ] - } - }, - "Checksum": 229855270 - }` - - fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile} - err := fakeCheckpoint.WriteToFile([]byte(sampleData)) - Expect(err).NotTo(HaveOccurred()) -}) - -var _ = Describe("Kubelet checkpoint data read operations", func() { - Context("Using /tmp/kubelet_internal_checkpoint file", func() { - var ( - cp Checkpoint - err error - resourceMap map[string]*types.ResourceInfo - resourceInfo *types.ResourceInfo - resourceAnnot = "intel.com/sriov_net_A" - ) - - It("should get a Checkpoint instance from file", func() { - cp, err = getCheckpoint(fakeTempFile) - Expect(err).NotTo(HaveOccurred()) - }) - - It("should return a ResourceMap instance", func() { - rmap, err := cp.GetComputeDeviceMap("970a395d-bb3b-11e8-89df-408d5c537d23") - Expect(err).NotTo(HaveOccurred()) - Expect(rmap).NotTo(BeEmpty()) - resourceMap = rmap - }) - - It("resourceMap should have value for \"intel.com/sriov_net_A\"", func() { - rInfo, ok := resourceMap[resourceAnnot] - Expect(ok).To(BeTrue()) - resourceInfo = rInfo - }) - - It("should have 2 deviceIDs", func() { - Expect(len(resourceInfo.DeviceIDs)).To(BeEquivalentTo(2)) - }) - - It("should have \"0000:03:02.3\" in deviceIDs[0]", func() { - Expect(resourceInfo.DeviceIDs[0]).To(BeEquivalentTo("0000:03:02.3")) - }) - - It("should have \"0000:03:02.0\" in deviceIDs[1]", func() { - Expect(resourceInfo.DeviceIDs[1]).To(BeEquivalentTo("0000:03:02.0")) - }) - }) -}) - -var _ = AfterSuite(func() { - fakeCheckpoint := &fakeCheckpoint{fileName: fakeTempFile} - err := fakeCheckpoint.DeleteFile() - Expect(err).NotTo(HaveOccurred()) -}) diff --git a/glide.lock b/glide.lock index 20cbe0cc0..4ae361858 100644 --- a/glide.lock +++ b/glide.lock @@ -1,13 +1,11 @@ -hash: 0c4ea2a342364d2ff3b43242730cb3b1db3b7e8456f6cf43da3c51dbb67e18da -updated: 2018-07-27T03:29:02.093332104+01:00 +hash: 4288149814e91e63396f7874b87151aca74047110f0d1f43027b60dd6014988f +updated: 2019-01-09T10:19:29.688011713Z imports: - name: github.com/containernetworking/cni version: 07c1a6da47b7fbf8b357f4949ecce2113e598491 subpackages: - libcni - pkg/invoke - - pkg/ip - - pkg/ipam - pkg/skel - pkg/types - pkg/types/020 @@ -16,6 +14,8 @@ imports: - name: github.com/containernetworking/plugins version: 2b8b1ac0af4568e928d96ccc5f47b075416eeabd subpackages: + - pkg/ip + - pkg/ipam - pkg/ns - pkg/testutils - name: github.com/ghodss/yaml @@ -23,7 +23,9 @@ imports: - name: github.com/gogo/protobuf version: c0656edd0d9eab7c66d1eb0c568f9039345796f7 subpackages: + - gogoproto - proto + - protoc-gen-gogo/descriptor - sortkeys - name: github.com/golang/glog version: 44145f04b68cf362d9c4df2182967c2275eaefed @@ -53,6 +55,8 @@ imports: version: 6633656539c1639d9d78127b7d47c622b5d7b6dc - name: github.com/json-iterator/go version: f2b4162afba35581b6d4a50d3b8f34e33c144682 +- name: github.com/Microsoft/go-winio + version: 78439966b38d69bf38227fbf57ac8a6fee70f69a - name: github.com/modern-go/concurrent version: bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94 - name: github.com/modern-go/reflect2 @@ -61,6 +65,7 @@ imports: version: 7f8ab55aaf3b86885aa55b762e803744d1674700 subpackages: - config + - extensions/table - internal/codelocation - internal/containernode - internal/failer @@ -91,15 +96,15 @@ imports: - name: github.com/peterbourgon/diskv version: 5f041e8faa004a95c88a202771f4cc3e991971e6 - name: github.com/pkg/errors - version: 816c9085562cd7ee03e7f8188a1cfd942858cded + version: ffb6e22f01932bf7ac35e0bad9be11f01d1c8685 - name: github.com/spf13/pflag version: 583c0c0531f06d5278b7d917446061adc344b5cd - name: github.com/vishvananda/netlink - version: 6e453822d85ef5721799774b654d4d02fed62afb + version: b2de5d10e38ecce8607e6b438b6d174f389a004e subpackages: - nl - name: github.com/vishvananda/netns - version: 54f0e4339ce73702a0607f49922aaa1e749b418d + version: be1fbeda19366dea804f00efff2dd73a1642fdcc - name: golang.org/x/crypto version: 49796115aa4b964c318aad4f3084fdb41e9aa067 subpackages: @@ -111,7 +116,9 @@ imports: - http2 - http2/hpack - idna + - internal/timeseries - lex/httplex + - trace - name: golang.org/x/sys version: 95c6576299259db960f6c5b9b69ea52422860fce subpackages: @@ -128,6 +135,40 @@ imports: version: f51c12702a4d776e4c1fa9b0fabab841babae631 subpackages: - rate +- name: google.golang.org/genproto + version: 09f6ed296fc66555a25fe4ce95173148778dfa85 + subpackages: + - googleapis/rpc/status +- name: google.golang.org/grpc + version: 168a6198bcb0ef175f7dacec0b8691fc141dc9b8 + subpackages: + - balancer + - balancer/base + - balancer/roundrobin + - codes + - connectivity + - credentials + - encoding + - encoding/proto + - grpclb/grpc_lb_v1/messages + - grpclog + - health + - health/grpc_health_v1 + - internal + - internal/backoff + - internal/channelz + - internal/grpcrand + - keepalive + - metadata + - naming + - peer + - resolver + - resolver/dns + - resolver/passthrough + - stats + - status + - tap + - transport - name: gopkg.in/inf.v0 version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 - name: gopkg.in/yaml.v2 @@ -135,6 +176,7 @@ imports: - name: k8s.io/api version: 2d6f90ab1293a1fb871cf149423ebb72aa7423aa subpackages: + - admission/v1beta1 - admissionregistration/v1alpha1 - admissionregistration/v1beta1 - apps/v1 @@ -258,4 +300,12 @@ imports: - util/homedir - util/integer - util/retry +- name: k8s.io/klog + version: 8139d8cb77af419532b33dfa7dd09fbc5f1d344f +- name: k8s.io/kubernetes + version: ddf47ac13c1a9483ea035a79cd7c10005ff21a6d + subpackages: + - pkg/kubelet/apis/podresources + - pkg/kubelet/apis/podresources/v1alpha1 + - pkg/kubelet/util testImports: [] diff --git a/glide.yaml b/glide.yaml index 8fbf50306..0c2265dae 100644 --- a/glide.yaml +++ b/glide.yaml @@ -32,5 +32,11 @@ import: - kubernetes - tools/clientcmd - util/retry +- package: k8s.io/kubernetes + version: v1.13.0 + subpackages: + - pkg/kubelet/apis/podresources + - pkg/kubelet/apis/podresources/v1alpha1 + - pkg/kubelet/util - package: github.com/vishvananda/netns - package: github.com/pkg/errors diff --git a/k8sclient/k8sclient.go b/k8sclient/k8sclient.go index 446e41f0b..886fc7da4 100644 --- a/k8sclient/k8sclient.go +++ b/k8sclient/k8sclient.go @@ -31,7 +31,7 @@ import ( "github.com/containernetworking/cni/libcni" "github.com/containernetworking/cni/pkg/skel" cnitypes "github.com/containernetworking/cni/pkg/types" - "github.com/intel/multus-cni/checkpoint" + "github.com/intel/multus-cni/kubeletclient" "github.com/intel/multus-cni/logging" "github.com/intel/multus-cni/types" ) @@ -137,16 +137,16 @@ func setPodNetworkAnnotation(client KubeClient, namespace string, pod *v1.Pod, n return pod, nil } -func getPodNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, string, string, error) { +func getPodNetworkAnnotation(client KubeClient, k8sArgs *types.K8sArgs) (string, string, error) { var err error logging.Debugf("getPodNetworkAnnotation: %v, %v", client, k8sArgs) pod, err := client.GetPod(string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) if err != nil { - return "", "", "", logging.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err) + return "", "", logging.Errorf("getPodNetworkAnnotation: failed to query the pod %v in out of cluster comm: %v", string(k8sArgs.K8S_POD_NAME), err) } - return pod.Annotations["k8s.v1.cni.cncf.io/networks"], pod.ObjectMeta.Namespace, string(pod.UID), nil + return pod.Annotations["k8s.v1.cni.cncf.io/networks"], pod.ObjectMeta.Namespace, nil } func parsePodNetworkObjectName(podnetwork string) (string, string, string, error) { @@ -332,7 +332,7 @@ func cniConfigFromNetworkResource(customResource *types.NetworkAttachmentDefinit return config, nil } -func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string, podID string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) { +func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement, confdir string, podName string, ns string, resourceMap map[string]*types.ResourceInfo) (*types.DelegateNetConf, map[string]*types.ResourceInfo, error) { logging.Debugf("getKubernetesDelegate: %v, %v, %s", client, net, confdir) rawPath := fmt.Sprintf("/apis/k8s.cni.cncf.io/v1/namespaces/%s/network-attachment-definitions/%s", net.Namespace, net.Name) @@ -349,18 +349,18 @@ func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement // Get resourceName annotation from NetworkAttachmentDefinition deviceID := "" resourceName, ok := customResource.Metadata.Annotations[resourceNameAnnot] - if ok && podID != "" { + if ok && podName != "" && ns != "" { // ResourceName annotation is found; try to get device info from resourceMap logging.Debugf("getKubernetesDelegate: found resourceName annotation : %s", resourceName) if resourceMap == nil { - checkpoint, err := checkpoint.GetCheckpoint() + ck, err := kubeletclient.GetResourceClient() if err != nil { - return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get a checkpoint instance: %v", err) + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get a ResourceClient instance: %v", err) } - resourceMap, err = checkpoint.GetComputeDeviceMap(podID) + resourceMap, err = ck.GetPodResourceMap(podName, ns) if err != nil { - return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get resourceMap from kubelet checkpoint file: %v", err) + return nil, resourceMap, logging.Errorf("getKubernetesDelegate: failed to get resourceMap from ResourceClient: %v", err) } logging.Debugf("getKubernetesDelegate(): resourceMap instance: %+v", resourceMap) } @@ -369,7 +369,7 @@ func getKubernetesDelegate(client KubeClient, net *types.NetworkSelectionElement if ok { if idCount := len(entry.DeviceIDs); idCount > 0 && idCount > entry.Index { deviceID = entry.DeviceIDs[entry.Index] - logging.Debugf("getKubernetesDelegate: podID: %s deviceID: %s", podID, deviceID) + logging.Debugf("getKubernetesDelegate: podName: %s deviceID: %s", podName, deviceID) entry.Index++ // increment Index for next delegate } } @@ -493,16 +493,14 @@ func GetK8sClient(kubeconfig string, kubeClient KubeClient) (KubeClient, error) func GetPodNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string, confnamespaceisolation bool) ([]*types.DelegateNetConf, error) { logging.Debugf("GetPodNetwork: %v, %v, %v", k8sclient, k8sArgs, confdir) + podName := string(k8sArgs.K8S_POD_NAME) + podNs := string(k8sArgs.K8S_POD_NAMESPACE) - netAnnot, defaultNamespace, podID, err := getPodNetworkAnnotation(k8sclient, k8sArgs) + netAnnot, defaultNamespace, err := getPodNetworkAnnotation(k8sclient, k8sArgs) if err != nil { return nil, err } - if err != nil { - return nil, logging.Errorf("GetK8sNetwork: failed to get resourceMap for PodUID: %v %v", podID, err) - } - if len(netAnnot) == 0 { return nil, &NoK8sNetworkError{"no kubernetes network found"} } @@ -528,7 +526,7 @@ func GetPodNetwork(k8sclient KubeClient, k8sArgs *types.K8sArgs, confdir string, } } - delegate, updatedResourceMap, err := getKubernetesDelegate(k8sclient, net, confdir, podID, resourceMap) + delegate, updatedResourceMap, err := getKubernetesDelegate(k8sclient, net, confdir, podName, podNs, resourceMap) if err != nil { return nil, logging.Errorf("GetPodNetwork: failed getting the delegate: %v", err) } @@ -686,7 +684,7 @@ func tryLoadK8sPodDefaultNetwork(k8sArgs *types.K8sArgs, conf *types.NetConf, ku return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: more than one default network is specified: %s", netAnnot) } - delegate, _, err := getKubernetesDelegate(kubeClient, networks[0], conf.ConfDir, "", nil) + delegate, _, err := getKubernetesDelegate(kubeClient, networks[0], conf.ConfDir, "", "", nil) if err != nil { return nil, logging.Errorf("tryLoadK8sPodDefaultNetwork: failed getting the delegate: %v", err) } diff --git a/kubeletclient/kubeletclient.go b/kubeletclient/kubeletclient.go new file mode 100644 index 000000000..dda473509 --- /dev/null +++ b/kubeletclient/kubeletclient.go @@ -0,0 +1,90 @@ +package kubeletclient + +import ( + "time" + + "github.com/intel/multus-cni/logging" + "github.com/intel/multus-cni/types" + "golang.org/x/net/context" + "k8s.io/kubernetes/pkg/kubelet/apis/podresources" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" + "k8s.io/kubernetes/pkg/kubelet/util" +) + +const ( + defaultPodResourcesPath = "/var/lib/kubelet/pod-resources" + defaultPodResourcesMaxSize = 1024 * 1024 * 16 // 16 Mb +) + +var ( + kubeletSocket string +) + +// ResourceClient provides a kubelet Pod resource handle +type ResourceClient interface { + // GetPodResourceMap returns an instance of a map of Pod ResourceInfo given a (Pod name, namespace) tuple + GetPodResourceMap(name, ns string) (map[string]*types.ResourceInfo, error) +} + +type resourceClient struct { + resources []*podresourcesapi.PodResources +} + +// GetResourceClient returns an instance of ResourceClient interface initialized with Pod resource information +func GetResourceClient() (ResourceClient, error) { + + newClient := &resourceClient{} + if kubeletSocket == "" { + kubeletSocket = util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket) + } + + client, conn, err := podresources.GetClient(kubeletSocket, 10*time.Second, defaultPodResourcesMaxSize) + if err != nil { + return nil, logging.Errorf("GetResourceClient(): error getting grpc client: %v\n", err) + } + defer conn.Close() + + if err := newClient.getPodResources(client); err != nil { + return nil, logging.Errorf("GetResourceClient(): error getting resource client: %v\n", err) + } + + return newClient, nil +} + +func (rc *resourceClient) getPodResources(client podresourcesapi.PodResourcesListerClient) error { + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + resp, err := client.List(ctx, &podresourcesapi.ListPodResourcesRequest{}) + if err != nil { + return logging.Errorf("getPodResources(): %v.Get(_) = _, %v", client, err) + } + + rc.resources = resp.PodResources + return nil +} + +// GetPodResourceMap returns an instance of a map of Pod ResourceInfo given a (Pod name, namespace) tuple +func (rc *resourceClient) GetPodResourceMap(name, ns string) (map[string]*types.ResourceInfo, error) { + resourceMap := make(map[string]*types.ResourceInfo) + + if name == "" || ns == "" { + return nil, logging.Errorf("GetPodResourcesMap(): Pod name or namespace cannot be empty") + } + + for _, pr := range rc.resources { + if pr.Name == name && pr.Namespace == ns { + for _, cnt := range pr.Containers { + for _, dev := range cnt.Devices { + if rInfo, ok := resourceMap[dev.ResourceName]; ok { + rInfo.DeviceIDs = append(rInfo.DeviceIDs, dev.DeviceIds...) + } else { + resourceMap[dev.ResourceName] = &types.ResourceInfo{DeviceIDs: dev.DeviceIds} + } + } + } + } + } + return resourceMap, nil +} diff --git a/kubeletclient/kubeletclient_test.go b/kubeletclient/kubeletclient_test.go new file mode 100644 index 000000000..687f61a60 --- /dev/null +++ b/kubeletclient/kubeletclient_test.go @@ -0,0 +1,150 @@ +package kubeletclient + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "google.golang.org/grpc" + "k8s.io/kubernetes/pkg/kubelet/util" + + mtypes "github.com/intel/multus-cni/types" + podresourcesapi "k8s.io/kubernetes/pkg/kubelet/apis/podresources/v1alpha1" +) + +var ( + socketDir string + socketName string + fakeServer *fakeResourceServer +) + +type fakeResourceServer struct { + server *grpc.Server +} + +func (m *fakeResourceServer) List(ctx context.Context, req *podresourcesapi.ListPodResourcesRequest) (*podresourcesapi.ListPodResourcesResponse, error) { + podName := "pod-name" + podNamespace := "pod-namespace" + containerName := "container-name" + + devs := []*podresourcesapi.ContainerDevices{ + { + ResourceName: "resource", + DeviceIds: []string{"dev0", "dev1"}, + }, + } + + resp := &podresourcesapi.ListPodResourcesResponse{ + PodResources: []*podresourcesapi.PodResources{ + { + Name: podName, + Namespace: podNamespace, + Containers: []*podresourcesapi.ContainerResources{ + { + Name: containerName, + Devices: devs, + }, + }, + }, + }, + } + return resp, nil +} + +func TestKubeletclient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Kubeletclient Suite") +} + +func setUp() error { + tempSocketDir, err := ioutil.TempDir("", "kubelet-resource-client") + if err != nil { + return err + } + socketDir = tempSocketDir + socketName = filepath.Join(socketDir, "kubelet.sock") + + fakeServer = &fakeResourceServer{server: grpc.NewServer()} + podresourcesapi.RegisterPodResourcesListerServer(fakeServer.server, fakeServer) + lis, err := util.CreateListener(socketName) + if err != nil { + return nil + } + go fakeServer.server.Serve(lis) + return nil +} + +func tearDown(path string) error { + if fakeServer != nil { + fakeServer.server.Stop() + } + if err := os.RemoveAll(path); err != nil { + return err + } + return nil +} + +var _ = BeforeSuite(func() { + err := setUp() + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + err := tearDown(socketDir) + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = Describe("Kubelet resource endpoint data read operations", func() { + var ( + client ResourceClient + ) + + Context("GetResourceClient()", func() { + It("should return no error", func() { + var err error + kubeletSocket = socketName + client, err = GetResourceClient() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("GetPodResourceMap() with valid pod name and namespace", func() { + It("should return no error", func() { + outputRMap := map[string]*mtypes.ResourceInfo{ + "resource": &mtypes.ResourceInfo{DeviceIDs: []string{"dev0", "dev1"}}, + } + resourceMap, err := client.GetPodResourceMap("pod-name", "pod-namespace") + Expect(err).NotTo(HaveOccurred()) + Expect(resourceMap).ShouldNot(BeNil()) + Expect(resourceMap).To(Equal(outputRMap)) + }) + }) + + Context("GetPodResourceMap() with empty podname", func() { + It("should return error", func() { + _, err := client.GetPodResourceMap("", "pod-namespace") + Expect(err).To(HaveOccurred()) + }) + }) + + Context("GetPodResourceMap() with empty namespace", func() { + It("should return error", func() { + _, err := client.GetPodResourceMap("pod-name", "") + Expect(err).To(HaveOccurred()) + }) + }) + + Context("GetPodResourceMap() with non-existent podname and namespace", func() { + It("should return no error", func() { + emptyRMap := map[string]*mtypes.ResourceInfo{} + resourceMap, err := client.GetPodResourceMap("whateverpod", "whatevernamespace") + Expect(err).NotTo(HaveOccurred()) + Expect(resourceMap).ShouldNot(BeNil()) + Expect(resourceMap).To(Equal(emptyRMap)) + }) + }) +})