From 946aff8847a27415ef3b824a3f524be69fb70936 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Wed, 15 Aug 2018 15:11:47 +0800 Subject: [PATCH 1/2] Add dns capabilities to CNI for windows containers --- pkg/kubelet/dockershim/docker_sandbox.go | 13 +++++- pkg/kubelet/dockershim/docker_service_test.go | 1 - pkg/kubelet/dockershim/network/cni/cni.go | 40 +++++++++++++++---- .../dockershim/network/cni/cni_others.go | 6 +++ .../dockershim/network/cni/cni_test.go | 2 +- .../dockershim/network/cni/cni_windows.go | 16 +++++++- .../network/kubenet/kubenet_linux.go | 2 +- .../network/kubenet/kubenet_unsupported.go | 2 +- pkg/kubelet/dockershim/network/plugins.go | 8 ++-- .../network/testing/mock_network_plugin.go | 2 +- .../network/testing/plugins_test.go | 8 ++-- 11 files changed, 77 insertions(+), 23 deletions(-) diff --git a/pkg/kubelet/dockershim/docker_sandbox.go b/pkg/kubelet/dockershim/docker_sandbox.go index b114bf052b468..744ebdf1c8e25 100644 --- a/pkg/kubelet/dockershim/docker_sandbox.go +++ b/pkg/kubelet/dockershim/docker_sandbox.go @@ -18,6 +18,7 @@ package dockershim import ( "context" + "encoding/json" "fmt" "os" "strings" @@ -27,7 +28,6 @@ import ( dockercontainer "github.com/docker/docker/api/types/container" dockerfilters "github.com/docker/docker/api/types/filters" "github.com/golang/glog" - utilerrors "k8s.io/apimachinery/pkg/util/errors" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" @@ -165,7 +165,16 @@ func (ds *dockerService) RunPodSandbox(ctx context.Context, r *runtimeapi.RunPod // on the host as well, to satisfy parts of the pod spec that aren't // recognized by the CNI standard yet. cID := kubecontainer.BuildContainerID(runtimeName, createResp.ID) - err = ds.network.SetUpPod(config.GetMetadata().Namespace, config.GetMetadata().Name, cID, config.Annotations) + networkOptions := make(map[string]string) + if dnsConfig := config.GetDnsConfig(); dnsConfig != nil { + // Build DNS options. + dnsOption, err := json.Marshal(dnsConfig) + if err != nil { + return nil, fmt.Errorf("failed to marshal dns config for pod %q: %v", config.Metadata.Name, err) + } + networkOptions["dns"] = string(dnsOption) + } + err = ds.network.SetUpPod(config.GetMetadata().Namespace, config.GetMetadata().Name, cID, config.Annotations, networkOptions) if err != nil { errList := []error{fmt.Errorf("failed to set up sandbox container %q network for pod %q: %v", createResp.ID, config.Metadata.Name, err)} diff --git a/pkg/kubelet/dockershim/docker_service_test.go b/pkg/kubelet/dockershim/docker_service_test.go index cc42ef5cabb4b..a1a0ee3e479b0 100644 --- a/pkg/kubelet/dockershim/docker_service_test.go +++ b/pkg/kubelet/dockershim/docker_service_test.go @@ -27,7 +27,6 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "k8s.io/apimachinery/pkg/util/clock" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" "k8s.io/kubernetes/pkg/kubelet/checkpointmanager" diff --git a/pkg/kubelet/dockershim/network/cni/cni.go b/pkg/kubelet/dockershim/network/cni/cni.go index c8fbdbb315f3e..901614b0fe19c 100644 --- a/pkg/kubelet/dockershim/network/cni/cni.go +++ b/pkg/kubelet/dockershim/network/cni/cni.go @@ -17,6 +17,7 @@ limitations under the License. package cni import ( + "encoding/json" "errors" "fmt" "sort" @@ -27,6 +28,7 @@ import ( cnitypes "github.com/containernetworking/cni/pkg/types" "github.com/golang/glog" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/network" "k8s.io/kubernetes/pkg/util/bandwidth" @@ -89,6 +91,18 @@ type cniIpRange struct { Subnet string `json:"subnet"` } +// cniDNSConfig maps to the windows CNI dns Capability. +// see: https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md +// Note that dns capability is only used for Windows containers. +type cniDNSConfig struct { + // List of DNS servers of the cluster. + Servers []string `json:"servers,omitempty"` + // List of DNS search domains of the cluster. + Searches []string `json:"searches,omitempty"` + // List of DNS options. + Options []string `json:"options,omitempty"` +} + func SplitDirs(dirs string) []string { // Use comma rather than colon to work better with Windows too return strings.Split(dirs, ",") @@ -256,7 +270,7 @@ func (plugin *cniNetworkPlugin) Status() error { return plugin.checkInitialized() } -func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error { if err := plugin.checkInitialized(); err != nil { return err } @@ -267,12 +281,12 @@ func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubec // Windows doesn't have loNetwork. It comes only with Linux if plugin.loNetwork != nil { - if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath, annotations); err != nil { + if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath, annotations, options); err != nil { return err } } - _, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations) + _, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations, options) return err } @@ -294,8 +308,8 @@ func podDesc(namespace, name string, id kubecontainer.ContainerID) string { return fmt.Sprintf("%s_%s/%s", namespace, name, id.ID) } -func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) (cnitypes.Result, error) { - rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations) +func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations, options map[string]string) (cnitypes.Result, error) { + rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations, options) if err != nil { glog.Errorf("Error adding network when building cni runtime conf: %v", err) return nil, err @@ -314,7 +328,7 @@ func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string } func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) error { - rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations) + rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations, nil) if err != nil { glog.Errorf("Error deleting network when building cni runtime conf: %v", err) return err @@ -334,7 +348,7 @@ func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName s return nil } -func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) (*libcni.RuntimeConf, error) { +func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations, options map[string]string) (*libcni.RuntimeConf, error) { rt := &libcni.RuntimeConf{ ContainerID: podSandboxID.ID, NetNS: podNetnsPath, @@ -389,5 +403,17 @@ func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string // Set the PodCIDR rt.CapabilityArgs["ipRanges"] = [][]cniIpRange{{{Subnet: plugin.podCidr}}} + // Set dns capability args. + if dnsOptions, ok := options["dns"]; ok { + dnsConfig := runtimeapi.DNSConfig{} + err := json.Unmarshal([]byte(dnsOptions), &dnsConfig) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal dns config %q: %v", dnsOptions, err) + } + if dnsParam := buildDNSCapabilities(&dnsConfig); dnsParam != nil { + rt.CapabilityArgs["dns"] = *dnsParam + } + } + return rt, nil } diff --git a/pkg/kubelet/dockershim/network/cni/cni_others.go b/pkg/kubelet/dockershim/network/cni/cni_others.go index 56f75ca3a960c..1712f8abe3c49 100644 --- a/pkg/kubelet/dockershim/network/cni/cni_others.go +++ b/pkg/kubelet/dockershim/network/cni/cni_others.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/containernetworking/cni/libcni" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/network" ) @@ -75,3 +76,8 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin return &network.PodNetworkStatus{IP: ip}, nil } + +// buildDNSCapabilities builds cniDNSConfig from runtimeapi.DNSConfig. +func buildDNSCapabilities(dnsConfig *runtimeapi.DNSConfig) *cniDNSConfig { + return nil +} diff --git a/pkg/kubelet/dockershim/network/cni/cni_test.go b/pkg/kubelet/dockershim/network/cni/cni_test.go index 5d83d991e3bc6..963017698e18d 100644 --- a/pkg/kubelet/dockershim/network/cni/cni_test.go +++ b/pkg/kubelet/dockershim/network/cni/cni_test.go @@ -254,7 +254,7 @@ func TestCNIPlugin(t *testing.T) { bandwidthAnnotation["kubernetes.io/egress-bandwidth"] = "1M" // Set up the pod - err = plug.SetUpPod("podNamespace", "podName", containerID, bandwidthAnnotation) + err = plug.SetUpPod("podNamespace", "podName", containerID, bandwidthAnnotation, nil) if err != nil { t.Errorf("Expected nil: %v", err) } diff --git a/pkg/kubelet/dockershim/network/cni/cni_windows.go b/pkg/kubelet/dockershim/network/cni/cni_windows.go index 0c90444550185..29f8f31ee5f86 100644 --- a/pkg/kubelet/dockershim/network/cni/cni_windows.go +++ b/pkg/kubelet/dockershim/network/cni/cni_windows.go @@ -23,6 +23,7 @@ import ( cniTypes020 "github.com/containernetworking/cni/pkg/types/020" "github.com/golang/glog" + runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/dockershim/network" ) @@ -42,7 +43,7 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err) } - result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, nil) + result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, nil, nil) glog.V(5).Infof("GetPodNetworkStatus result %+v", result) if err != nil { @@ -59,3 +60,16 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin } return &network.PodNetworkStatus{IP: result020.IP4.IP.IP}, nil } + +// buildDNSCapabilities builds cniDNSConfig from runtimeapi.DNSConfig. +func buildDNSCapabilities(dnsConfig *runtimeapi.DNSConfig) *cniDNSConfig { + if dnsConfig != nil { + return &cniDNSConfig{ + Servers: dnsConfig.Servers, + Searches: dnsConfig.Searches, + Options: dnsConfig.Options, + } + } + + return nil +} diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go index 0aec8c38cf76e..3611a84aec374 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go @@ -379,7 +379,7 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube return nil } -func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error { plugin.mu.Lock() defer plugin.mu.Unlock() diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go index ebb56e40ec3a8..d2a59126b3e82 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_unsupported.go @@ -42,7 +42,7 @@ func (plugin *kubenetNetworkPlugin) Name() string { return "kubenet" } -func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error { return fmt.Errorf("Kubenet is not supported in this build") } diff --git a/pkg/kubelet/dockershim/network/plugins.go b/pkg/kubelet/dockershim/network/plugins.go index bcef49f77c58d..139d237e652a5 100644 --- a/pkg/kubelet/dockershim/network/plugins.go +++ b/pkg/kubelet/dockershim/network/plugins.go @@ -63,7 +63,7 @@ type NetworkPlugin interface { // SetUpPod is the method called after the infra container of // the pod has been created but before the other containers of the // pod are launched. - SetUpPod(namespace string, name string, podSandboxID kubecontainer.ContainerID, annotations map[string]string) error + SetUpPod(namespace string, name string, podSandboxID kubecontainer.ContainerID, annotations, options map[string]string) error // TearDownPod is the method called before a pod's infra container will be deleted TearDownPod(namespace string, name string, podSandboxID kubecontainer.ContainerID) error @@ -207,7 +207,7 @@ func (plugin *NoopNetworkPlugin) Capabilities() utilsets.Int { return utilsets.NewInt() } -func (plugin *NoopNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (plugin *NoopNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error { return nil } @@ -368,14 +368,14 @@ func (pm *PluginManager) GetPodNetworkStatus(podNamespace, podName string, id ku return netStatus, nil } -func (pm *PluginManager) SetUpPod(podNamespace, podName string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (pm *PluginManager) SetUpPod(podNamespace, podName string, id kubecontainer.ContainerID, annotations, options map[string]string) error { defer recordOperation("set_up_pod", time.Now()) fullPodName := kubecontainer.BuildPodFullName(podName, podNamespace) pm.podLock(fullPodName).Lock() defer pm.podUnlock(fullPodName) glog.V(3).Infof("Calling network plugin %s to set up pod %q", pm.plugin.Name(), fullPodName) - if err := pm.plugin.SetUpPod(podNamespace, podName, id, annotations); err != nil { + if err := pm.plugin.SetUpPod(podNamespace, podName, id, annotations, options); err != nil { return fmt.Errorf("NetworkPlugin %s failed to set up pod %q network: %v", pm.plugin.Name(), fullPodName, err) } diff --git a/pkg/kubelet/dockershim/network/testing/mock_network_plugin.go b/pkg/kubelet/dockershim/network/testing/mock_network_plugin.go index 43e6d74490371..afb0752e1df09 100644 --- a/pkg/kubelet/dockershim/network/testing/mock_network_plugin.go +++ b/pkg/kubelet/dockershim/network/testing/mock_network_plugin.go @@ -102,7 +102,7 @@ func (_mr *_MockNetworkPluginRecorder) Name() *gomock.Call { return _mr.mock.ctrl.RecordCall(_mr.mock, "Name") } -func (_m *MockNetworkPlugin) SetUpPod(_param0 string, _param1 string, _param2 container.ContainerID, annotations map[string]string) error { +func (_m *MockNetworkPlugin) SetUpPod(_param0 string, _param1 string, _param2 container.ContainerID, annotations, options map[string]string) error { ret := _m.ctrl.Call(_m, "SetUpPod", _param0, _param1, _param2) ret0, _ := ret[0].(error) return ret0 diff --git a/pkg/kubelet/dockershim/network/testing/plugins_test.go b/pkg/kubelet/dockershim/network/testing/plugins_test.go index 29ae5b148f51b..42c6ba9e8e1ef 100644 --- a/pkg/kubelet/dockershim/network/testing/plugins_test.go +++ b/pkg/kubelet/dockershim/network/testing/plugins_test.go @@ -108,7 +108,7 @@ func TestPluginManager(t *testing.T) { // concurrently. allCreatedWg.Wait() - if err := pm.SetUpPod("", name, id, nil); err != nil { + if err := pm.SetUpPod("", name, id, nil, nil); err != nil { t.Errorf("Failed to set up pod %q: %v", name, err) return } @@ -159,7 +159,7 @@ func (p *hookableFakeNetworkPlugin) Capabilities() utilsets.Int { return utilsets.NewInt() } -func (p *hookableFakeNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +func (p *hookableFakeNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error { if p.setupHook != nil { p.setupHook(namespace, name, id) } @@ -210,7 +210,7 @@ func TestMultiPodParallelNetworkOps(t *testing.T) { // Setup will block on the runner pod completing. If network // operations locking isn't correct (eg pod network operations // block other pods) setUpPod() will never return. - if err := pm.SetUpPod("", podName, containerID, nil); err != nil { + if err := pm.SetUpPod("", podName, containerID, nil, nil); err != nil { t.Errorf("Failed to set up waiter pod: %v", err) return } @@ -230,7 +230,7 @@ func TestMultiPodParallelNetworkOps(t *testing.T) { podName := "runner" containerID := kubecontainer.ContainerID{ID: podName} - if err := pm.SetUpPod("", podName, containerID, nil); err != nil { + if err := pm.SetUpPod("", podName, containerID, nil, nil); err != nil { t.Errorf("Failed to set up runner pod: %v", err) return } From 32cf0a90c6c9e6359e72259ced7f8fe96e575b61 Mon Sep 17 00:00:00 2001 From: Pengfei Ni Date: Wed, 15 Aug 2018 16:08:31 +0800 Subject: [PATCH 2/2] Update bazel build --- pkg/kubelet/dockershim/network/cni/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/kubelet/dockershim/network/cni/BUILD b/pkg/kubelet/dockershim/network/cni/BUILD index 9396ba84e0345..6f9b6cfee9140 100644 --- a/pkg/kubelet/dockershim/network/cni/BUILD +++ b/pkg/kubelet/dockershim/network/cni/BUILD @@ -16,6 +16,7 @@ go_library( importpath = "k8s.io/kubernetes/pkg/kubelet/dockershim/network/cni", deps = [ "//pkg/kubelet/apis/config:go_default_library", + "//pkg/kubelet/apis/cri/runtime/v1alpha2:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/dockershim/network:go_default_library", "//pkg/util/bandwidth:go_default_library",