From f11dfa643f8531814a60769b43f2ba9c9f749685 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Mon, 23 Apr 2018 16:48:53 -0700 Subject: [PATCH 01/20] configure dnssuffix & dnsServerList --- cni/netconfig.go | 3 +++ cni/network/network.go | 14 ++++++++++---- network/network_windows.go | 3 +++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cni/netconfig.go b/cni/netconfig.go index 5ec081e6ae..ee65f216ce 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -5,6 +5,8 @@ package cni import ( "encoding/json" + + cniTypes "github.com/containernetworking/cni/pkg/types" ) // NetworkConfig represents Azure CNI plugin network configuration. @@ -25,6 +27,7 @@ type NetworkConfig struct { Address string `json:"ipAddress,omitempty"` QueryInterval string `json:"queryInterval,omitempty"` } + DNS cniTypes.DNS } // ParseNetworkConfig unmarshals network configuration from bytes. diff --git a/cni/network/network.go b/cni/network/network.go index fb4b258dc7..3a375454a2 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -5,6 +5,7 @@ package network import ( "net" + "strings" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/common" @@ -19,7 +20,8 @@ import ( const ( // Plugin name. - name = "azure-vnet" + name = "azure-vnet" + podK8sNamespace = "default" ) // NetPlugin represents the CNI network plugin. @@ -201,6 +203,10 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { }, }, BridgeName: nwCfg.Bridge, + DNS: network.DNSInfo{ + Servers: nwCfg.DNS.Nameservers, + Suffix: strings.Join(nwCfg.DNS.Search, ","), + }, } err = plugin.nm.CreateNetwork(&nwInfo) @@ -252,9 +258,9 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } - // Populate DNS info. - epInfo.DNS.Suffix = result.DNS.Domain - epInfo.DNS.Servers = result.DNS.Nameservers + // Populate DNS info. This info is only used by Windows. Namespace is hardcoded to be "default" for now. + epInfo.DNS.Suffix = podK8sNamespace + "." + nwInfo.DNS.Suffix + epInfo.DNS.Servers = nwInfo.DNS.Servers // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) diff --git a/network/network_windows.go b/network/network_windows.go index 8e3ce4c429..f651570379 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -7,6 +7,7 @@ package network import ( "encoding/json" + "strings" "github.com/Azure/azure-container-networking/log" "github.com/Microsoft/hcsshim" @@ -27,6 +28,8 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt hnsNetwork := &hcsshim.HNSNetwork{ Name: nwInfo.Id, NetworkAdapterName: extIf.Name, + DNSSuffix: nwInfo.DNS.Suffix, + DNSServerList: strings.Join(nwInfo.DNS.Servers, ","), } // Set network mode. From 66793301ad67e00107b5f0d96699b891ab4f06d4 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Mon, 23 Apr 2018 17:00:42 -0700 Subject: [PATCH 02/20] add dns info to conflist --- cni/azure-linux.conflist | 37 ++++++++++++++++++++----------------- cni/azure-windows.conflist | 31 ++++++++++++++++++++----------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/cni/azure-linux.conflist b/cni/azure-linux.conflist index 1d28e033d7..2e07c28d79 100644 --- a/cni/azure-linux.conflist +++ b/cni/azure-linux.conflist @@ -1,18 +1,21 @@ { - "cniVersion": "0.3.0", - "name": "azure", - "plugins": [{ - "type": "azure-vnet", - "mode": "bridge", - "bridge": "azure0", - "ipam": { - "type": "azure-vnet-ipam" - } - }, - { - "type": "portmap", - "capabilities": {"portMappings": true}, - "snat": true - } - ] -} + "cniVersion":"0.3.0", + "name":"azure", + "plugins":[ + { + "type":"azure-vnet", + "mode":"bridge", + "bridge":"azure0", + "ipam":{ + "type":"azure-vnet-ipam" + } + }, + { + "type":"portmap", + "capabilities":{ + "portMappings":true + }, + "snat":true + } + ] +} \ No newline at end of file diff --git a/cni/azure-windows.conflist b/cni/azure-windows.conflist index f1ab30a95d..b5f7554dca 100644 --- a/cni/azure-windows.conflist +++ b/cni/azure-windows.conflist @@ -1,13 +1,22 @@ { - "cniVersion": "0.3.0", - "name": "azure", - "plugins": [{ - "type": "azure-vnet", - "mode": "bridge", - "bridge": "azure0", - "ipam": { - "type": "azure-vnet-ipam" - } - } - ] + "cniVersion":"0.3.0", + "name":"azure", + "dns":{ + "Nameservers":[ + "168.63.129.16" + ], + "Search":[ + "svc.local" + ] + }, + "plugins":[ + { + "type":"azure-vnet", + "mode":"bridge", + "bridge":"azure0", + "ipam":{ + "type":"azure-vnet-ipam" + } + } + ] } \ No newline at end of file From 53d08b62bd0961c428ba05b76fc1a346123062fc Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Tue, 24 Apr 2018 11:39:55 -0700 Subject: [PATCH 03/20] change case for dns info --- cni/azure-windows.conflist | 4 ++-- cni/netconfig.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cni/azure-windows.conflist b/cni/azure-windows.conflist index b5f7554dca..b1ede4d597 100644 --- a/cni/azure-windows.conflist +++ b/cni/azure-windows.conflist @@ -2,10 +2,10 @@ "cniVersion":"0.3.0", "name":"azure", "dns":{ - "Nameservers":[ + "nameservers":[ "168.63.129.16" ], - "Search":[ + "search":[ "svc.local" ] }, diff --git a/cni/netconfig.go b/cni/netconfig.go index ee65f216ce..f2381ec78a 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -27,7 +27,7 @@ type NetworkConfig struct { Address string `json:"ipAddress,omitempty"` QueryInterval string `json:"queryInterval,omitempty"` } - DNS cniTypes.DNS + DNS cniTypes.DNS `json:"dns"` } // ParseNetworkConfig unmarshals network configuration from bytes. From a098531e008babbca0fde81e62a5ccd5b41185b3 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Tue, 24 Apr 2018 14:48:09 -0700 Subject: [PATCH 04/20] remove hardcoding --- cni/netconfig.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cni/netconfig.go b/cni/netconfig.go index f2381ec78a..b6a1640015 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -43,6 +43,11 @@ func ParseNetworkConfig(b []byte) (*NetworkConfig, error) { nwCfg.CNIVersion = defaultVersion } + nwCfg.DNS = cniTypes.DNS{ + Nameservers: []string{"168.63.129.16"}, + Search: []string{"svc.local"}, + } + return &nwCfg, nil } From f8c48032af84d770a2bb5e3cdd4e6cf0a927e18d Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Tue, 24 Apr 2018 16:03:37 -0700 Subject: [PATCH 05/20] remove hardcoding --- cni/netconfig.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cni/netconfig.go b/cni/netconfig.go index b6a1640015..f2381ec78a 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -43,11 +43,6 @@ func ParseNetworkConfig(b []byte) (*NetworkConfig, error) { nwCfg.CNIVersion = defaultVersion } - nwCfg.DNS = cniTypes.DNS{ - Nameservers: []string{"168.63.129.16"}, - Search: []string{"svc.local"}, - } - return &nwCfg, nil } From 7a1ad42770be4834d468e1fa64db962614283899 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Thu, 26 Apr 2018 11:19:31 -0700 Subject: [PATCH 06/20] parse k8s pod info --- cni/azure-windows.conflist | 2 +- cni/netconfig.go | 18 ++++++++++++++++++ cni/network/network.go | 24 ++++++++++++++++-------- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/cni/azure-windows.conflist b/cni/azure-windows.conflist index b1ede4d597..7b26cf9b1d 100644 --- a/cni/azure-windows.conflist +++ b/cni/azure-windows.conflist @@ -6,7 +6,7 @@ "168.63.129.16" ], "search":[ - "svc.local" + "svc.cluster.local" ] }, "plugins":[ diff --git a/cni/netconfig.go b/cni/netconfig.go index f2381ec78a..3188a2af8d 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -30,6 +30,24 @@ type NetworkConfig struct { DNS cniTypes.DNS `json:"dns"` } +type K8SPodEnvArgs struct { + cniTypes.CommonArgs + K8S_POD_NAMESPACE cniTypes.UnmarshallableString `json:"K8S_POD_NAMESPACE,omitempty"` + K8S_POD_NAME cniTypes.UnmarshallableString `json:"K8S_POD_NAME,omitempty"` + K8S_POD_INFRA_CONTAINER_ID cniTypes.UnmarshallableString `json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` +} + +// ParseCniArgs unmarshals cni arguments. +func ParseCniArgs(args string) (*K8SPodEnvArgs, error) { + podCfg := K8SPodEnvArgs{} + err := cniTypes.LoadArgs(args, &podCfg) + if err != nil { + return nil, err + } + + return &podCfg, nil +} + // ParseNetworkConfig unmarshals network configuration from bytes. func ParseNetworkConfig(b []byte) (*NetworkConfig, error) { nwCfg := NetworkConfig{} diff --git a/cni/network/network.go b/cni/network/network.go index 3a375454a2..cd54c76506 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -20,8 +20,7 @@ import ( const ( // Plugin name. - name = "azure-vnet" - podK8sNamespace = "default" + name = "azure-vnet" ) // NetPlugin represents the CNI network plugin. @@ -133,6 +132,13 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { defer func() { log.Printf("[cni-net] ADD command completed with result:%+v err:%v.", result, err) }() + // Parse Pod arguments. + podCfg, err := cni.ParseCniArgs(args.Args) + k8sNamespace := "default" + if err == nil { + k8sNamespace = string(podCfg.K8S_POD_NAMESPACE) + } + // Parse network configuration from stdin. nwCfg, err := cni.ParseNetworkConfig(args.StdinData) if err != nil { @@ -203,10 +209,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { }, }, BridgeName: nwCfg.Bridge, - DNS: network.DNSInfo{ - Servers: nwCfg.DNS.Nameservers, - Suffix: strings.Join(nwCfg.DNS.Search, ","), - }, } err = plugin.nm.CreateNetwork(&nwInfo) @@ -240,6 +242,12 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { }() } + // Fill in DNS info. + nwInfo.DNS = network.DNSInfo{ + Servers: nwCfg.DNS.Nameservers, + Suffix: strings.Join(nwCfg.DNS.Search, ","), + } + // Initialize endpoint info. epInfo := &network.EndpointInfo{ Id: endpointId, @@ -258,8 +266,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } - // Populate DNS info. This info is only used by Windows. Namespace is hardcoded to be "default" for now. - epInfo.DNS.Suffix = podK8sNamespace + "." + nwInfo.DNS.Suffix + // Populate DNS info. This info is only used by Windows. + epInfo.DNS.Suffix = k8sNamespace + "." + nwInfo.DNS.Suffix epInfo.DNS.Servers = nwInfo.DNS.Servers // Create the endpoint. From 35167e5f152e385b576d71a46e23aac172a6fb85 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Sat, 28 Apr 2018 19:57:23 -0700 Subject: [PATCH 07/20] add AdditionalArgs and Dns info to conflist --- cni/azure-windows.conflist | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/cni/azure-windows.conflist b/cni/azure-windows.conflist index 7b26cf9b1d..b3e39236a8 100644 --- a/cni/azure-windows.conflist +++ b/cni/azure-windows.conflist @@ -1,14 +1,6 @@ { "cniVersion":"0.3.0", "name":"azure", - "dns":{ - "nameservers":[ - "168.63.129.16" - ], - "search":[ - "svc.cluster.local" - ] - }, "plugins":[ { "type":"azure-vnet", @@ -16,7 +8,28 @@ "bridge":"azure0", "ipam":{ "type":"azure-vnet-ipam" - } + }, + "dns":{ + "Nameservers":[ + "168.63.129.16", + "10.0.0.10" + ], + "Search":[ + "svc.cluster.local" + ] + }, + "AdditionalArgs":[ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [ + "10.240.0.0/16", + "10.0.0.0/8" + ] + } + } + ] } ] } \ No newline at end of file From 6d84f1469c4748fb06f4b8e9f5b2ec47b4524447 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Sat, 28 Apr 2018 20:41:13 -0700 Subject: [PATCH 08/20] serialize policies --- cni/netconfig.go | 12 +++++++++++- cni/network/network.go | 31 +++++++++++++++++++++---------- network/endpoint.go | 1 + network/endpoint_windows.go | 7 ++++--- network/network.go | 1 + network/network_windows.go | 11 +++++++++++ network/policy.go | 19 +++++++++++++++++++ 7 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 network/policy.go diff --git a/cni/netconfig.go b/cni/netconfig.go index 3188a2af8d..a408cc9010 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -6,9 +6,16 @@ package cni import ( "encoding/json" + "github.com/Azure/azure-container-networking/log" cniTypes "github.com/containernetworking/cni/pkg/types" ) +// KVPair represents a K-V pair of a json object. +type KVPair struct { + Name string `json:"name"` + Value json.RawMessage `json:"value"` +} + // NetworkConfig represents Azure CNI plugin network configuration. type NetworkConfig struct { CNIVersion string `json:"cniVersion"` @@ -27,7 +34,8 @@ type NetworkConfig struct { Address string `json:"ipAddress,omitempty"` QueryInterval string `json:"queryInterval,omitempty"` } - DNS cniTypes.DNS `json:"dns"` + DNS cniTypes.DNS `json:"dns"` + AdditionalArgs []KVPair } type K8SPodEnvArgs struct { @@ -52,6 +60,8 @@ func ParseCniArgs(args string) (*K8SPodEnvArgs, error) { func ParseNetworkConfig(b []byte) (*NetworkConfig, error) { nwCfg := NetworkConfig{} + log.Printf("-----\n\n\n\n%s\n\n\n\n-----", string(b[:])) + err := json.Unmarshal(b, &nwCfg) if err != nil { return nil, err diff --git a/cni/network/network.go b/cni/network/network.go index cd54c76506..aaa4e0bed9 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -209,6 +209,21 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { }, }, BridgeName: nwCfg.Bridge, + DNS: network.DNSInfo{ + Servers: nwCfg.DNS.Nameservers, + Suffix: strings.Join(nwCfg.DNS.Search, ","), + }, + } + + // Fill in policy info. + for _, pair := range nwCfg.AdditionalArgs { + if strings.Contains(pair.Name, "Policy") { + policy := network.Policy{ + Type: network.CNIPolicyType(pair.Name), + Data: pair.Value, + } + nwInfo.Policies = append(nwInfo.Policies, policy) + } } err = plugin.nm.CreateNetwork(&nwInfo) @@ -242,18 +257,18 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { }() } - // Fill in DNS info. - nwInfo.DNS = network.DNSInfo{ - Servers: nwCfg.DNS.Nameservers, - Suffix: strings.Join(nwCfg.DNS.Search, ","), - } - // Initialize endpoint info. epInfo := &network.EndpointInfo{ Id: endpointId, ContainerID: args.ContainerID, NetNsPath: args.Netns, IfName: args.IfName, + // Windows only DNS info. + DNS: network.DNSInfo{ + Suffix: k8sNamespace + "." + nwInfo.DNS.Suffix, + Servers: nwInfo.DNS.Servers, + }, + Policies: nwInfo.Policies, } // Populate addresses. @@ -266,10 +281,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } - // Populate DNS info. This info is only used by Windows. - epInfo.DNS.Suffix = k8sNamespace + "." + nwInfo.DNS.Suffix - epInfo.DNS.Servers = nwInfo.DNS.Servers - // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) err = plugin.nm.CreateEndpoint(networkId, epInfo) diff --git a/network/endpoint.go b/network/endpoint.go index cb88b87cbd..aa241d9c19 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -30,6 +30,7 @@ type EndpointInfo struct { IPAddresses []net.IPNet Routes []RouteInfo DNS DNSInfo + Policies []Policy Data map[string]interface{} } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index f5e26e9b53..00dc3867d6 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -73,11 +73,12 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { VirtualNetwork: nw.HnsId, DNSSuffix: epInfo.DNS.Suffix, DNSServerList: strings.Join(epInfo.DNS.Servers, ","), + Policies: SerializeNwPolicies(epInfo.Policies), } - //enable outbound NAT - var enableOutBoundNat = json.RawMessage(`{"Type": "OutBoundNAT"}`) - hnsEndpoint.Policies = append(hnsEndpoint.Policies, enableOutBoundNat) + // TODO: remove this. enable outbound NAT + // var enableOutBoundNat = json.RawMessage(`{"Type": "OutBoundNAT"}`) + // hnsEndpoint.Policies = append(hnsEndpoint.Policies, enableOutBoundNat) // HNS currently supports only one IP address per endpoint. if epInfo.IPAddresses != nil { diff --git a/network/network.go b/network/network.go index 3340dcf9ce..a599ea2d2c 100644 --- a/network/network.go +++ b/network/network.go @@ -46,6 +46,7 @@ type NetworkInfo struct { Mode string Subnets []SubnetInfo DNS DNSInfo + Policies []Policy BridgeName string Options map[string]interface{} } diff --git a/network/network_windows.go b/network/network_windows.go index f651570379..2c48a5a5f8 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -22,6 +22,16 @@ const ( // Windows implementation of route. type route interface{} +// SerializeNwPolicies serializes network policies to json. +func SerializeNwPolicies(policies []Policy) []json.RawMessage { + var jsonPolicies []json.RawMessage + for _, policy := range policies { + jsonPolicies = append(jsonPolicies, policy.Data) + } + + return jsonPolicies +} + // NewNetworkImpl creates a new container network. func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { // Initialize HNS network. @@ -30,6 +40,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt NetworkAdapterName: extIf.Name, DNSSuffix: nwInfo.DNS.Suffix, DNSServerList: strings.Join(nwInfo.DNS.Servers, ","), + Policies: SerializeNwPolicies(nwInfo.Policies), } // Set network mode. diff --git a/network/policy.go b/network/policy.go new file mode 100644 index 0000000000..8789e66a4a --- /dev/null +++ b/network/policy.go @@ -0,0 +1,19 @@ +// Copyright Microsoft Corp. +// All rights reserved. + +package network + +import ( + "encoding/json" +) + +type CNIPolicyType string + +const ( + OutBoundNatPolicy CNIPolicyType = "OutBoundNatPolicy" +) + +type Policy struct { + Type CNIPolicyType + Data json.RawMessage +} From c6bfe4b81782da84b085534a7afc91801806af4c Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Sun, 29 Apr 2018 16:04:56 -0700 Subject: [PATCH 09/20] program route info --- cni/azure-windows.conflist | 37 +++++++++++++++++++++++++++---------- cni/network/network.go | 19 +++++-------------- network/endpoint_windows.go | 2 +- network/network_windows.go | 12 +----------- network/policy.go | 36 +++++++++++++++++++++++++++++++++--- 5 files changed, 67 insertions(+), 39 deletions(-) diff --git a/cni/azure-windows.conflist b/cni/azure-windows.conflist index b3e39236a8..aec42eef18 100644 --- a/cni/azure-windows.conflist +++ b/cni/azure-windows.conflist @@ -19,16 +19,33 @@ ] }, "AdditionalArgs":[ - { - "Name": "EndpointPolicy", - "Value": { - "Type": "OutBoundNAT", - "ExceptionList": [ - "10.240.0.0/16", - "10.0.0.0/8" - ] - } - } + { + "Name":"EndpointPolicy", + "Value":{ + "Type":"OutBoundNAT", + "ExceptionList":[ + "10.240.0.0/16", + "10.0.0.0/8", + "10.124.24.0/23" + ] + } + }, + { + "Name":"EndpointPolicy", + "Value":{ + "Type":"ROUTE", + "DestinationPrefix":"10.0.0.0/8", + "NeedEncap":true + } + }, + { + "Name":"EndpointPolicy", + "Value":{ + "Type":"ROUTE", + "DestinationPrefix":"10.124.24.196/32", + "NeedEncap":true + } + } ] } ] diff --git a/cni/network/network.go b/cni/network/network.go index aaa4e0bed9..04bb0175e5 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -151,6 +151,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Initialize values from network config. networkId := nwCfg.Name endpointId := plugin.GetEndpointID(args) + policies := network.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) // Check whether the network already exists. nwInfo, err := plugin.nm.GetNetworkInfo(networkId) @@ -213,17 +214,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { Servers: nwCfg.DNS.Nameservers, Suffix: strings.Join(nwCfg.DNS.Search, ","), }, - } - - // Fill in policy info. - for _, pair := range nwCfg.AdditionalArgs { - if strings.Contains(pair.Name, "Policy") { - policy := network.Policy{ - Type: network.CNIPolicyType(pair.Name), - Data: pair.Value, - } - nwInfo.Policies = append(nwInfo.Policies, policy) - } + Policies: policies, } err = plugin.nm.CreateNetwork(&nwInfo) @@ -265,10 +256,10 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { IfName: args.IfName, // Windows only DNS info. DNS: network.DNSInfo{ - Suffix: k8sNamespace + "." + nwInfo.DNS.Suffix, - Servers: nwInfo.DNS.Servers, + Suffix: k8sNamespace + "." + strings.Join(nwCfg.DNS.Search, ","), + Servers: nwCfg.DNS.Nameservers, }, - Policies: nwInfo.Policies, + Policies: policies, } // Populate addresses. diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 00dc3867d6..ade60cd77f 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -73,7 +73,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { VirtualNetwork: nw.HnsId, DNSSuffix: epInfo.DNS.Suffix, DNSServerList: strings.Join(epInfo.DNS.Servers, ","), - Policies: SerializeNwPolicies(epInfo.Policies), + Policies: SerializePolicies(EndpointPolicy, epInfo.Policies), } // TODO: remove this. enable outbound NAT diff --git a/network/network_windows.go b/network/network_windows.go index 2c48a5a5f8..2a810bce0e 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -22,16 +22,6 @@ const ( // Windows implementation of route. type route interface{} -// SerializeNwPolicies serializes network policies to json. -func SerializeNwPolicies(policies []Policy) []json.RawMessage { - var jsonPolicies []json.RawMessage - for _, policy := range policies { - jsonPolicies = append(jsonPolicies, policy.Data) - } - - return jsonPolicies -} - // NewNetworkImpl creates a new container network. func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { // Initialize HNS network. @@ -40,7 +30,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt NetworkAdapterName: extIf.Name, DNSSuffix: nwInfo.DNS.Suffix, DNSServerList: strings.Join(nwInfo.DNS.Servers, ","), - Policies: SerializeNwPolicies(nwInfo.Policies), + Policies: SerializePolicies(NetworkPolicy, nwInfo.Policies), } // Set network mode. diff --git a/network/policy.go b/network/policy.go index 8789e66a4a..67bb58a889 100644 --- a/network/policy.go +++ b/network/policy.go @@ -1,15 +1,18 @@ -// Copyright Microsoft Corp. -// All rights reserved. - package network import ( "encoding/json" + "strings" + + "github.com/Azure/azure-container-networking/cni" ) type CNIPolicyType string const ( + PolicyStr string = "Policy" + NetworkPolicy CNIPolicyType = "NetworkPolicy" + EndpointPolicy CNIPolicyType = "EndpointPolicy" OutBoundNatPolicy CNIPolicyType = "OutBoundNatPolicy" ) @@ -17,3 +20,30 @@ type Policy struct { Type CNIPolicyType Data json.RawMessage } + +// SerializePolicies serializes policies to json. +func SerializePolicies(policyType CNIPolicyType, policies []Policy) []json.RawMessage { + var jsonPolicies []json.RawMessage + for _, policy := range policies { + if policy.Type == policyType { + jsonPolicies = append(jsonPolicies, policy.Data) + } + } + return jsonPolicies +} + +// GetPoliciesFromNwCfg returns network policies from network config. +func GetPoliciesFromNwCfg(kvp []cni.KVPair) []Policy { + var policies []Policy + for _, pair := range kvp { + if strings.Contains(pair.Name, PolicyStr) { + policy := Policy{ + Type: CNIPolicyType(pair.Name), + Data: pair.Value, + } + policies = append(policies, policy) + } + } + + return policies +} From d4792acc4fb7d6414f4d40d1de1c639f617e1503 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Mon, 30 Apr 2018 16:27:58 -0700 Subject: [PATCH 10/20] start addressign ipam ip leak --- cni/azure-windows.conflist | 11 +---------- cni/network/network.go | 31 +++++++++++++++++++++++++------ cni/plugin.go | 10 ---------- ipam/api.go | 4 ++++ network/api.go | 4 ++++ network/endpoint.go | 11 +++++++++++ network/endpoint_windows.go | 34 +++++++++++++++++++++++----------- 7 files changed, 68 insertions(+), 37 deletions(-) diff --git a/cni/azure-windows.conflist b/cni/azure-windows.conflist index aec42eef18..ae6727f141 100644 --- a/cni/azure-windows.conflist +++ b/cni/azure-windows.conflist @@ -25,8 +25,7 @@ "Type":"OutBoundNAT", "ExceptionList":[ "10.240.0.0/16", - "10.0.0.0/8", - "10.124.24.0/23" + "10.0.0.0/8" ] } }, @@ -37,14 +36,6 @@ "DestinationPrefix":"10.0.0.0/8", "NeedEncap":true } - }, - { - "Name":"EndpointPolicy", - "Value":{ - "Type":"ROUTE", - "DestinationPrefix":"10.124.24.196/32", - "NeedEncap":true - } } ] } diff --git a/cni/network/network.go b/cni/network/network.go index 04bb0175e5..b5980a36b7 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -9,12 +9,14 @@ import ( "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/ipam" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/telemetry" cniSkel "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types/current" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) @@ -150,9 +152,20 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Initialize values from network config. networkId := nwCfg.Name - endpointId := plugin.GetEndpointID(args) - policies := network.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) + endpointId := network.GetEndpointID(args) + /* Handle consecutive ADD calls for infrastructure containers. + * This is a temporary work around for issue #57253 of Kubernetes. + * We can delete this if statement once they fix it. + * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 + */ + ep, _ := plugin.nm.GetEndpointInfo(networkId, endpointId) + if ep != nil { + log.Printf("[cni-net] Endpoint already exists. Exit.") + return nil + } + policies := network.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) + var ipconfig *current.IPConfig // Check whether the network already exists. nwInfo, err := plugin.nm.GetNetworkInfo(networkId) if err != nil { @@ -167,7 +180,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } // Derive the subnet prefix from allocated IP address. - ipconfig := result.IPs[0] + ipconfig = result.IPs[0] subnetPrefix := ipconfig.Address subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask) @@ -237,7 +250,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { return err } - ipconfig := result.IPs[0] + ipconfig = result.IPs[0] // On failure, call into IPAM plugin to release the address. defer func() { @@ -275,11 +288,17 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) err = plugin.nm.CreateEndpoint(networkId, epInfo) - if err != nil { + if err != nil && err != ipam.ErrAddressExists { err = plugin.Errorf("Failed to create endpoint: %v", err) return err } + // Call IPAM to release the ip address. + if err == ipam.ErrAddressExists { + nwCfg.Ipam.Address = ipconfig.Address.IP.String() + plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) + } + // Add Interfaces to result. iface := &cniTypesCurr.Interface{ Name: epInfo.IfName, @@ -319,7 +338,7 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { // Initialize values from network config. networkId := nwCfg.Name - endpointId := plugin.GetEndpointID(args) + endpointId := network.GetEndpointID(args) // Query the network. nwInfo, err := plugin.nm.GetNetworkInfo(networkId) diff --git a/cni/plugin.go b/cni/plugin.go index 25c7e66df3..af2127c609 100644 --- a/cni/plugin.go +++ b/cni/plugin.go @@ -158,16 +158,6 @@ func (plugin *Plugin) DelegateDel(pluginName string, nwCfg *NetworkConfig) error return nil } -// GetEndpointID returns a unique endpoint ID based on the CNI args. -func (plugin *Plugin) GetEndpointID(args *cniSkel.CmdArgs) string { - containerID := args.ContainerID - if len(containerID) > 8 { - containerID = containerID[:8] - } - - return containerID + "-" + args.IfName -} - // Error creates and logs a structured CNI error. func (plugin *Plugin) Error(err error) *cniTypes.Error { var cniErr *cniTypes.Error diff --git a/ipam/api.go b/ipam/api.go index dc3be1d725..11d5956b1e 100644 --- a/ipam/api.go +++ b/ipam/api.go @@ -31,3 +31,7 @@ var ( OptAddressType = "azure.address.type" OptAddressTypeGateway = "gateway" ) + +var ( + ErrAddressExists = fmt.Errorf("No available address pools") +) diff --git a/network/api.go b/network/api.go index 4b6b93ba81..ea5d1c7d0d 100644 --- a/network/api.go +++ b/network/api.go @@ -18,3 +18,7 @@ var ( errEndpointInUse = fmt.Errorf("Endpoint is already joined to a sandbox") errEndpointNotInUse = fmt.Errorf("Endpoint is not joined to a sandbox") ) + +var ( + ErrEndpointNotFound = fmt.Errorf("Endpoint not found") +) diff --git a/network/endpoint.go b/network/endpoint.go index aa241d9c19..c895f3f844 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -7,6 +7,7 @@ import ( "net" "github.com/Azure/azure-container-networking/log" + cniSkel "github.com/containernetworking/cni/pkg/skel" ) // Endpoint represents a container network interface. @@ -40,6 +41,16 @@ type RouteInfo struct { Gw net.IP } +// GetEndpointID returns a unique endpoint ID based on the CNI args. +func GetEndpointID(args *cniSkel.CmdArgs) string { + containerID := args.ContainerID + if len(containerID) > 8 { + containerID = containerID[:8] + } + + return containerID + "-" + args.IfName +} + // NewEndpoint creates a new endpoint in the network. func (nw *network) newEndpoint(epInfo *EndpointInfo) (*endpoint, error) { var ep *endpoint diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index ade60cd77f..47d5eba6a6 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -42,25 +42,16 @@ func ConstructEpName(containerID string, netNsPath string, ifName string) (strin // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // Get Infrastructure containerID. Handle ADD calls for workload container. - infraEpName, workloadEpName := ConstructEpName(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) + infraEpName, _ := ConstructEpName(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) /* Handle consecutive ADD calls for infrastructure containers. * This is a temporary work around for issue #57253 of Kubernetes. * We can delete this if statement once they fix it. * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 */ - if workloadEpName == "" { - if nw.Endpoints[infraEpName] != nil { - log.Printf("[net] Found existing endpoint %v, return immediately.", infraEpName) - return nw.Endpoints[infraEpName], nil - } - } - - log.Printf("[net] infraEpName: %v", infraEpName) - hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(infraEpName) if hnsEndpoint != nil { - log.Printf("[net] Found existing endpoint through hcsshim%v", infraEpName) + log.Printf("[net] Found existing endpoint through hcsshim: %v", infraEpName) log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, epInfo.ContainerID) if err := hcsshim.HotAttachEndpoint(epInfo.ContainerID, hnsEndpoint.Id); err != nil { return nil, err @@ -68,6 +59,27 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { return nw.Endpoints[infraEpName], nil } + /* + if workloadEpName == "" { + if nw.Endpoints[infraEpName] != nil { + log.Printf("[net] Found existing endpoint %v, return immediately.", infraEpName) + return nw.Endpoints[infraEpName], nil + } + } + + log.Printf("[net] infraEpName: %v", infraEpName) + + + hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(infraEpName) + if hnsEndpoint != nil { + log.Printf("[net] Found existing endpoint through hcsshim: %v", infraEpName) + log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, epInfo.ContainerID) + if err := hcsshim.HotAttachEndpoint(epInfo.ContainerID, hnsEndpoint.Id); err != nil { + return nil, err + } + return nw.Endpoints[infraEpName], nil + } + */ hnsEndpoint = &hcsshim.HNSEndpoint{ Name: infraEpName, VirtualNetwork: nw.HnsId, From 41c627b085163b3523bafec8d1f1698ae94d5326 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Mon, 30 Apr 2018 21:48:33 -0700 Subject: [PATCH 11/20] fix 1) ip inconsistency in k8s & pod. 2) IP leak in IPAM. --- cni/network/network.go | 110 ++++++++++++++++++++++++------------ network/endpoint.go | 12 ++-- network/endpoint_windows.go | 38 +------------ 3 files changed, 81 insertions(+), 79 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index b5980a36b7..c378f16355 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -9,14 +9,14 @@ import ( "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/common" - "github.com/Azure/azure-container-networking/ipam" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/telemetry" + "github.com/Microsoft/hcsshim" cniSkel "github.com/containernetworking/cni/pkg/skel" - "github.com/containernetworking/cni/pkg/types/current" + cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) @@ -126,13 +126,35 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre // Add handles CNI add commands. func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { - var result *cniTypesCurr.Result - var err error + var ( + result *cniTypesCurr.Result + err error + nwCfg *cni.NetworkConfig + ipconfig *cniTypesCurr.IPConfig + epInfo *network.EndpointInfo + iface *cniTypesCurr.Interface + ) log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) - defer func() { log.Printf("[cni-net] ADD command completed with result:%+v err:%v.", result, err) }() + defer func() { + // Add Interfaces to result. + iface = &cniTypesCurr.Interface{ + Name: args.IfName, + } + result.Interfaces = append(result.Interfaces, iface) + + // Convert result to the requested CNI version. + res, err := result.GetAsVersion(nwCfg.CNIVersion) + if err != nil { + err = plugin.Error(err) + } + + // Output the result to stdout. + res.Print() + log.Printf("[cni-net] ADD command completed with result:%+v err:%v.", result, err) + }() // Parse Pod arguments. podCfg, err := cni.ParseCniArgs(args.Args) @@ -142,7 +164,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } // Parse network configuration from stdin. - nwCfg, err := cni.ParseNetworkConfig(args.StdinData) + nwCfg, err = cni.ParseNetworkConfig(args.StdinData) if err != nil { err = plugin.Errorf("Failed to parse network configuration: %v.", err) return err @@ -153,22 +175,58 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Initialize values from network config. networkId := nwCfg.Name endpointId := network.GetEndpointID(args) + + nwInfo, nwInfoErr := plugin.nm.GetNetworkInfo(networkId) + /* Handle consecutive ADD calls for infrastructure containers. * This is a temporary work around for issue #57253 of Kubernetes. * We can delete this if statement once they fix it. * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 */ - ep, _ := plugin.nm.GetEndpointInfo(networkId, endpointId) - if ep != nil { - log.Printf("[cni-net] Endpoint already exists. Exit.") - return nil + epInfo, _ = plugin.nm.GetEndpointInfo(networkId, endpointId) + if epInfo != nil { + log.Printf("[cni-net] Consecutive ADD call for the same endpoint %v", epInfo) + hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(endpointId) + if hnsEndpoint != nil { + log.Printf("[net] Found existing endpoint through hcsshim: %+v", hnsEndpoint) + log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, args.ContainerID) + + err = hcsshim.HotAttachEndpoint(args.ContainerID, hnsEndpoint.Id) + if err != nil { + log.Printf("[cni-net] Failed to hot attach shared endpoint to container [%v], err:%v.", epInfo, err) + return err + } + + // Populate result. + address := nwInfo.Subnets[0].Prefix + address.IP = hnsEndpoint.IPAddress + result = &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Version: "4", + Address: address, + Gateway: net.ParseIP(hnsEndpoint.GatewayAddress), + }, + }, + Routes: []*cniTypes.Route{ + { + Dst: net.IPNet{net.IPv4zero, net.IPv4Mask(0, 0, 0, 0)}, + GW: net.ParseIP(hnsEndpoint.GatewayAddress), + }, + }, + } + + // Populate DNS servers. + result.DNS.Nameservers = nwCfg.DNS.Nameservers + + return nil + } } policies := network.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) - var ipconfig *current.IPConfig + // Check whether the network already exists. - nwInfo, err := plugin.nm.GetNetworkInfo(networkId) - if err != nil { + if nwInfoErr != nil { // Network does not exist. log.Printf("[cni-net] Creating network %v.", networkId) @@ -262,7 +320,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } // Initialize endpoint info. - epInfo := &network.EndpointInfo{ + epInfo = &network.EndpointInfo{ Id: endpointId, ContainerID: args.ContainerID, NetNsPath: args.Netns, @@ -288,33 +346,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) err = plugin.nm.CreateEndpoint(networkId, epInfo) - if err != nil && err != ipam.ErrAddressExists { - err = plugin.Errorf("Failed to create endpoint: %v", err) - return err - } - - // Call IPAM to release the ip address. - if err == ipam.ErrAddressExists { - nwCfg.Ipam.Address = ipconfig.Address.IP.String() - plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) - } - - // Add Interfaces to result. - iface := &cniTypesCurr.Interface{ - Name: epInfo.IfName, - } - result.Interfaces = append(result.Interfaces, iface) - - // Convert result to the requested CNI version. - res, err := result.GetAsVersion(nwCfg.CNIVersion) if err != nil { - err = plugin.Error(err) + err = plugin.Errorf("Failed to create endpoint: %v", err) return err } - // Output the result to stdout. - res.Print() - return nil } diff --git a/network/endpoint.go b/network/endpoint.go index c895f3f844..3f3323074b 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -7,6 +7,7 @@ import ( "net" "github.com/Azure/azure-container-networking/log" + "github.com/Microsoft/hcsshim" cniSkel "github.com/containernetworking/cni/pkg/skel" ) @@ -43,12 +44,13 @@ type RouteInfo struct { // GetEndpointID returns a unique endpoint ID based on the CNI args. func GetEndpointID(args *cniSkel.CmdArgs) string { - containerID := args.ContainerID - if len(containerID) > 8 { - containerID = containerID[:8] - } + infraEpId, _ := ConstructEpName(args.ContainerID, args.Netns, args.IfName) + return infraEpId +} - return containerID + "-" + args.IfName +// HotAttachEndpoint is a wrapper of hcsshim's HotAttachEndpoint. +func (endpoint *EndpointInfo) HotAttachEndpoint(containerID string) error { + return hcsshim.HotAttachEndpoint(containerID, endpoint.Id) } // NewEndpoint creates a new endpoint in the network. diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 47d5eba6a6..6f3491e8c6 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -44,43 +44,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // Get Infrastructure containerID. Handle ADD calls for workload container. infraEpName, _ := ConstructEpName(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) - /* Handle consecutive ADD calls for infrastructure containers. - * This is a temporary work around for issue #57253 of Kubernetes. - * We can delete this if statement once they fix it. - * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 - */ - hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(infraEpName) - if hnsEndpoint != nil { - log.Printf("[net] Found existing endpoint through hcsshim: %v", infraEpName) - log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, epInfo.ContainerID) - if err := hcsshim.HotAttachEndpoint(epInfo.ContainerID, hnsEndpoint.Id); err != nil { - return nil, err - } - return nw.Endpoints[infraEpName], nil - } - - /* - if workloadEpName == "" { - if nw.Endpoints[infraEpName] != nil { - log.Printf("[net] Found existing endpoint %v, return immediately.", infraEpName) - return nw.Endpoints[infraEpName], nil - } - } - - log.Printf("[net] infraEpName: %v", infraEpName) - - - hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(infraEpName) - if hnsEndpoint != nil { - log.Printf("[net] Found existing endpoint through hcsshim: %v", infraEpName) - log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, epInfo.ContainerID) - if err := hcsshim.HotAttachEndpoint(epInfo.ContainerID, hnsEndpoint.Id); err != nil { - return nil, err - } - return nw.Endpoints[infraEpName], nil - } - */ - hnsEndpoint = &hcsshim.HNSEndpoint{ + hnsEndpoint := &hcsshim.HNSEndpoint{ Name: infraEpName, VirtualNetwork: nw.HnsId, DNSSuffix: epInfo.DNS.Suffix, From ec3f5ed854133b7a38bd5049a47160ede3a4f69e Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Mon, 30 Apr 2018 21:52:03 -0700 Subject: [PATCH 12/20] remove comments --- network/endpoint_windows.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 6f3491e8c6..6a55939549 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -52,10 +52,6 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { Policies: SerializePolicies(EndpointPolicy, epInfo.Policies), } - // TODO: remove this. enable outbound NAT - // var enableOutBoundNat = json.RawMessage(`{"Type": "OutBoundNAT"}`) - // hnsEndpoint.Policies = append(hnsEndpoint.Policies, enableOutBoundNat) - // HNS currently supports only one IP address per endpoint. if epInfo.IPAddresses != nil { hnsEndpoint.IPAddress = epInfo.IPAddresses[0].IP From 1dd10528c8a53e6129365ee8a691b17638efbedc Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Tue, 1 May 2018 15:10:46 -0700 Subject: [PATCH 13/20] separate windows & linux --- cni/network/network.go | 42 ++++-------------------- cni/network/network_linux.go | 11 +++++++ cni/network/network_windows.go | 58 ++++++++++++++++++++++++++++++++++ {network => cni}/policy.go | 6 ++-- network/endpoint.go | 3 +- network/endpoint_windows.go | 3 +- network/network.go | 3 +- network/network_windows.go | 3 +- 8 files changed, 85 insertions(+), 44 deletions(-) create mode 100644 cni/network/network_linux.go create mode 100644 cni/network/network_windows.go rename {network => cni}/policy.go (88%) diff --git a/cni/network/network.go b/cni/network/network.go index c378f16355..e6068a5ac9 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -13,10 +13,8 @@ import ( "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/telemetry" - "github.com/Microsoft/hcsshim" cniSkel "github.com/containernetworking/cni/pkg/skel" - cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) @@ -185,45 +183,17 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { */ epInfo, _ = plugin.nm.GetEndpointInfo(networkId, endpointId) if epInfo != nil { - log.Printf("[cni-net] Consecutive ADD call for the same endpoint %v", epInfo) - hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(endpointId) - if hnsEndpoint != nil { - log.Printf("[net] Found existing endpoint through hcsshim: %+v", hnsEndpoint) - log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, args.ContainerID) - - err = hcsshim.HotAttachEndpoint(args.ContainerID, hnsEndpoint.Id) - if err != nil { - log.Printf("[cni-net] Failed to hot attach shared endpoint to container [%v], err:%v.", epInfo, err) - return err - } - - // Populate result. - address := nwInfo.Subnets[0].Prefix - address.IP = hnsEndpoint.IPAddress - result = &cniTypesCurr.Result{ - IPs: []*cniTypesCurr.IPConfig{ - { - Version: "4", - Address: address, - Gateway: net.ParseIP(hnsEndpoint.GatewayAddress), - }, - }, - Routes: []*cniTypes.Route{ - { - Dst: net.IPNet{net.IPv4zero, net.IPv4Mask(0, 0, 0, 0)}, - GW: net.ParseIP(hnsEndpoint.GatewayAddress), - }, - }, - } - - // Populate DNS servers. - result.DNS.Nameservers = nwCfg.DNS.Nameservers + result, err = HandleConsecutiveAdd(args.ContainerID, endpointId, nwCfg.DNS.Nameservers, nwInfo, nwCfg) + if err != nil { + return err + } + if result != nil { return nil } } - policies := network.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) + policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) // Check whether the network already exists. if nwInfoErr != nil { diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go new file mode 100644 index 0000000000..76f2995967 --- /dev/null +++ b/cni/network/network_linux.go @@ -0,0 +1,11 @@ +package network + +import ( + "github.com/Azure/azure-container-networking/cni" + cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" +) + +// HandleConsecutiveAdd is a dummy function for Linux platform. +func HandleConsecutiveAdd(containerId, endpointId string, dnsServers []string, nwInfo *NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { + return nil, nil +} diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go new file mode 100644 index 0000000000..55842e3a78 --- /dev/null +++ b/cni/network/network_windows.go @@ -0,0 +1,58 @@ +package network + +import ( + "net" + + "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network" + "github.com/Microsoft/hcsshim" + + cniTypes "github.com/containernetworking/cni/pkg/types" + cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" +) + +/* HandleConsecutiveAdd handles consecutive add calls for infrastructure containers on Windows platform. + * This is a temporary work around for issue #57253 of Kubernetes. + * We can delete this if statement once they fix it. + * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 + */ +func HandleConsecutiveAdd(containerId, endpointId string, dnsServers []string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { + hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(endpointId) + if hnsEndpoint != nil { + log.Printf("[net] Found existing endpoint through hcsshim: %+v", hnsEndpoint) + log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, containerId) + + err := hcsshim.HotAttachEndpoint(containerId, hnsEndpoint.Id) + if err != nil { + log.Printf("[cni-net] Failed to hot attach shared endpoint to container [%s], err:%v.", hnsEndpoint.Id, err) + return nil, err + } + + // Populate result. + address := nwInfo.Subnets[0].Prefix + address.IP = hnsEndpoint.IPAddress + result := &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Version: "4", + Address: address, + Gateway: net.ParseIP(hnsEndpoint.GatewayAddress), + }, + }, + Routes: []*cniTypes.Route{ + { + Dst: net.IPNet{net.IPv4zero, net.IPv4Mask(0, 0, 0, 0)}, + GW: net.ParseIP(hnsEndpoint.GatewayAddress), + }, + }, + } + + // Populate DNS servers. + result.DNS.Nameservers = nwCfg.DNS.Nameservers + + return result, nil + } + + return nil, nil +} diff --git a/network/policy.go b/cni/policy.go similarity index 88% rename from network/policy.go rename to cni/policy.go index 67bb58a889..cd21cde5e5 100644 --- a/network/policy.go +++ b/cni/policy.go @@ -1,10 +1,8 @@ -package network +package cni import ( "encoding/json" "strings" - - "github.com/Azure/azure-container-networking/cni" ) type CNIPolicyType string @@ -33,7 +31,7 @@ func SerializePolicies(policyType CNIPolicyType, policies []Policy) []json.RawMe } // GetPoliciesFromNwCfg returns network policies from network config. -func GetPoliciesFromNwCfg(kvp []cni.KVPair) []Policy { +func GetPoliciesFromNwCfg(kvp []KVPair) []Policy { var policies []Policy for _, pair := range kvp { if strings.Contains(pair.Name, PolicyStr) { diff --git a/network/endpoint.go b/network/endpoint.go index 3f3323074b..dda99288ac 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -6,6 +6,7 @@ package network import ( "net" + "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/log" "github.com/Microsoft/hcsshim" cniSkel "github.com/containernetworking/cni/pkg/skel" @@ -32,7 +33,7 @@ type EndpointInfo struct { IPAddresses []net.IPNet Routes []RouteInfo DNS DNSInfo - Policies []Policy + Policies []cni.Policy Data map[string]interface{} } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 6a55939549..7070d96380 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -10,6 +10,7 @@ import ( "net" "strings" + "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/log" "github.com/Microsoft/hcsshim" ) @@ -49,7 +50,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { VirtualNetwork: nw.HnsId, DNSSuffix: epInfo.DNS.Suffix, DNSServerList: strings.Join(epInfo.DNS.Servers, ","), - Policies: SerializePolicies(EndpointPolicy, epInfo.Policies), + Policies: cni.SerializePolicies(cni.EndpointPolicy, epInfo.Policies), } // HNS currently supports only one IP address per endpoint. diff --git a/network/network.go b/network/network.go index a599ea2d2c..391d1673d9 100644 --- a/network/network.go +++ b/network/network.go @@ -6,6 +6,7 @@ package network import ( "net" + "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/platform" ) @@ -46,7 +47,7 @@ type NetworkInfo struct { Mode string Subnets []SubnetInfo DNS DNSInfo - Policies []Policy + Policies []cni.Policy BridgeName string Options map[string]interface{} } diff --git a/network/network_windows.go b/network/network_windows.go index 2a810bce0e..9bfda1d885 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -9,6 +9,7 @@ import ( "encoding/json" "strings" + "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/log" "github.com/Microsoft/hcsshim" ) @@ -30,7 +31,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt NetworkAdapterName: extIf.Name, DNSSuffix: nwInfo.DNS.Suffix, DNSServerList: strings.Join(nwInfo.DNS.Servers, ","), - Policies: SerializePolicies(NetworkPolicy, nwInfo.Policies), + Policies: cni.SerializePolicies(cni.NetworkPolicy, nwInfo.Policies), } // Set network mode. From d8a4d646c3fa5f9f3677eed6388a3a18326ad639 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Wed, 2 May 2018 14:02:00 -0700 Subject: [PATCH 14/20] remove dnsServers --- cni/network/network.go | 2 +- cni/network/network_windows.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index e6068a5ac9..5c89dd0746 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -183,7 +183,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { */ epInfo, _ = plugin.nm.GetEndpointInfo(networkId, endpointId) if epInfo != nil { - result, err = HandleConsecutiveAdd(args.ContainerID, endpointId, nwCfg.DNS.Nameservers, nwInfo, nwCfg) + result, err = HandleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg) if err != nil { return err } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 55842e3a78..c627001a40 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -17,7 +17,7 @@ import ( * We can delete this if statement once they fix it. * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 */ -func HandleConsecutiveAdd(containerId, endpointId string, dnsServers []string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { +func HandleConsecutiveAdd(containerId, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(endpointId) if hnsEndpoint != nil { log.Printf("[net] Found existing endpoint through hcsshim: %+v", hnsEndpoint) From 1a7a2d881a015c4dc1bd12f23987dd3885c6cc54 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Wed, 2 May 2018 15:46:11 -0700 Subject: [PATCH 15/20] remove comments --- cni/netconfig.go | 3 --- cni/network/network.go | 7 ++++--- cni/network/network_linux.go | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cni/netconfig.go b/cni/netconfig.go index a408cc9010..2347f5bd8c 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -6,7 +6,6 @@ package cni import ( "encoding/json" - "github.com/Azure/azure-container-networking/log" cniTypes "github.com/containernetworking/cni/pkg/types" ) @@ -60,8 +59,6 @@ func ParseCniArgs(args string) (*K8SPodEnvArgs, error) { func ParseNetworkConfig(b []byte) (*NetworkConfig, error) { nwCfg := NetworkConfig{} - log.Printf("-----\n\n\n\n%s\n\n\n\n-----", string(b[:])) - err := json.Unmarshal(b, &nwCfg) if err != nil { return nil, err diff --git a/cni/network/network.go b/cni/network/network.go index 5c89dd0746..5cdac9b0ad 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -156,9 +156,10 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Parse Pod arguments. podCfg, err := cni.ParseCniArgs(args.Args) - k8sNamespace := "default" - if err == nil { - k8sNamespace = string(podCfg.K8S_POD_NAMESPACE) + k8sNamespace := string(podCfg.K8S_POD_NAMESPACE) + if len(k8sNamespace) == 0 { + err = plugin.Errorf("No k8s pod namespace provided.") + return err } // Parse network configuration from stdin. diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 76f2995967..a87e6f18e4 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -6,6 +6,6 @@ import ( ) // HandleConsecutiveAdd is a dummy function for Linux platform. -func HandleConsecutiveAdd(containerId, endpointId string, dnsServers []string, nwInfo *NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { +func HandleConsecutiveAdd(containerId, endpointId string, nwInfo *NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { return nil, nil } From 201548607d4e3e6a562d50694187f44964911ae2 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Wed, 2 May 2018 18:04:11 -0700 Subject: [PATCH 16/20] address comments --- cni/netconfig.go | 23 ++++++++++++++++++ cni/network/network.go | 24 +++++++++++++------ cni/network/network_linux.go | 9 ++++--- cni/network/network_windows.go | 9 +++++-- ipam/api.go | 4 ---- network/api.go | 4 ---- network/endpoint.go | 10 ++------ network/endpoint_windows.go | 9 +++++-- network/network.go | 4 ++-- network/network_windows.go | 4 ++-- .../policy/policy_windows.go | 20 +--------------- 11 files changed, 67 insertions(+), 53 deletions(-) rename cni/policy.go => network/policy/policy_windows.go (58%) diff --git a/cni/netconfig.go b/cni/netconfig.go index 2347f5bd8c..33252f27a5 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -5,10 +5,17 @@ package cni import ( "encoding/json" + "strings" + + "github.com/Azure/azure-container-networking/network/policy" cniTypes "github.com/containernetworking/cni/pkg/types" ) +const ( + PolicyStr string = "Policy" +) + // KVPair represents a K-V pair of a json object. type KVPair struct { Name string `json:"name"` @@ -71,6 +78,22 @@ func ParseNetworkConfig(b []byte) (*NetworkConfig, error) { return &nwCfg, nil } +// GetPoliciesFromNwCfg returns network policies from network config. +func GetPoliciesFromNwCfg(kvp []KVPair) []policy.Policy { + var policies []policy.Policy + for _, pair := range kvp { + if strings.Contains(pair.Name, PolicyStr) { + policy := policy.Policy{ + Type: policy.CNIPolicyType(pair.Name), + Data: pair.Value, + } + policies = append(policies, policy) + } + } + + return policies +} + // Serialize marshals a network configuration to bytes. func (nwcfg *NetworkConfig) Serialize() []byte { bytes, _ := json.Marshal(nwcfg) diff --git a/cni/network/network.go b/cni/network/network.go index 5cdac9b0ad..b37636c67f 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -184,7 +184,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { */ epInfo, _ = plugin.nm.GetEndpointInfo(networkId, endpointId) if epInfo != nil { - result, err = HandleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg) + result, err = handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg) if err != nil { return err } @@ -291,17 +291,25 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } // Initialize endpoint info. + var dns network.DNSInfo + if len(nwCfg.DNS.Search) > 0 { + dns = network.DNSInfo{ + Servers: nwCfg.DNS.Nameservers, + Suffix: strings.Join(nwCfg.DNS.Search, ","), + } + } else { + dns = network.DNSInfo{ + Suffix: result.DNS.Domain, + Servers: result.DNS.Nameservers, + } + } + epInfo = &network.EndpointInfo{ Id: endpointId, ContainerID: args.ContainerID, NetNsPath: args.Netns, IfName: args.IfName, - // Windows only DNS info. - DNS: network.DNSInfo{ - Suffix: k8sNamespace + "." + strings.Join(nwCfg.DNS.Search, ","), - Servers: nwCfg.DNS.Nameservers, - }, - Policies: policies, + DNS: dns, } // Populate addresses. @@ -314,6 +322,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } + setPolicies(epInfo, policies) + // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) err = plugin.nm.CreateEndpoint(networkId, epInfo) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index a87e6f18e4..9f7627d93a 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -1,11 +1,14 @@ package network import ( - "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/network" + "github.com/Azure/azure-container-networking/network/policy" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) -// HandleConsecutiveAdd is a dummy function for Linux platform. -func HandleConsecutiveAdd(containerId, endpointId string, nwInfo *NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { +// handleConsecutiveAdd is a dummy function for Linux platform. +func handleConsecutiveAdd(containerId, endpointId string, nwInfo *NetworkInfo, nwCfg *NetworkConfig) (*cniTypesCurr.Result, error) { return nil, nil } + +func setPolicies(epInfo *network.EndpointInfo, policies []policy.Policy) {} diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index c627001a40..5f95c0a0f0 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -6,18 +6,19 @@ import ( "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" + "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) -/* HandleConsecutiveAdd handles consecutive add calls for infrastructure containers on Windows platform. +/* handleConsecutiveAdd handles consecutive add calls for infrastructure containers on Windows platform. * This is a temporary work around for issue #57253 of Kubernetes. * We can delete this if statement once they fix it. * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 */ -func HandleConsecutiveAdd(containerId, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { +func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(endpointId) if hnsEndpoint != nil { log.Printf("[net] Found existing endpoint through hcsshim: %+v", hnsEndpoint) @@ -56,3 +57,7 @@ func HandleConsecutiveAdd(containerId, endpointId string, nwInfo *network.Networ return nil, nil } + +func setPolicies(epInfo *network.EndpointInfo, policies []policy.Policy) { + epInfo.Policies = policies +} diff --git a/ipam/api.go b/ipam/api.go index 11d5956b1e..dc3be1d725 100644 --- a/ipam/api.go +++ b/ipam/api.go @@ -31,7 +31,3 @@ var ( OptAddressType = "azure.address.type" OptAddressTypeGateway = "gateway" ) - -var ( - ErrAddressExists = fmt.Errorf("No available address pools") -) diff --git a/network/api.go b/network/api.go index ea5d1c7d0d..4b6b93ba81 100644 --- a/network/api.go +++ b/network/api.go @@ -18,7 +18,3 @@ var ( errEndpointInUse = fmt.Errorf("Endpoint is already joined to a sandbox") errEndpointNotInUse = fmt.Errorf("Endpoint is not joined to a sandbox") ) - -var ( - ErrEndpointNotFound = fmt.Errorf("Endpoint not found") -) diff --git a/network/endpoint.go b/network/endpoint.go index dda99288ac..8631006c2d 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -6,9 +6,8 @@ package network import ( "net" - "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/log" - "github.com/Microsoft/hcsshim" + "github.com/Azure/azure-container-networking/network/policy" cniSkel "github.com/containernetworking/cni/pkg/skel" ) @@ -33,7 +32,7 @@ type EndpointInfo struct { IPAddresses []net.IPNet Routes []RouteInfo DNS DNSInfo - Policies []cni.Policy + Policies []policy.Policy Data map[string]interface{} } @@ -49,11 +48,6 @@ func GetEndpointID(args *cniSkel.CmdArgs) string { return infraEpId } -// HotAttachEndpoint is a wrapper of hcsshim's HotAttachEndpoint. -func (endpoint *EndpointInfo) HotAttachEndpoint(containerID string) error { - return hcsshim.HotAttachEndpoint(containerID, endpoint.Id) -} - // NewEndpoint creates a new endpoint in the network. func (nw *network) newEndpoint(epInfo *EndpointInfo) (*endpoint, error) { var ep *endpoint diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 7070d96380..cd1e309f89 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -10,8 +10,8 @@ import ( "net" "strings" - "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" ) @@ -40,6 +40,11 @@ func ConstructEpName(containerID string, netNsPath string, ifName string) (strin return infraEpName, workloadEpName } +// HotAttachEndpoint is a wrapper of hcsshim's HotAttachEndpoint. +func (endpoint *EndpointInfo) HotAttachEndpoint(containerID string) error { + return hcsshim.HotAttachEndpoint(containerID, endpoint.Id) +} + // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // Get Infrastructure containerID. Handle ADD calls for workload container. @@ -50,7 +55,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { VirtualNetwork: nw.HnsId, DNSSuffix: epInfo.DNS.Suffix, DNSServerList: strings.Join(epInfo.DNS.Servers, ","), - Policies: cni.SerializePolicies(cni.EndpointPolicy, epInfo.Policies), + Policies: policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies), } // HNS currently supports only one IP address per endpoint. diff --git a/network/network.go b/network/network.go index 391d1673d9..433c342dd2 100644 --- a/network/network.go +++ b/network/network.go @@ -6,8 +6,8 @@ package network import ( "net" - "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" ) @@ -47,7 +47,7 @@ type NetworkInfo struct { Mode string Subnets []SubnetInfo DNS DNSInfo - Policies []cni.Policy + Policies []policy.Policy BridgeName string Options map[string]interface{} } diff --git a/network/network_windows.go b/network/network_windows.go index 9bfda1d885..29f23cf84d 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -9,8 +9,8 @@ import ( "encoding/json" "strings" - "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" ) @@ -31,7 +31,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt NetworkAdapterName: extIf.Name, DNSSuffix: nwInfo.DNS.Suffix, DNSServerList: strings.Join(nwInfo.DNS.Servers, ","), - Policies: cni.SerializePolicies(cni.NetworkPolicy, nwInfo.Policies), + Policies: policy.SerializePolicies(policy.NetworkPolicy, nwInfo.Policies), } // Set network mode. diff --git a/cni/policy.go b/network/policy/policy_windows.go similarity index 58% rename from cni/policy.go rename to network/policy/policy_windows.go index cd21cde5e5..90cc65cf6c 100644 --- a/cni/policy.go +++ b/network/policy/policy_windows.go @@ -1,14 +1,12 @@ -package cni +package policy import ( "encoding/json" - "strings" ) type CNIPolicyType string const ( - PolicyStr string = "Policy" NetworkPolicy CNIPolicyType = "NetworkPolicy" EndpointPolicy CNIPolicyType = "EndpointPolicy" OutBoundNatPolicy CNIPolicyType = "OutBoundNatPolicy" @@ -29,19 +27,3 @@ func SerializePolicies(policyType CNIPolicyType, policies []Policy) []json.RawMe } return jsonPolicies } - -// GetPoliciesFromNwCfg returns network policies from network config. -func GetPoliciesFromNwCfg(kvp []KVPair) []Policy { - var policies []Policy - for _, pair := range kvp { - if strings.Contains(pair.Name, PolicyStr) { - policy := Policy{ - Type: CNIPolicyType(pair.Name), - Data: pair.Value, - } - policies = append(policies, policy) - } - } - - return policies -} From 2bc7f5c69e2b815fcbac655de8968cc87f23bece Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Thu, 3 May 2018 14:42:08 -0700 Subject: [PATCH 17/20] add dns verification --- cni/network/network.go | 5 +++++ network/policy/policy_linux.go | 10 ++++++++++ 2 files changed, 15 insertions(+) create mode 100644 network/policy/policy_linux.go diff --git a/cni/network/network.go b/cni/network/network.go index b37636c67f..3838e8ee74 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -292,6 +292,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Initialize endpoint info. var dns network.DNSInfo + if (len(nwCfg.DNS.Search) == 0) != (len(nwCfg.DNS.Nameservers) == 0) { + err = plugin.Errorf("Wrong DNS configuration: %+v", nwCfg.DNS) + return err + } + if len(nwCfg.DNS.Search) > 0 { dns = network.DNSInfo{ Servers: nwCfg.DNS.Nameservers, diff --git a/network/policy/policy_linux.go b/network/policy/policy_linux.go new file mode 100644 index 0000000000..0efa618c5b --- /dev/null +++ b/network/policy/policy_linux.go @@ -0,0 +1,10 @@ +package policy + +import ( + "encoding/json" +) + +type Policy struct { + Type CNIPolicyType + Data json.RawMessage +} From 06f7d3aaaafa85ffd206af4e02eb015bae7fde04 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Thu, 3 May 2018 14:44:47 -0700 Subject: [PATCH 18/20] abstract linux policy struct --- network/policy/policy_linux.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/network/policy/policy_linux.go b/network/policy/policy_linux.go index 0efa618c5b..207f2e7b44 100644 --- a/network/policy/policy_linux.go +++ b/network/policy/policy_linux.go @@ -1,10 +1,3 @@ package policy -import ( - "encoding/json" -) - -type Policy struct { - Type CNIPolicyType - Data json.RawMessage -} +type Policy struct{} From fc861cb33572b9eb839f1fbf9e571dd45a66b135 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Thu, 3 May 2018 14:51:14 -0700 Subject: [PATCH 19/20] remove setPolicies --- cni/network/network.go | 3 +-- cni/network/network_linux.go | 4 ---- cni/network/network_windows.go | 5 ----- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 3838e8ee74..7afaf7bf01 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -315,6 +315,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { NetNsPath: args.Netns, IfName: args.IfName, DNS: dns, + Policies: policies, } // Populate addresses. @@ -327,8 +328,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } - setPolicies(epInfo, policies) - // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) err = plugin.nm.CreateEndpoint(networkId, epInfo) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 9f7627d93a..c3b0c858be 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -1,8 +1,6 @@ package network import ( - "github.com/Azure/azure-container-networking/network" - "github.com/Azure/azure-container-networking/network/policy" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) @@ -10,5 +8,3 @@ import ( func handleConsecutiveAdd(containerId, endpointId string, nwInfo *NetworkInfo, nwCfg *NetworkConfig) (*cniTypesCurr.Result, error) { return nil, nil } - -func setPolicies(epInfo *network.EndpointInfo, policies []policy.Policy) {} diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 5f95c0a0f0..d98be65894 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -6,7 +6,6 @@ import ( "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" - "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" cniTypes "github.com/containernetworking/cni/pkg/types" @@ -57,7 +56,3 @@ func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.Networ return nil, nil } - -func setPolicies(epInfo *network.EndpointInfo, policies []policy.Policy) { - epInfo.Policies = policies -} From 890bcb99c906bc8bfd60a657448500a34f9d1535 Mon Sep 17 00:00:00 2001 From: Yongli Chen Date: Thu, 3 May 2018 15:31:36 -0700 Subject: [PATCH 20/20] separte Windows & Linux code --- cni/network/network.go | 6 ++++++ network/endpoint.go | 29 ++++++++++++++++++++++++----- network/endpoint_windows.go | 27 +-------------------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 7afaf7bf01..c6b258e75a 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -117,6 +117,12 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre return "" } +// GetEndpointID returns a unique endpoint ID based on the CNI args. +func GetEndpointID(args *cniSkel.CmdArgs) string { + infraEpId, _ := network.ConstructEndpointID(args.ContainerID, args.Netns, args.IfName) + return infraEpId +} + // // CNI implementation // https://github.com/containernetworking/cni/blob/master/SPEC.md diff --git a/network/endpoint.go b/network/endpoint.go index 8631006c2d..a1d2f1c753 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -5,10 +5,10 @@ package network import ( "net" + "strings" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/policy" - cniSkel "github.com/containernetworking/cni/pkg/skel" ) // Endpoint represents a container network interface. @@ -42,10 +42,29 @@ type RouteInfo struct { Gw net.IP } -// GetEndpointID returns a unique endpoint ID based on the CNI args. -func GetEndpointID(args *cniSkel.CmdArgs) string { - infraEpId, _ := ConstructEpName(args.ContainerID, args.Netns, args.IfName) - return infraEpId +// ConstructEndpointID constructs endpoint name from netNsPath. +func ConstructEndpointID(containerID string, netNsPath string, ifName string) (string, string) { + infraEpName, workloadEpName := "", "" + + if len(containerID) > 8 { + containerID = containerID[:8] + } + + if netNsPath != "" { + splits := strings.Split(netNsPath, ":") + // For workload containers, we extract its linking infrastructure container ID. + if len(splits) == 2 { + if len(splits[1]) > 8 { + splits[1] = splits[1][:8] + } + infraEpName = splits[1] + "-" + ifName + workloadEpName = containerID + "-" + ifName + } else { + // For infrastructure containers, we just use its container ID. + infraEpName = containerID + "-" + ifName + } + } + return infraEpName, workloadEpName } // NewEndpoint creates a new endpoint in the network. diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index cd1e309f89..80a57e47f3 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -15,31 +15,6 @@ import ( "github.com/Microsoft/hcsshim" ) -// ConstructEpName constructs endpoint name from netNsPath. -func ConstructEpName(containerID string, netNsPath string, ifName string) (string, string) { - infraEpName, workloadEpName := "", "" - - if len(containerID) > 8 { - containerID = containerID[:8] - } - - if netNsPath != "" { - splits := strings.Split(netNsPath, ":") - // For workload containers, we extract its linking infrastructure container ID. - if len(splits) == 2 { - if len(splits[1]) > 8 { - splits[1] = splits[1][:8] - } - infraEpName = splits[1] + "-" + ifName - workloadEpName = containerID + "-" + ifName - } else { - // For infrastructure containers, we just use its container ID. - infraEpName = containerID + "-" + ifName - } - } - return infraEpName, workloadEpName -} - // HotAttachEndpoint is a wrapper of hcsshim's HotAttachEndpoint. func (endpoint *EndpointInfo) HotAttachEndpoint(containerID string) error { return hcsshim.HotAttachEndpoint(containerID, endpoint.Id) @@ -48,7 +23,7 @@ func (endpoint *EndpointInfo) HotAttachEndpoint(containerID string) error { // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // Get Infrastructure containerID. Handle ADD calls for workload container. - infraEpName, _ := ConstructEpName(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) + infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) hnsEndpoint := &hcsshim.HNSEndpoint{ Name: infraEpName,