diff --git a/cni/azure-linux.conflist b/cni/azure-linux.conflist index 1d28e033d7..2e07c28d79 100644 --- a/cni/azure-linux.conflist +++ b/cni/azure-linux.conflist @@ -1,18 +1,21 @@ { - "cniVersion": "0.3.0", - "name": "azure", - "plugins": [{ - "type": "azure-vnet", - "mode": "bridge", - "bridge": "azure0", - "ipam": { - "type": "azure-vnet-ipam" - } - }, - { - "type": "portmap", - "capabilities": {"portMappings": true}, - "snat": true - } - ] -} + "cniVersion":"0.3.0", + "name":"azure", + "plugins":[ + { + "type":"azure-vnet", + "mode":"bridge", + "bridge":"azure0", + "ipam":{ + "type":"azure-vnet-ipam" + } + }, + { + "type":"portmap", + "capabilities":{ + "portMappings":true + }, + "snat":true + } + ] +} \ No newline at end of file diff --git a/cni/azure-windows.conflist b/cni/azure-windows.conflist index f1ab30a95d..ae6727f141 100644 --- a/cni/azure-windows.conflist +++ b/cni/azure-windows.conflist @@ -1,13 +1,43 @@ { - "cniVersion": "0.3.0", - "name": "azure", - "plugins": [{ - "type": "azure-vnet", - "mode": "bridge", - "bridge": "azure0", - "ipam": { - "type": "azure-vnet-ipam" - } + "cniVersion":"0.3.0", + "name":"azure", + "plugins":[ + { + "type":"azure-vnet", + "mode":"bridge", + "bridge":"azure0", + "ipam":{ + "type":"azure-vnet-ipam" + }, + "dns":{ + "Nameservers":[ + "168.63.129.16", + "10.0.0.10" + ], + "Search":[ + "svc.cluster.local" + ] + }, + "AdditionalArgs":[ + { + "Name":"EndpointPolicy", + "Value":{ + "Type":"OutBoundNAT", + "ExceptionList":[ + "10.240.0.0/16", + "10.0.0.0/8" + ] + } + }, + { + "Name":"EndpointPolicy", + "Value":{ + "Type":"ROUTE", + "DestinationPrefix":"10.0.0.0/8", + "NeedEncap":true + } } - ] + ] + } + ] } \ No newline at end of file diff --git a/cni/netconfig.go b/cni/netconfig.go index 5ec081e6ae..33252f27a5 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -5,8 +5,23 @@ package cni import ( "encoding/json" + "strings" + + "github.com/Azure/azure-container-networking/network/policy" + + cniTypes "github.com/containernetworking/cni/pkg/types" ) +const ( + PolicyStr string = "Policy" +) + +// KVPair represents a K-V pair of a json object. +type KVPair struct { + Name string `json:"name"` + Value json.RawMessage `json:"value"` +} + // NetworkConfig represents Azure CNI plugin network configuration. type NetworkConfig struct { CNIVersion string `json:"cniVersion"` @@ -25,6 +40,26 @@ type NetworkConfig struct { Address string `json:"ipAddress,omitempty"` QueryInterval string `json:"queryInterval,omitempty"` } + DNS cniTypes.DNS `json:"dns"` + AdditionalArgs []KVPair +} + +type K8SPodEnvArgs struct { + cniTypes.CommonArgs + K8S_POD_NAMESPACE cniTypes.UnmarshallableString `json:"K8S_POD_NAMESPACE,omitempty"` + K8S_POD_NAME cniTypes.UnmarshallableString `json:"K8S_POD_NAME,omitempty"` + K8S_POD_INFRA_CONTAINER_ID cniTypes.UnmarshallableString `json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` +} + +// ParseCniArgs unmarshals cni arguments. +func ParseCniArgs(args string) (*K8SPodEnvArgs, error) { + podCfg := K8SPodEnvArgs{} + err := cniTypes.LoadArgs(args, &podCfg) + if err != nil { + return nil, err + } + + return &podCfg, nil } // ParseNetworkConfig unmarshals network configuration from bytes. @@ -43,6 +78,22 @@ func ParseNetworkConfig(b []byte) (*NetworkConfig, error) { return &nwCfg, nil } +// GetPoliciesFromNwCfg returns network policies from network config. +func GetPoliciesFromNwCfg(kvp []KVPair) []policy.Policy { + var policies []policy.Policy + for _, pair := range kvp { + if strings.Contains(pair.Name, PolicyStr) { + policy := policy.Policy{ + Type: policy.CNIPolicyType(pair.Name), + Data: pair.Value, + } + policies = append(policies, policy) + } + } + + return policies +} + // Serialize marshals a network configuration to bytes. func (nwcfg *NetworkConfig) Serialize() []byte { bytes, _ := json.Marshal(nwcfg) diff --git a/cni/network/network.go b/cni/network/network.go index fb4b258dc7..7afaf7bf01 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -5,6 +5,7 @@ package network import ( "net" + "strings" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/common" @@ -123,16 +124,46 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre // Add handles CNI add commands. func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { - var result *cniTypesCurr.Result - var err error + var ( + result *cniTypesCurr.Result + err error + nwCfg *cni.NetworkConfig + ipconfig *cniTypesCurr.IPConfig + epInfo *network.EndpointInfo + iface *cniTypesCurr.Interface + ) log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) - defer func() { log.Printf("[cni-net] ADD command completed with result:%+v err:%v.", result, err) }() + defer func() { + // Add Interfaces to result. + iface = &cniTypesCurr.Interface{ + Name: args.IfName, + } + result.Interfaces = append(result.Interfaces, iface) + + // Convert result to the requested CNI version. + res, err := result.GetAsVersion(nwCfg.CNIVersion) + if err != nil { + err = plugin.Error(err) + } + + // Output the result to stdout. + res.Print() + log.Printf("[cni-net] ADD command completed with result:%+v err:%v.", result, err) + }() + + // Parse Pod arguments. + podCfg, err := cni.ParseCniArgs(args.Args) + k8sNamespace := string(podCfg.K8S_POD_NAMESPACE) + if len(k8sNamespace) == 0 { + err = plugin.Errorf("No k8s pod namespace provided.") + return err + } // Parse network configuration from stdin. - nwCfg, err := cni.ParseNetworkConfig(args.StdinData) + nwCfg, err = cni.ParseNetworkConfig(args.StdinData) if err != nil { err = plugin.Errorf("Failed to parse network configuration: %v.", err) return err @@ -142,11 +173,31 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Initialize values from network config. networkId := nwCfg.Name - endpointId := plugin.GetEndpointID(args) + endpointId := network.GetEndpointID(args) + + nwInfo, nwInfoErr := plugin.nm.GetNetworkInfo(networkId) + + /* Handle consecutive ADD calls for infrastructure containers. + * This is a temporary work around for issue #57253 of Kubernetes. + * We can delete this if statement once they fix it. + * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 + */ + epInfo, _ = plugin.nm.GetEndpointInfo(networkId, endpointId) + if epInfo != nil { + result, err = handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg) + if err != nil { + return err + } + + if result != nil { + return nil + } + } + + policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) // Check whether the network already exists. - nwInfo, err := plugin.nm.GetNetworkInfo(networkId) - if err != nil { + if nwInfoErr != nil { // Network does not exist. log.Printf("[cni-net] Creating network %v.", networkId) @@ -158,7 +209,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } // Derive the subnet prefix from allocated IP address. - ipconfig := result.IPs[0] + ipconfig = result.IPs[0] subnetPrefix := ipconfig.Address subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask) @@ -201,6 +252,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { }, }, BridgeName: nwCfg.Bridge, + DNS: network.DNSInfo{ + Servers: nwCfg.DNS.Nameservers, + Suffix: strings.Join(nwCfg.DNS.Search, ","), + }, + Policies: policies, } err = plugin.nm.CreateNetwork(&nwInfo) @@ -223,7 +279,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { return err } - ipconfig := result.IPs[0] + ipconfig = result.IPs[0] // On failure, call into IPAM plugin to release the address. defer func() { @@ -235,11 +291,31 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { } // Initialize endpoint info. - epInfo := &network.EndpointInfo{ + var dns network.DNSInfo + if (len(nwCfg.DNS.Search) == 0) != (len(nwCfg.DNS.Nameservers) == 0) { + err = plugin.Errorf("Wrong DNS configuration: %+v", nwCfg.DNS) + return err + } + + if len(nwCfg.DNS.Search) > 0 { + dns = network.DNSInfo{ + Servers: nwCfg.DNS.Nameservers, + Suffix: strings.Join(nwCfg.DNS.Search, ","), + } + } else { + dns = network.DNSInfo{ + Suffix: result.DNS.Domain, + Servers: result.DNS.Nameservers, + } + } + + epInfo = &network.EndpointInfo{ Id: endpointId, ContainerID: args.ContainerID, NetNsPath: args.Netns, IfName: args.IfName, + DNS: dns, + Policies: policies, } // Populate addresses. @@ -252,10 +328,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } - // Populate DNS info. - epInfo.DNS.Suffix = result.DNS.Domain - epInfo.DNS.Servers = result.DNS.Nameservers - // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) err = plugin.nm.CreateEndpoint(networkId, epInfo) @@ -264,22 +336,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { return err } - // Add Interfaces to result. - iface := &cniTypesCurr.Interface{ - Name: epInfo.IfName, - } - result.Interfaces = append(result.Interfaces, iface) - - // Convert result to the requested CNI version. - res, err := result.GetAsVersion(nwCfg.CNIVersion) - if err != nil { - err = plugin.Error(err) - return err - } - - // Output the result to stdout. - res.Print() - return nil } @@ -303,7 +359,7 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { // Initialize values from network config. networkId := nwCfg.Name - endpointId := plugin.GetEndpointID(args) + endpointId := network.GetEndpointID(args) // Query the network. nwInfo, err := plugin.nm.GetNetworkInfo(networkId) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go new file mode 100644 index 0000000000..c3b0c858be --- /dev/null +++ b/cni/network/network_linux.go @@ -0,0 +1,10 @@ +package network + +import ( + cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" +) + +// handleConsecutiveAdd is a dummy function for Linux platform. +func handleConsecutiveAdd(containerId, endpointId string, nwInfo *NetworkInfo, nwCfg *NetworkConfig) (*cniTypesCurr.Result, error) { + return nil, nil +} diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go new file mode 100644 index 0000000000..d98be65894 --- /dev/null +++ b/cni/network/network_windows.go @@ -0,0 +1,58 @@ +package network + +import ( + "net" + + "github.com/Azure/azure-container-networking/cni" + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network" + "github.com/Microsoft/hcsshim" + + cniTypes "github.com/containernetworking/cni/pkg/types" + cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" +) + +/* handleConsecutiveAdd handles consecutive add calls for infrastructure containers on Windows platform. + * This is a temporary work around for issue #57253 of Kubernetes. + * We can delete this if statement once they fix it. + * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 + */ +func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { + hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(endpointId) + if hnsEndpoint != nil { + log.Printf("[net] Found existing endpoint through hcsshim: %+v", hnsEndpoint) + log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, containerId) + + err := hcsshim.HotAttachEndpoint(containerId, hnsEndpoint.Id) + if err != nil { + log.Printf("[cni-net] Failed to hot attach shared endpoint to container [%s], err:%v.", hnsEndpoint.Id, err) + return nil, err + } + + // Populate result. + address := nwInfo.Subnets[0].Prefix + address.IP = hnsEndpoint.IPAddress + result := &cniTypesCurr.Result{ + IPs: []*cniTypesCurr.IPConfig{ + { + Version: "4", + Address: address, + Gateway: net.ParseIP(hnsEndpoint.GatewayAddress), + }, + }, + Routes: []*cniTypes.Route{ + { + Dst: net.IPNet{net.IPv4zero, net.IPv4Mask(0, 0, 0, 0)}, + GW: net.ParseIP(hnsEndpoint.GatewayAddress), + }, + }, + } + + // Populate DNS servers. + result.DNS.Nameservers = nwCfg.DNS.Nameservers + + return result, nil + } + + return nil, nil +} diff --git a/cni/plugin.go b/cni/plugin.go index 25c7e66df3..af2127c609 100644 --- a/cni/plugin.go +++ b/cni/plugin.go @@ -158,16 +158,6 @@ func (plugin *Plugin) DelegateDel(pluginName string, nwCfg *NetworkConfig) error return nil } -// GetEndpointID returns a unique endpoint ID based on the CNI args. -func (plugin *Plugin) GetEndpointID(args *cniSkel.CmdArgs) string { - containerID := args.ContainerID - if len(containerID) > 8 { - containerID = containerID[:8] - } - - return containerID + "-" + args.IfName -} - // Error creates and logs a structured CNI error. func (plugin *Plugin) Error(err error) *cniTypes.Error { var cniErr *cniTypes.Error diff --git a/network/endpoint.go b/network/endpoint.go index cb88b87cbd..8631006c2d 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -7,6 +7,8 @@ import ( "net" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/policy" + cniSkel "github.com/containernetworking/cni/pkg/skel" ) // Endpoint represents a container network interface. @@ -30,6 +32,7 @@ type EndpointInfo struct { IPAddresses []net.IPNet Routes []RouteInfo DNS DNSInfo + Policies []policy.Policy Data map[string]interface{} } @@ -39,6 +42,12 @@ type RouteInfo struct { Gw net.IP } +// GetEndpointID returns a unique endpoint ID based on the CNI args. +func GetEndpointID(args *cniSkel.CmdArgs) string { + infraEpId, _ := ConstructEpName(args.ContainerID, args.Netns, args.IfName) + return infraEpId +} + // NewEndpoint creates a new endpoint in the network. func (nw *network) newEndpoint(epInfo *EndpointInfo) (*endpoint, error) { var ep *endpoint diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index f5e26e9b53..cd1e309f89 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" ) @@ -39,46 +40,24 @@ func ConstructEpName(containerID string, netNsPath string, ifName string) (strin return infraEpName, workloadEpName } +// HotAttachEndpoint is a wrapper of hcsshim's HotAttachEndpoint. +func (endpoint *EndpointInfo) HotAttachEndpoint(containerID string) error { + return hcsshim.HotAttachEndpoint(containerID, endpoint.Id) +} + // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { // Get Infrastructure containerID. Handle ADD calls for workload container. - infraEpName, workloadEpName := ConstructEpName(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) - - /* Handle consecutive ADD calls for infrastructure containers. - * This is a temporary work around for issue #57253 of Kubernetes. - * We can delete this if statement once they fix it. - * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 - */ - if workloadEpName == "" { - if nw.Endpoints[infraEpName] != nil { - log.Printf("[net] Found existing endpoint %v, return immediately.", infraEpName) - return nw.Endpoints[infraEpName], nil - } - } - - log.Printf("[net] infraEpName: %v", infraEpName) + infraEpName, _ := ConstructEpName(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) - hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(infraEpName) - if hnsEndpoint != nil { - log.Printf("[net] Found existing endpoint through hcsshim%v", infraEpName) - log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, epInfo.ContainerID) - if err := hcsshim.HotAttachEndpoint(epInfo.ContainerID, hnsEndpoint.Id); err != nil { - return nil, err - } - return nw.Endpoints[infraEpName], nil - } - - hnsEndpoint = &hcsshim.HNSEndpoint{ + hnsEndpoint := &hcsshim.HNSEndpoint{ Name: infraEpName, VirtualNetwork: nw.HnsId, DNSSuffix: epInfo.DNS.Suffix, DNSServerList: strings.Join(epInfo.DNS.Servers, ","), + Policies: policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies), } - //enable outbound NAT - var enableOutBoundNat = json.RawMessage(`{"Type": "OutBoundNAT"}`) - hnsEndpoint.Policies = append(hnsEndpoint.Policies, enableOutBoundNat) - // HNS currently supports only one IP address per endpoint. if epInfo.IPAddresses != nil { hnsEndpoint.IPAddress = epInfo.IPAddresses[0].IP diff --git a/network/network.go b/network/network.go index 3340dcf9ce..433c342dd2 100644 --- a/network/network.go +++ b/network/network.go @@ -7,6 +7,7 @@ import ( "net" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/policy" "github.com/Azure/azure-container-networking/platform" ) @@ -46,6 +47,7 @@ type NetworkInfo struct { Mode string Subnets []SubnetInfo DNS DNSInfo + Policies []policy.Policy BridgeName string Options map[string]interface{} } diff --git a/network/network_windows.go b/network/network_windows.go index 8e3ce4c429..29f23cf84d 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -7,8 +7,10 @@ package network import ( "encoding/json" + "strings" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" ) @@ -27,6 +29,9 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt hnsNetwork := &hcsshim.HNSNetwork{ Name: nwInfo.Id, NetworkAdapterName: extIf.Name, + DNSSuffix: nwInfo.DNS.Suffix, + DNSServerList: strings.Join(nwInfo.DNS.Servers, ","), + Policies: policy.SerializePolicies(policy.NetworkPolicy, nwInfo.Policies), } // Set network mode. diff --git a/network/policy/policy_linux.go b/network/policy/policy_linux.go new file mode 100644 index 0000000000..207f2e7b44 --- /dev/null +++ b/network/policy/policy_linux.go @@ -0,0 +1,3 @@ +package policy + +type Policy struct{} diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go new file mode 100644 index 0000000000..90cc65cf6c --- /dev/null +++ b/network/policy/policy_windows.go @@ -0,0 +1,29 @@ +package policy + +import ( + "encoding/json" +) + +type CNIPolicyType string + +const ( + NetworkPolicy CNIPolicyType = "NetworkPolicy" + EndpointPolicy CNIPolicyType = "EndpointPolicy" + OutBoundNatPolicy CNIPolicyType = "OutBoundNatPolicy" +) + +type Policy struct { + Type CNIPolicyType + Data json.RawMessage +} + +// SerializePolicies serializes policies to json. +func SerializePolicies(policyType CNIPolicyType, policies []Policy) []json.RawMessage { + var jsonPolicies []json.RawMessage + for _, policy := range policies { + if policy.Type == policyType { + jsonPolicies = append(jsonPolicies, policy.Data) + } + } + return jsonPolicies +}