From 258aee2a191e09daf34f6cf35e3516db9629805a Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Thu, 31 Jan 2019 09:28:30 -0800 Subject: [PATCH 01/37] Support hyper-v containers with containerd (*COW) --- cni/network/network.go | 3 +- cni/network/network_linux.go | 2 +- cni/network/network_windows.go | 14 +- network/endpoint.go | 1 + network/endpoint_windows.go | 156 +++++++++++++++++++++ network/network.go | 2 + network/network_windows.go | 161 +++++++++++++++++++++- network/policy/policy.go | 1 + network/policy/policy_windows.go | 223 ++++++++++++++++++++++++++++--- vendor.conf | 2 +- 10 files changed, 540 insertions(+), 25 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 61e620effb..3ba1e16d71 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -302,7 +302,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { */ epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId) if epInfo != nil { - resultConsAdd, errConsAdd := handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg) + resultConsAdd, errConsAdd := handleConsecutiveAdd(args.ContainerID, endpointId, args.Netns, nwInfo, nwCfg) if errConsAdd != nil { log.Printf("handleConsecutiveAdd failed with error %v", errConsAdd) result = resultConsAdd @@ -393,6 +393,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { EnableSnatOnHost: nwCfg.EnableSnatOnHost, DNS: nwDNSInfo, Policies: policies, + NetNs: args.Netns, } nwInfo.Options = make(map[string]interface{}) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index 6a8ccbf88d..d74a37bda4 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -19,7 +19,7 @@ const ( ) // handleConsecutiveAdd is a dummy function for Linux platform. -func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { +func handleConsecutiveAdd(containerId, endpointId, netNs string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { return nil, nil } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index ad0a0db3c0..98c2df9c13 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -13,6 +13,8 @@ import ( "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" + "github.com/Microsoft/hcsshim/hcn" + "github.com/google/uuid" cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" @@ -23,7 +25,17 @@ import ( * We can delete this if statement once they fix it. * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 */ -func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { +func handleConsecutiveAdd(containerId, endpointId, netNs string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { + // Check if the netNs is a valid GUID to decide on HNSv1 or HNSv2 + if _, err := uuid.Parse(netNs); err == nil { + if err = hcn.V2ApiSupported(); err != nil { + log.Printf("HNSV2 is not supported on this windows platform") + return nil, err + } + + return nil, nil + } + hnsEndpoint, err := hcsshim.GetHNSEndpointByName(endpointId) if hnsEndpoint != nil { log.Printf("[net] Found existing endpoint through hcsshim: %+v", hnsEndpoint) diff --git a/network/endpoint.go b/network/endpoint.go index b7c44e584d..fd0f4cba64 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -37,6 +37,7 @@ type endpoint struct { PODName string `json:",omitempty"` PODNameSpace string `json:",omitempty"` InfraVnetAddressSpace string `json:",omitempty"` + NetNs string } // EndpointInfo contains read-only information about an endpoint. diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 742d338c21..6a61735537 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -11,6 +11,8 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" + "github.com/Microsoft/hcsshim/hcn" + "github.com/google/uuid" ) // HotAttachEndpoint is a wrapper of hcsshim's HotAttachEndpoint. @@ -44,6 +46,21 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { + // Check if the netNsPath is a valid GUID to decide on HNSv1 or HNSv2 + if _, err := uuid.Parse(epInfo.NetNsPath); err == nil { + if err = hcn.V2ApiSupported(); err != nil { + log.Printf("HNSV2 is not supported on this windows platform") + return nil, err + } + + return nw.newEndpointImplHnsV2(epInfo) + } + + return nw.newEndpointImplHnsV1(epInfo) +} + +// newEndpointImplHnsV1 creates a new endpoint in the network using HnsV1 +func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo) (*endpoint, error) { var vlanid int if epInfo.Data != nil { @@ -113,6 +130,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { DNS: epInfo.DNS, VlanID: vlanid, EnableSnatOnHost: epInfo.EnableSnatOnHost, + NetNs: epInfo.NetNsPath, } for _, route := range epInfo.Routes { @@ -124,9 +142,133 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { return ep, nil } +// newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 +func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { + var vlanid int + if epInfo.Data != nil { + if _, ok := epInfo.Data[VlanIDKey]; ok { + vlanid = epInfo.Data[VlanIDKey].(int) + } + } + + infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) + + hnsEndpoint := &hcn.HostComputeEndpoint{ + Name: infraEpName, + HostComputeNetwork: nw.HnsId, + //HostComputeNamespace: epInfo.NetNsPath, + Dns: hcn.Dns{ + Domain: epInfo.DNS.Suffix, + ServerList: epInfo.DNS.Servers, + }, + SchemaVersion: hcn.SchemaVersion{ + Major: 2, + Minor: 0, + }, + MacAddress: epInfo.MacAddress.String(), + } + + if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { + for _, epPolicy := range endpointPolicies { + hnsEndpoint.Policies = append(hnsEndpoint.Policies, epPolicy) + } + } else { + log.Printf("[net] Failed to get endpoint policies due to error: %v", err) + return nil, err + } + + for _, route := range epInfo.Routes { + hcnRoute := hcn.Route{ + NextHop: route.Gw.String(), + DestinationPrefix: route.Dst.String(), + } + + hnsEndpoint.Routes = append(hnsEndpoint.Routes, hcnRoute) + } + + for _, ipAddress := range epInfo.IPAddresses { + prefixLength, _ := ipAddress.Mask.Size() + ipConfiguration := hcn.IpConfig{ + IpAddress: ipAddress.IP.String(), + PrefixLength: uint8(prefixLength), + } + hnsEndpoint.IpConfigurations = append(hnsEndpoint.IpConfigurations, ipConfiguration) + } + + // Create the HNS endpoint. + log.Printf("[net] HostComputeEndpoint CREATE: %+v", hnsEndpoint) + hnsResponse, err := hnsEndpoint.Create() + log.Printf("[net] HostComputeEndpoint CREATE response: %+v err: %v", hnsResponse, err) + if err != nil { + return nil, err + } + + defer func() { + if err != nil { + log.Printf("[net] HostComputeEndpoint DELETE id: %v", hnsResponse.Id) + err = hnsResponse.Delete() + log.Printf("[net] HostComputeEndpoint DELETE err: %v", err) + } + }() + + log.Printf("[net] GetNamespaceByID id: %s", epInfo.NetNsPath) + namespace, e := hcn.GetNamespaceByID(epInfo.NetNsPath) + log.Printf("[net] GetNamespaceByID result: %+v", namespace) + if e != nil { + err = e + log.Printf("[net] GetNamespaceByID err: %v", e) + return nil, err + } + + log.Printf("[net] AddNamespaceEndpoint ns id: %s, ep id: %s", namespace.Id, hnsResponse.Id) + if e = hcn.AddNamespaceEndpoint(namespace.Id, hnsResponse.Id); e != nil { + log.Printf("[net] AddNamespaceEndpoint err %v", e) + err = e + return nil, err + } + + // Create the endpoint object. + ep := &endpoint{ + Id: infraEpName, + HnsId: hnsResponse.Id, + SandboxKey: epInfo.ContainerID, + IfName: epInfo.IfName, + IPAddresses: epInfo.IPAddresses, + Gateways: []net.IP{net.ParseIP(hnsResponse.Routes[0].NextHop)}, + DNS: epInfo.DNS, + VlanID: vlanid, + EnableSnatOnHost: epInfo.EnableSnatOnHost, + NetNs: epInfo.NetNsPath, + } + + for _, route := range epInfo.Routes { + ep.Routes = append(ep.Routes, route) + } + + ep.MacAddress, _ = net.ParseMAC(hnsResponse.MacAddress) + + return ep, nil + +} + // deleteEndpointImpl deletes an existing endpoint from the network. func (nw *network) deleteEndpointImpl(ep *endpoint) error { // Delete the HNS endpoint. + // Check if the netNs is a valid GUID to decide on HNSv1 or HNSv2 + if _, err := uuid.Parse(ep.NetNs); err == nil { + if err = hcn.V2ApiSupported(); err != nil { + log.Printf("HNSV2 is not supported on this windows platform") + return err + } + + return nw.deleteEndpointImplHnsV2(ep) + } + + return nw.deleteEndpointImplHnsV1(ep) +} + +// deleteEndpointImplHnsV1 deletes an existing endpoint from the network using HNS v1. +func (nw *network) deleteEndpointImplHnsV1(ep *endpoint) error { log.Printf("[net] HNSEndpointRequest DELETE id:%v", ep.HnsId) hnsResponse, err := hcsshim.HNSEndpointRequest("DELETE", ep.HnsId, "") log.Printf("[net] HNSEndpointRequest DELETE response:%+v err:%v.", hnsResponse, err) @@ -134,6 +276,20 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { return err } +// deleteEndpointImplHnsV2 deletes an existing endpoint from the network using HNS v2. +func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { + var hnsEndpoint *hcn.HostComputeEndpoint + var err error + log.Printf("[net] HostComputeEndpoint DELETE id:%v", ep.HnsId) + if hnsEndpoint, err = hcn.GetEndpointByID(ep.HnsId); err == nil { + err = hnsEndpoint.Delete() + } + + log.Printf("[net] HostComputeEndpoint DELETE err:%v.", err) + + return err +} + // getInfoImpl returns information about the endpoint. func (ep *endpoint) getInfoImpl(epInfo *EndpointInfo) { epInfo.Data["hnsid"] = ep.HnsId diff --git a/network/network.go b/network/network.go index e0a11fe567..e8532c3f34 100644 --- a/network/network.go +++ b/network/network.go @@ -44,6 +44,7 @@ type network struct { extIf *externalInterface DNS DNSInfo EnableSnatOnHost bool + NetNs string } // NetworkInfo contains read-only information about a container network. @@ -56,6 +57,7 @@ type NetworkInfo struct { Policies []policy.Policy BridgeName string EnableSnatOnHost bool + NetNs string Options map[string]interface{} } diff --git a/network/network_windows.go b/network/network_windows.go index 21258111dd..4cd87e5425 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -12,6 +12,8 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" + "github.com/Microsoft/hcsshim/hcn" + "github.com/google/uuid" ) const ( @@ -24,8 +26,8 @@ const ( // Windows implementation of route. type route interface{} -// NewNetworkImpl creates a new container network. -func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { +// newNetworkImplHnsV1 creates a new container network for HNSv1. +func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { var vlanid int networkAdapterName := extIf.Name // FixMe: Find a better way to check if a nic that is selected is not part of a vSwitch @@ -99,6 +101,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt extIf: extIf, VlanId: vlanid, EnableSnatOnHost: nwInfo.EnableSnatOnHost, + NetNs: nwInfo.NetNs, } globals, err := hcsshim.GetHNSGlobals() @@ -112,9 +115,149 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt return nw, nil } +// newNetworkImplHnsV2 creates a new container network for HNSv2. +func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { + // Initialize HNS network. + hnsNetwork := &hcn.HostComputeNetwork{ + Name: nwInfo.Id, + Dns: hcn.Dns{ + Domain: nwInfo.DNS.Suffix, + ServerList: nwInfo.DNS.Servers, + }, + Ipams: []hcn.Ipam{ + hcn.Ipam{ + Type: "Static", + }, + }, + SchemaVersion: hcn.SchemaVersion{ + Major: 2, + Minor: 0, + }, + } + + // Set hcn network adaptor name policy + // FixMe: Find a better way to check if a nic that is selected is not part of a vSwitch + if !strings.HasPrefix(extIf.Name, "vEthernet") { + netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy(extIf.Name) + if err != nil { + log.Printf("[net] Failed to serialize network adapter policy due to error: %v", err) + return nil, err + } + + hnsNetwork.Policies = append(hnsNetwork.Policies, netAdapterNamePolicy) + } + + // Set hcn subnet policy + var vlanid int + var subnetPolicy []byte + opt, _ := nwInfo.Options[genericData].(map[string]interface{}) + if opt != nil && opt[VlanIDKey] != nil { + var err error + vlanID, _ := strconv.ParseUint(opt[VlanIDKey].(string), 10, 32) + subnetPolicy, err = policy.SerializeHcnSubnetVlanPolicy((uint32)(vlanID)) + if err != nil { + log.Printf("[net] Failed to serialize subnet vlan policy due to error: %v", err) + return nil, err + } + + vlanid = (int)(vlanID) + } + + // Set network mode. + switch nwInfo.Mode { + case opModeBridge: + hnsNetwork.Type = hcn.L2Bridge + case opModeTunnel: + hnsNetwork.Type = hcn.L2Tunnel + default: + return nil, errNetworkModeInvalid + } + + // Populate subnets. + for _, subnet := range nwInfo.Subnets { + hnsSubnet := hcn.Subnet{ + IpAddressPrefix: subnet.Prefix.String(), + Routes: []hcn.Route{ + hcn.Route{ + NextHop: subnet.Gateway.String(), + DestinationPrefix: "0.0.0.0/0", + }, + }, + } + + // Set the subnet policy + if vlanid > 0 { + hnsSubnet.Policies = append(hnsSubnet.Policies, subnetPolicy) + } + + hnsNetwork.Ipams[0].Subnets = append(hnsNetwork.Ipams[0].Subnets, hnsSubnet) + } + + // Create the HNS network. + log.Printf("[net] HostComputeNetwork Create: %+v", hnsNetwork) + hnsResponse, err := hnsNetwork.Create() + log.Printf("[net] HostComputeNetwork Create response: %+v err: %v.", hnsResponse, err) + + if err != nil { + return nil, err + } + + // Create the network object. + nw := &network{ + Id: nwInfo.Id, + HnsId: hnsResponse.Id, + Mode: nwInfo.Mode, + Endpoints: make(map[string]*endpoint), + extIf: extIf, + VlanId: vlanid, + EnableSnatOnHost: nwInfo.EnableSnatOnHost, + NetNs: nwInfo.NetNs, + } + + globals, err := hcn.GetGlobals() + if err != nil || globals.Version.Major <= hcn.HNSVersion1803.Major { + // err would be not nil for windows 1709 & below + // Sleep for 10 seconds as a workaround for windows 1803 & below + // This is done only when the network is created. + time.Sleep(time.Duration(10) * time.Second) + } + + return nw, nil +} + +// NewNetworkImpl creates a new container network. +func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { + // Check if the netNs is a valid GUID to decide on HNSv1 or HNSv2 + if _, err := uuid.Parse(nwInfo.NetNs); err == nil { + if err = hcn.V2ApiSupported(); err != nil { + log.Printf("HNSV2 is not supported on this windows platform") + return nil, err + } + + return nm.newNetworkImplHnsV2(nwInfo, extIf) + } + + return nm.newNetworkImplHnsV1(nwInfo, extIf) +} + // DeleteNetworkImpl deletes an existing container network. func (nm *networkManager) deleteNetworkImpl(nw *network) error { // Delete the HNS network. + // Check if the netNs is a valid GUID to decide on HNSv1 or HNSv2 + if _, err := uuid.Parse(nw.NetNs); err == nil { + if err = hcn.V2ApiSupported(); err != nil { + log.Printf("HNSV2 is not supported on this windows platform") + return err + } + + return nm.deleteNetworkImplHnsV2(nw) + } + + return nm.deleteNetworkImplHnsV1(nw) +} + +// DeleteNetworkImplHnsV1 deletes an existing container network using HnsV1. +func (nm *networkManager) deleteNetworkImplHnsV1(nw *network) error { log.Printf("[net] HNSNetworkRequest DELETE id:%v", nw.HnsId) hnsResponse, err := hcsshim.HNSNetworkRequest("DELETE", nw.HnsId, "") log.Printf("[net] HNSNetworkRequest DELETE response:%+v err:%v.", hnsResponse, err) @@ -122,5 +265,19 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error { return err } +// DeleteNetworkImplHnsV2 deletes an existing container network using HnsV2. +func (nm *networkManager) deleteNetworkImplHnsV2(nw *network) error { + var hnsNetwork *hcn.HostComputeNetwork + var err error + log.Printf("[net] HostComputeNetwork DELETE id:%v", nw.HnsId) + if hnsNetwork, err = hcn.GetNetworkByID(nw.HnsId); err == nil { + err = hnsNetwork.Delete() + } + + log.Printf("[net] HostComputeNetwork DELETE err:%v.", err) + + return err +} + func getNetworkInfoImpl(nwInfo *NetworkInfo, nw *network) { } diff --git a/network/policy/policy.go b/network/policy/policy.go index 730cd0e171..6c40482c2d 100644 --- a/network/policy/policy.go +++ b/network/policy/policy.go @@ -8,6 +8,7 @@ const ( NetworkPolicy CNIPolicyType = "NetworkPolicy" EndpointPolicy CNIPolicyType = "EndpointPolicy" OutBoundNatPolicy CNIPolicyType = "OutBoundNAT" + RoutePolicy CNIPolicyType = "ROUTE" ) type CNIPolicyType string diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index 763fe9caa7..112efefc40 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -3,9 +3,10 @@ package policy import ( "encoding/json" "fmt" - "log" + "github.com/Azure/azure-container-networking/log" "github.com/Microsoft/hcsshim" + "github.com/Microsoft/hcsshim/hcn" ) // SerializePolicies serializes policies to json. @@ -14,7 +15,7 @@ func SerializePolicies(policyType CNIPolicyType, policies []Policy, epInfoData m for _, policy := range policies { if policy.Type == policyType { if isPolicyTypeOutBoundNAT := IsPolicyTypeOutBoundNAT(policy); isPolicyTypeOutBoundNAT { - if serializedOutboundNatPolicy, err := SerializeOutBoundNATPolicy(policies, epInfoData); err != nil { + if serializedOutboundNatPolicy, err := SerializeOutBoundNATPolicy(policy, epInfoData); err != nil { log.Printf("Failed to serialize OutBoundNAT policy") } else { jsonPolicies = append(jsonPolicies, serializedOutboundNatPolicy) @@ -28,28 +29,24 @@ func SerializePolicies(policyType CNIPolicyType, policies []Policy, epInfoData m } // GetOutBoundNatExceptionList returns exception list for outbound nat policy -func GetOutBoundNatExceptionList(policies []Policy) ([]string, error) { +func GetOutBoundNatExceptionList(policy Policy) ([]string, error) { type KVPair struct { Type CNIPolicyType `json:"Type"` ExceptionList json.RawMessage `json:"ExceptionList"` } - for _, policy := range policies { - if policy.Type == EndpointPolicy { - var data KVPair - if err := json.Unmarshal(policy.Data, &data); err != nil { - return nil, err - } - - if data.Type == OutBoundNatPolicy { - var exceptionList []string - if err := json.Unmarshal(data.ExceptionList, &exceptionList); err != nil { - return nil, err - } + var data KVPair + if err := json.Unmarshal(policy.Data, &data); err != nil { + return nil, err + } - return exceptionList, nil - } + if data.Type == OutBoundNatPolicy { + var exceptionList []string + if err := json.Unmarshal(data.ExceptionList, &exceptionList); err != nil { + return nil, err } + + return exceptionList, nil } log.Printf("OutBoundNAT policy not set") @@ -77,11 +74,11 @@ func IsPolicyTypeOutBoundNAT(policy Policy) bool { } // SerializeOutBoundNATPolicy formulates OutBoundNAT policy and returns serialized json -func SerializeOutBoundNATPolicy(policies []Policy, epInfoData map[string]interface{}) (json.RawMessage, error) { +func SerializeOutBoundNATPolicy(policy Policy, epInfoData map[string]interface{}) (json.RawMessage, error) { outBoundNatPolicy := hcsshim.OutboundNatPolicy{} outBoundNatPolicy.Policy.Type = hcsshim.OutboundNat - exceptionList, err := GetOutBoundNatExceptionList(policies) + exceptionList, err := GetOutBoundNatExceptionList(policy) if err != nil { log.Printf("Failed to parse outbound NAT policy %v", err) return nil, err @@ -108,3 +105,191 @@ func SerializeOutBoundNATPolicy(policies []Policy, epInfoData map[string]interfa return nil, fmt.Errorf("OutBoundNAT policy not set") } + +// GetPolicyType parses the policy and returns the policy type +func GetPolicyType(policy Policy) CNIPolicyType { + // Check if the type is OutBoundNAT + type KVPairOutBoundNAT struct { + Type CNIPolicyType `json:"Type"` + ExceptionList json.RawMessage `json:"ExceptionList"` + } + var dataOutBoundNAT KVPairOutBoundNAT + if err := json.Unmarshal(policy.Data, &dataOutBoundNAT); err == nil { + if dataOutBoundNAT.Type == OutBoundNatPolicy { + return OutBoundNatPolicy + } + } + + // Check if the type is Route + type KVPairRoute struct { + Type CNIPolicyType `json:"Type"` + DestinationPrefix string `json:"DestinationPrefix"` + NeedEncap bool `json:"NeedEncap"` + } + var dataRoute KVPairRoute + if err := json.Unmarshal(policy.Data, &dataRoute); err == nil { + if dataRoute.Type == RoutePolicy { + return RoutePolicy + } + } + + // Return empty string if the policy type is invalid + log.Printf("Returning policyType INVALID") + return "" +} + +// SerializeHcnSubnetVlanPolicy serializes subnet policy for VLAN to json. +func SerializeHcnSubnetVlanPolicy(vlanID uint32) ([]byte, error) { + vlanPolicySetting := &hcn.VlanPolicySetting{ + IsolationId: vlanID, + } + vlanPolicySettingJSON, err := json.Marshal(vlanPolicySetting) + if err != nil { + return nil, err + } + + vlanSubnetPolicy := &hcn.SubnetPolicy{ + Type: hcn.VLAN, + Settings: vlanPolicySettingJSON, + } + vlanSubnetPolicyJSON, err := json.Marshal(vlanSubnetPolicy) + if err != nil { + return nil, err + } + + return vlanSubnetPolicyJSON, nil +} + +// GetHcnNetAdapterPolicy returns network adapter name policy. +func GetHcnNetAdapterPolicy(networkAdapterName string) (hcn.NetworkPolicy, error) { + networkAdapterNamePolicy := hcn.NetworkPolicy{ + Type: hcn.NetAdapterName, + } + + netAdapterNamePolicySetting := &hcn.NetAdapterNameNetworkPolicySetting{ + NetworkAdapterName: networkAdapterName, + } + netAdapterNamePolicySettingJSON, err := json.Marshal(netAdapterNamePolicySetting) + if err != nil { + return networkAdapterNamePolicy, err + } + + networkAdapterNamePolicy.Settings = netAdapterNamePolicySettingJSON + + return networkAdapterNamePolicy, nil +} + +// GetHcnOutBoundNATPolicy returns outBoundNAT policy. +func GetHcnOutBoundNATPolicy(policy Policy, epInfoData map[string]interface{}) (hcn.EndpointPolicy, error) { + outBoundNATPolicy := hcn.EndpointPolicy{ + Type: hcn.OutBoundNAT, + } + + outBoundNATPolicySetting := hcn.OutboundNatPolicySetting{} + exceptionList, err := GetOutBoundNatExceptionList(policy) + if err != nil { + log.Printf("Failed to parse outbound NAT policy %v", err) + return outBoundNATPolicy, err + } + + if exceptionList != nil { + for _, ipAddress := range exceptionList { + outBoundNATPolicySetting.Exceptions = append(outBoundNATPolicySetting.Exceptions, ipAddress) + } + } + + if epInfoData["cnetAddressSpace"] != nil { + if cnetAddressSpace := epInfoData["cnetAddressSpace"].([]string); cnetAddressSpace != nil { + for _, ipAddress := range cnetAddressSpace { + outBoundNATPolicySetting.Exceptions = append(outBoundNATPolicySetting.Exceptions, ipAddress) + } + } + } + + if outBoundNATPolicySetting.Exceptions != nil { + outBoundNATPolicySettingJSON, err := json.Marshal(outBoundNATPolicySetting) + if err != nil { + return outBoundNATPolicy, err + } + + outBoundNATPolicy.Settings = outBoundNATPolicySettingJSON + return outBoundNATPolicy, nil + } + + return outBoundNATPolicy, fmt.Errorf("OutBoundNAT policy not set") +} + +// GetHcnRoutePolicy returns Route policy. +func GetHcnRoutePolicy(policy Policy) (hcn.EndpointPolicy, error) { + routePolicy := hcn.EndpointPolicy{ + Type: hcn.SDNRoute, + } + + type KVPair struct { + Type CNIPolicyType `json:"Type"` + DestinationPrefix json.RawMessage `json:"DestinationPrefix"` + NeedEncap json.RawMessage `json:"NeedEncap"` + } + + var data KVPair + if err := json.Unmarshal(policy.Data, &data); err != nil { + return routePolicy, err + } + + if data.Type == RoutePolicy { + var destinationPrefix string + var needEncap bool + if err := json.Unmarshal(data.DestinationPrefix, &destinationPrefix); err != nil { + return routePolicy, err + } + + if err := json.Unmarshal(data.NeedEncap, &needEncap); err != nil { + return routePolicy, err + } + + sdnRoutePolicySetting := &hcn.SDNRoutePolicySetting{ + DestinationPrefix: destinationPrefix, + NeedEncap: needEncap, + } + routePolicySettingJSON, err := json.Marshal(sdnRoutePolicySetting) + if err != nil { + return routePolicy, err + } + + routePolicy.Settings = routePolicySettingJSON + + return routePolicy, nil + + } + + return routePolicy, fmt.Errorf("Invalid policy: %+v. Expecting Route policy", policy) +} + +// GetHcnEndpointPolicies returns array of all endpoint policies. +func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}) ([]hcn.EndpointPolicy, error) { + var hcnEndPointPolicies []hcn.EndpointPolicy + for _, policy := range policies { + if policy.Type == policyType { + var err error + var endpointPolicy hcn.EndpointPolicy + if OutBoundNatPolicy == GetPolicyType(policy) { + endpointPolicy, err = GetHcnOutBoundNATPolicy(policy, epInfoData) + } else if RoutePolicy == GetPolicyType(policy) { + endpointPolicy, err = GetHcnRoutePolicy(policy) + } else { + // return error as we should be able to parse all the policies specified + return hcnEndPointPolicies, fmt.Errorf("Failed to set Policy: Type: %s, Data: %s", policy.Type, policy.Data) + } + + if err != nil { + log.Printf("Failed to parse policy: %+v with error %v", policy.Data, err) + return hcnEndPointPolicies, err + } + + hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) + log.Printf("Successfully set the policy: %+v", endpointPolicy) + } + } + + return hcnEndPointPolicies, nil +} diff --git a/vendor.conf b/vendor.conf index 6bcd700fc2..f8d8347efd 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,4 +1,4 @@ -github.com/Microsoft/hcsshim 20640a7b6c723aedeac213b40ae0824fbe38c52c +github.com/Microsoft/hcsshim 79a8f772c4265236cf9da6af7f766b5caf2afb80 github.com/containernetworking/cni 2ce2c24cc2e3c8dbde3c857c5506ef960b2e2c20 k8s.io/client-go 03b9b1062ab5bdfcbd93c27a426d2e1d6b380c73 k8s.io/apimachinery 6c74df1a640b56d1178390c708336f5fb66d7cd8 From b7962daa144a0b06cf430b208cfd17dc02f9f19a Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Sun, 7 Apr 2019 23:26:32 -0700 Subject: [PATCH 02/37] Support hyper-v containers - portmapping --- network/policy/policy.go | 1 + network/policy/policy_windows.go | 73 ++++++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/network/policy/policy.go b/network/policy/policy.go index 6c40482c2d..419b2fc1b9 100644 --- a/network/policy/policy.go +++ b/network/policy/policy.go @@ -9,6 +9,7 @@ const ( EndpointPolicy CNIPolicyType = "EndpointPolicy" OutBoundNatPolicy CNIPolicyType = "OutBoundNAT" RoutePolicy CNIPolicyType = "ROUTE" + PortMappingPolicy CNIPolicyType = "NAT" ) type CNIPolicyType string diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index 112efefc40..b104da23fb 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -3,6 +3,7 @@ package policy import ( "encoding/json" "fmt" + "strings" "github.com/Azure/azure-container-networking/log" "github.com/Microsoft/hcsshim" @@ -133,6 +134,21 @@ func GetPolicyType(policy Policy) CNIPolicyType { } } + // Check if the type if Port mapping / NAT + type KVPairPortMapping struct { + Type CNIPolicyType `json:"Type"` + Protocol string + InternalPort uint16 + ExternalPort uint16 + } + + var dataPortMapping KVPairPortMapping + if err := json.Unmarshal(policy.Data, &dataPortMapping); err == nil { + if dataPortMapping.Type == PortMappingPolicy { + return PortMappingPolicy + } + } + // Return empty string if the policy type is invalid log.Printf("Returning policyType INVALID") return "" @@ -259,12 +275,57 @@ func GetHcnRoutePolicy(policy Policy) (hcn.EndpointPolicy, error) { routePolicy.Settings = routePolicySettingJSON return routePolicy, nil - } return routePolicy, fmt.Errorf("Invalid policy: %+v. Expecting Route policy", policy) } +// GetHcnPortMappingPolicy returns port mapping policy. +func GetHcnPortMappingPolicy(policy Policy) (hcn.EndpointPolicy, error) { + portMappingPolicy := hcn.EndpointPolicy{ + Type: hcn.PortMapping, + } + + type KVPairPortMapping struct { + Type CNIPolicyType `json:"Type"` + ExternalPort uint16 `json:"ExternalPort"` + InternalPort uint16 `json:"InternalPort"` + Protocol string `json:"Protocol"` + } + + var dataPortMapping KVPairPortMapping + if err := json.Unmarshal(policy.Data, &dataPortMapping); err != nil { + return portMappingPolicy, + fmt.Errorf("Invalid policy: %+v. Expecting PortMapping policy. Error: %v", policy, err) + } + + portMappingPolicySetting := &hcn.PortMappingPolicySetting{ + InternalPort: dataPortMapping.InternalPort, + ExternalPort: dataPortMapping.ExternalPort, + } + + protocol := strings.ToUpper(strings.TrimSpace(dataPortMapping.Protocol)) + switch protocol { + // TODO: Replace protocol const as runtime.Protocol_TCP from + // runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" + case "TCP": + portMappingPolicySetting.Protocol = 6 + case "UDP": + portMappingPolicySetting.Protocol = 17 + default: + return portMappingPolicy, fmt.Errorf("Invalid protocol: %s for port mapping", protocol) + } + + portMappingPolicySettingJSON, err := json.Marshal(portMappingPolicySetting) + if err != nil { + return portMappingPolicy, err + } + + portMappingPolicy.Settings = portMappingPolicySettingJSON + + return portMappingPolicy, nil +} + // GetHcnEndpointPolicies returns array of all endpoint policies. func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}) ([]hcn.EndpointPolicy, error) { var hcnEndPointPolicies []hcn.EndpointPolicy @@ -272,11 +333,15 @@ func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoD if policy.Type == policyType { var err error var endpointPolicy hcn.EndpointPolicy - if OutBoundNatPolicy == GetPolicyType(policy) { + + switch GetPolicyType(policy) { + case OutBoundNatPolicy: endpointPolicy, err = GetHcnOutBoundNATPolicy(policy, epInfoData) - } else if RoutePolicy == GetPolicyType(policy) { + case RoutePolicy: endpointPolicy, err = GetHcnRoutePolicy(policy) - } else { + case PortMappingPolicy: + endpointPolicy, err = GetHcnPortMappingPolicy(policy) + default: // return error as we should be able to parse all the policies specified return hcnEndPointPolicies, fmt.Errorf("Failed to set Policy: Type: %s, Data: %s", policy.Type, policy.Data) } From 45a0908d6ce1b410754de45b9403f72efa796a79 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Wed, 10 Apr 2019 07:59:36 -0700 Subject: [PATCH 03/37] Check for null ref before deleting --- network/network.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/network/network.go b/network/network.go index e8532c3f34..53b64109dc 100644 --- a/network/network.go +++ b/network/network.go @@ -210,7 +210,9 @@ func (nm *networkManager) deleteNetwork(networkId string) error { } // Remove the network object. - delete(nw.extIf.Networks, networkId) + if nw.extIf != nil { + delete(nw.extIf.Networks, networkId) + } log.Printf("[net] Deleted network %+v.", nw) return nil From 77210d168c39a7fc4fab961c1bc931fa841ece96 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Tue, 30 Apr 2019 16:16:41 -0700 Subject: [PATCH 04/37] *COW update --- cni/ipam/ipam.go | 8 +-- cni/network/multitenancy.go | 8 ++- cni/network/network.go | 8 +-- cni/network/network_windows.go | 13 +--- network/endpoint.go | 2 +- network/endpoint_windows.go | 102 +++++++++++++++++-------------- network/network_windows.go | 71 +++++++++++++-------- network/policy/policy_windows.go | 2 + 8 files changed, 121 insertions(+), 93 deletions(-) diff --git a/cni/ipam/ipam.go b/cni/ipam/ipam.go index c7171e1441..9aa458080a 100644 --- a/cni/ipam/ipam.go +++ b/cni/ipam/ipam.go @@ -138,8 +138,8 @@ func (plugin *ipamPlugin) Add(args *cniSkel.CmdArgs) error { var result *cniTypesCurr.Result var err error - log.Printf("[cni-ipam] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", - args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) + log.Printf("[cni-ipam] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.", + args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) defer func() { log.Printf("[cni-ipam] ADD command completed with result:%+v err:%v.", result, err) }() @@ -259,8 +259,8 @@ func (plugin *ipamPlugin) Get(args *cniSkel.CmdArgs) error { func (plugin *ipamPlugin) Delete(args *cniSkel.CmdArgs) error { var err error - log.Printf("[cni-ipam] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", - args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) + log.Printf("[cni-ipam] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.", + args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) defer func() { log.Printf("[cni-ipam] DEL command completed with err:%v.", err) }() diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 8cde0de007..a7b6ebc707 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "net" + "strconv" + "strings" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns" @@ -119,13 +121,17 @@ func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse, ifName s } } + var sb strings.Builder + sb.WriteString("Adding cnetAddressspace routes ") for _, ipRouteSubnet := range networkConfig.CnetAddressSpace { - log.Printf("Adding cnetAddressspace routes %v %v", ipRouteSubnet.IPAddress, ipRouteSubnet.PrefixLength) + sb.WriteString(ipRouteSubnet.IPAddress + "/" + strconv.Itoa((int)(ipRouteSubnet.PrefixLength)) + ", ") routeIPnet := net.IPNet{IP: net.ParseIP(ipRouteSubnet.IPAddress), Mask: net.CIDRMask(int(ipRouteSubnet.PrefixLength), 32)} gwIP := net.ParseIP(ipconfig.GatewayIPAddress) result.Routes = append(result.Routes, &cniTypes.Route{Dst: routeIPnet, GW: gwIP}) } + log.Printf(sb.String()) + iface := &cniTypesCurr.Interface{Name: ifName} result.Interfaces = append(result.Interfaces, iface) diff --git a/cni/network/network.go b/cni/network/network.go index 8df6ae70d3..26cd670614 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -195,8 +195,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { enableInfraVnet bool ) - log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", - args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) + log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.", + args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) // Parse network configuration from stdin. nwCfg, err = cni.ParseNetworkConfig(args.StdinData) @@ -601,8 +601,8 @@ func (plugin *netPlugin) Get(args *cniSkel.CmdArgs) error { func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { var err error - log.Printf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", - args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) + log.Printf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v, StdinData:%s}.", + args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData) defer func() { log.Printf("[cni-net] DEL command completed with err:%v.", err) }() diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 6d854ed053..d7ae29624a 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -13,8 +13,6 @@ import ( "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" - "github.com/Microsoft/hcsshim/hcn" - "github.com/google/uuid" cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" @@ -26,14 +24,9 @@ import ( * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 */ func handleConsecutiveAdd(containerId, endpointId, netNs string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { - // Check if the netNs is a valid GUID to decide on HNSv1 or HNSv2 - if _, err := uuid.Parse(netNs); err == nil { - if err = hcn.V2ApiSupported(); err != nil { - log.Printf("HNSV2 is not supported on this windows platform") - return nil, err - } - - return nil, nil + // Return in case of HNSv2 as consecutive add call doesn't need to be handled + if useHnsV2, err := network.UseHnsV2(netNs); useHnsV2 { + return nil, err } hnsEndpoint, err := hcsshim.GetHNSEndpointByName(endpointId) diff --git a/network/endpoint.go b/network/endpoint.go index 8828cab275..1e8f8ac2e3 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -37,7 +37,7 @@ type endpoint struct { PODName string `json:",omitempty"` PODNameSpace string `json:",omitempty"` InfraVnetAddressSpace string `json:",omitempty"` - NetNs string + NetNs string `json:",omitempty"` } // EndpointInfo contains read-only information about an endpoint. diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 634fd19e94..4f40d0980b 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -5,6 +5,7 @@ package network import ( "encoding/json" + "fmt" "net" "strings" @@ -12,7 +13,17 @@ import ( "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" - "github.com/google/uuid" +) + +const ( + // hcnSchemaVersionMajor indicates major version number for hcn schema + hcnSchemaVersionMajor = 2 + + // hcnSchemaVersionMinor indicates minor version number for hcn schema + hcnSchemaVersionMinor = 0 + + // hcnIpamTypeStatic indicates the static type of ipam + hcnIpamTypeStatic = "Static" ) // HotAttachEndpoint is a wrapper of hcsshim's HotAttachEndpoint. @@ -46,10 +57,8 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s // newEndpointImpl creates a new endpoint in the network. func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { - // Check if the netNsPath is a valid GUID to decide on HNSv1 or HNSv2 - if _, err := uuid.Parse(epInfo.NetNsPath); err == nil { - if err = hcn.V2ApiSupported(); err != nil { - log.Printf("HNSV2 is not supported on this windows platform") + if useHnsV2, err := UseHnsV2(epInfo.NetNsPath); useHnsV2 { + if err != nil { return nil, err } @@ -72,7 +81,6 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo) (*endpoint, error) // Get Infrastructure containerID. Handle ADD calls for workload container. var err error infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) - hnsEndpoint := &hcsshim.HNSEndpoint{ Name: infraEpName, VirtualNetwork: nw.HnsId, @@ -151,31 +159,30 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo) (*endpoint, error) func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { var vlanid int if epInfo.Data != nil { - if _, ok := epInfo.Data[VlanIDKey]; ok { - vlanid = epInfo.Data[VlanIDKey].(int) + if vlanData, ok := epInfo.Data[VlanIDKey]; ok { + vlanid = vlanData.(int) } } infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) - hnsEndpoint := &hcn.HostComputeEndpoint{ + hcnEndpoint := &hcn.HostComputeEndpoint{ Name: infraEpName, HostComputeNetwork: nw.HnsId, - //HostComputeNamespace: epInfo.NetNsPath, Dns: hcn.Dns{ Domain: epInfo.DNS.Suffix, ServerList: epInfo.DNS.Servers, }, SchemaVersion: hcn.SchemaVersion{ - Major: 2, - Minor: 0, + Major: hcnSchemaVersionMajor, + Minor: hcnSchemaVersionMinor, }, MacAddress: epInfo.MacAddress.String(), } if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { for _, epPolicy := range endpointPolicies { - hnsEndpoint.Policies = append(hnsEndpoint.Policies, epPolicy) + hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) } } else { log.Printf("[net] Failed to get endpoint policies due to error: %v", err) @@ -188,7 +195,7 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) DestinationPrefix: route.Dst.String(), } - hnsEndpoint.Routes = append(hnsEndpoint.Routes, hcnRoute) + hcnEndpoint.Routes = append(hcnEndpoint.Routes, hcnRoute) } for _, ipAddress := range epInfo.IPAddresses { @@ -197,39 +204,35 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) IpAddress: ipAddress.IP.String(), PrefixLength: uint8(prefixLength), } - hnsEndpoint.IpConfigurations = append(hnsEndpoint.IpConfigurations, ipConfiguration) + + hcnEndpoint.IpConfigurations = append(hcnEndpoint.IpConfigurations, ipConfiguration) } - // Create the HNS endpoint. - log.Printf("[net] HostComputeEndpoint CREATE: %+v", hnsEndpoint) - hnsResponse, err := hnsEndpoint.Create() - log.Printf("[net] HostComputeEndpoint CREATE response: %+v err: %v", hnsResponse, err) + // Create the HCN endpoint. + log.Printf("[net] Creating hcn endpoint: %+v", hcnEndpoint) + hnsResponse, err := hcnEndpoint.Create() if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to create endpoint: %s due to error: %v", hcnEndpoint.Name, err) } + log.Printf("[net] Successfully created hcn endpoint with response: %+v", hnsResponse) + defer func() { if err != nil { - log.Printf("[net] HostComputeEndpoint DELETE id: %v", hnsResponse.Id) + log.Printf("[net] Deleting hcn endpoint with id: %s", hnsResponse.Id) err = hnsResponse.Delete() - log.Printf("[net] HostComputeEndpoint DELETE err: %v", err) + log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", hnsResponse.Id, err) } }() - log.Printf("[net] GetNamespaceByID id: %s", epInfo.NetNsPath) - namespace, e := hcn.GetNamespaceByID(epInfo.NetNsPath) - log.Printf("[net] GetNamespaceByID result: %+v", namespace) - if e != nil { - err = e - log.Printf("[net] GetNamespaceByID err: %v", e) - return nil, err + var namespace *hcn.HostComputeNamespace + if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { + return nil, fmt.Errorf("Failed to get hcn namespace: %s due to error: %v", epInfo.NetNsPath, err) } - log.Printf("[net] AddNamespaceEndpoint ns id: %s, ep id: %s", namespace.Id, hnsResponse.Id) - if e = hcn.AddNamespaceEndpoint(namespace.Id, hnsResponse.Id); e != nil { - log.Printf("[net] AddNamespaceEndpoint err %v", e) - err = e - return nil, err + if err = hcn.AddNamespaceEndpoint(namespace.Id, hnsResponse.Id); err != nil { + return nil, fmt.Errorf("[net] Failed to add endpoint: %s to hcn namespace: %s due to error: %v", + hnsResponse.Id, namespace.Id, err) } // Create the endpoint object. @@ -253,16 +256,12 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) ep.MacAddress, _ = net.ParseMAC(hnsResponse.MacAddress) return ep, nil - } // deleteEndpointImpl deletes an existing endpoint from the network. func (nw *network) deleteEndpointImpl(ep *endpoint) error { - // Delete the HNS endpoint. - // Check if the netNs is a valid GUID to decide on HNSv1 or HNSv2 - if _, err := uuid.Parse(ep.NetNs); err == nil { - if err = hcn.V2ApiSupported(); err != nil { - log.Printf("HNSV2 is not supported on this windows platform") + if useHnsV2, err := UseHnsV2(ep.NetNs); useHnsV2 { + if err != nil { return err } @@ -283,16 +282,27 @@ func (nw *network) deleteEndpointImplHnsV1(ep *endpoint) error { // deleteEndpointImplHnsV2 deletes an existing endpoint from the network using HNS v2. func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { - var hnsEndpoint *hcn.HostComputeEndpoint + var hcnEndpoint *hcn.HostComputeEndpoint var err error - log.Printf("[net] HostComputeEndpoint DELETE id:%v", ep.HnsId) - if hnsEndpoint, err = hcn.GetEndpointByID(ep.HnsId); err == nil { - err = hnsEndpoint.Delete() + log.Printf("[net] Deleting hcn endpoint with id: %s", ep.HnsId) + + if hcnEndpoint, err = hcn.GetEndpointByID(ep.HnsId); err != nil { + return fmt.Errorf("Failed to get hcn endpoint with id: %s due to err: %v", ep.HnsId, err) } - log.Printf("[net] HostComputeEndpoint DELETE err:%v.", err) + // Remove this endpoint from the namespace + if err = hcn.RemoveNamespaceEndpoint(hcnEndpoint.HostComputeNamespace, hcnEndpoint.Id); err != nil { + return fmt.Errorf("Failed to remove hcn endpoint: %s from namespace: %s due to error: %v", ep.HnsId, + hcnEndpoint.HostComputeNamespace, err) + } - return err + if err = hcnEndpoint.Delete(); err != nil { + return fmt.Errorf("Failed to delete hcn endpoint: %s due to error: %v", ep.HnsId, err) + } + + log.Printf("[net] Successfully deleted hcn endpoint with id: %s", ep.HnsId) + + return nil } // getInfoImpl returns information about the endpoint. diff --git a/network/network_windows.go b/network/network_windows.go index 4cd87e5425..99f9f5c1f3 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -5,6 +5,7 @@ package network import ( "encoding/json" + "fmt" "strconv" "strings" "time" @@ -26,6 +27,21 @@ const ( // Windows implementation of route. type route interface{} +// UseHnsV2 indicates whether to use HNSv1 or HNSv2 +func UseHnsV2(netNs string) (bool, error) { + // Check if the netNs is a valid GUID to decide on HNSv1 or HNSv2 + useHnsV2 := false + var err error + if _, err = uuid.Parse(netNs); err == nil { + useHnsV2 = true + if err = hcn.V2ApiSupported(); err != nil { + log.Printf("HNSV2 is not supported on this windows platform") + } + } + + return useHnsV2, err +} + // newNetworkImplHnsV1 creates a new container network for HNSv1. func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { var vlanid int @@ -118,7 +134,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern // newNetworkImplHnsV2 creates a new container network for HNSv2. func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { // Initialize HNS network. - hnsNetwork := &hcn.HostComputeNetwork{ + hcnNetwork := &hcn.HostComputeNetwork{ Name: nwInfo.Id, Dns: hcn.Dns{ Domain: nwInfo.DNS.Suffix, @@ -126,12 +142,12 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern }, Ipams: []hcn.Ipam{ hcn.Ipam{ - Type: "Static", + Type: hcnIpamTypeStatic, }, }, SchemaVersion: hcn.SchemaVersion{ - Major: 2, - Minor: 0, + Major: hcnSchemaVersionMajor, + Minor: hcnSchemaVersionMinor, }, } @@ -144,7 +160,7 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern return nil, err } - hnsNetwork.Policies = append(hnsNetwork.Policies, netAdapterNamePolicy) + hcnNetwork.Policies = append(hcnNetwork.Policies, netAdapterNamePolicy) } // Set hcn subnet policy @@ -166,9 +182,9 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern // Set network mode. switch nwInfo.Mode { case opModeBridge: - hnsNetwork.Type = hcn.L2Bridge + hcnNetwork.Type = hcn.L2Bridge case opModeTunnel: - hnsNetwork.Type = hcn.L2Tunnel + hcnNetwork.Type = hcn.L2Tunnel default: return nil, errNetworkModeInvalid } @@ -179,7 +195,7 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern IpAddressPrefix: subnet.Prefix.String(), Routes: []hcn.Route{ hcn.Route{ - NextHop: subnet.Gateway.String(), + NextHop: subnet.Gateway.String(), DestinationPrefix: "0.0.0.0/0", }, }, @@ -190,18 +206,19 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern hnsSubnet.Policies = append(hnsSubnet.Policies, subnetPolicy) } - hnsNetwork.Ipams[0].Subnets = append(hnsNetwork.Ipams[0].Subnets, hnsSubnet) + hcnNetwork.Ipams[0].Subnets = append(hcnNetwork.Ipams[0].Subnets, hnsSubnet) } // Create the HNS network. - log.Printf("[net] HostComputeNetwork Create: %+v", hnsNetwork) - hnsResponse, err := hnsNetwork.Create() - log.Printf("[net] HostComputeNetwork Create response: %+v err: %v.", hnsResponse, err) + log.Printf("[net] Creating hcn network: %+v", hcnNetwork) + hnsResponse, err := hcnNetwork.Create() if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to create hcn network: %s due to error: %v", hcnNetwork.Name, err) } + log.Printf("[net] Successfully created hcn network with response: %+v", hnsResponse) + // Create the network object. nw := &network{ Id: nwInfo.Id, @@ -227,10 +244,8 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern // NewNetworkImpl creates a new container network. func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { - // Check if the netNs is a valid GUID to decide on HNSv1 or HNSv2 - if _, err := uuid.Parse(nwInfo.NetNs); err == nil { - if err = hcn.V2ApiSupported(); err != nil { - log.Printf("HNSV2 is not supported on this windows platform") + if useHnsV2, err := UseHnsV2(nwInfo.NetNs); useHnsV2 { + if err != nil { return nil, err } @@ -242,11 +257,8 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt // DeleteNetworkImpl deletes an existing container network. func (nm *networkManager) deleteNetworkImpl(nw *network) error { - // Delete the HNS network. - // Check if the netNs is a valid GUID to decide on HNSv1 or HNSv2 - if _, err := uuid.Parse(nw.NetNs); err == nil { - if err = hcn.V2ApiSupported(); err != nil { - log.Printf("HNSV2 is not supported on this windows platform") + if useHnsV2, err := UseHnsV2(nw.NetNs); useHnsV2 { + if err != nil { return err } @@ -267,14 +279,19 @@ func (nm *networkManager) deleteNetworkImplHnsV1(nw *network) error { // DeleteNetworkImplHnsV2 deletes an existing container network using HnsV2. func (nm *networkManager) deleteNetworkImplHnsV2(nw *network) error { - var hnsNetwork *hcn.HostComputeNetwork + var hcnNetwork *hcn.HostComputeNetwork var err error - log.Printf("[net] HostComputeNetwork DELETE id:%v", nw.HnsId) - if hnsNetwork, err = hcn.GetNetworkByID(nw.HnsId); err == nil { - err = hnsNetwork.Delete() + log.Printf("[net] Deleting hcn network with id: %s", nw.HnsId) + + if hcnNetwork, err = hcn.GetNetworkByID(nw.HnsId); err != nil { + return fmt.Errorf("Failed to get hcn network with id: %s due to err: %v", nw.HnsId, err) + } + + if err = hcnNetwork.Delete(); err != nil { + return fmt.Errorf("Failed to delete hcn network: %s due to error: %v", nw.HnsId, err) } - log.Printf("[net] HostComputeNetwork DELETE err:%v.", err) + log.Printf("[net] Successfully deleted hcn network with id: %s", nw.HnsId) return err } diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index b104da23fb..1782e4e93e 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -159,6 +159,7 @@ func SerializeHcnSubnetVlanPolicy(vlanID uint32) ([]byte, error) { vlanPolicySetting := &hcn.VlanPolicySetting{ IsolationId: vlanID, } + vlanPolicySettingJSON, err := json.Marshal(vlanPolicySetting) if err != nil { return nil, err @@ -185,6 +186,7 @@ func GetHcnNetAdapterPolicy(networkAdapterName string) (hcn.NetworkPolicy, error netAdapterNamePolicySetting := &hcn.NetAdapterNameNetworkPolicySetting{ NetworkAdapterName: networkAdapterName, } + netAdapterNamePolicySettingJSON, err := json.Marshal(netAdapterNamePolicySetting) if err != nil { return networkAdapterNamePolicy, err From 2205981261506fbaabc1f1dd022e442a69c59f73 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Wed, 1 May 2019 11:08:27 -0700 Subject: [PATCH 05/37] Make the protocol as const --- network/policy/policy_windows.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index 1782e4e93e..ee0ab62658 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -10,6 +10,14 @@ import ( "github.com/Microsoft/hcsshim/hcn" ) +const ( + // protocolTcp indicates tcp protocol id for portmapping + protocolTcp = 6 + + // protocolUdp indicates udp protocol id for portmapping + protocolUdp = 17 +) + // SerializePolicies serializes policies to json. func SerializePolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}) []json.RawMessage { var jsonPolicies []json.RawMessage @@ -308,12 +316,10 @@ func GetHcnPortMappingPolicy(policy Policy) (hcn.EndpointPolicy, error) { protocol := strings.ToUpper(strings.TrimSpace(dataPortMapping.Protocol)) switch protocol { - // TODO: Replace protocol const as runtime.Protocol_TCP from - // runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" case "TCP": - portMappingPolicySetting.Protocol = 6 + portMappingPolicySetting.Protocol = protocolTcp case "UDP": - portMappingPolicySetting.Protocol = 17 + portMappingPolicySetting.Protocol = protocolUdp default: return portMappingPolicy, fmt.Errorf("Invalid protocol: %s for port mapping", protocol) } From f68b917d441148a48a9dafe989c4cdd63488cd64 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Thu, 2 May 2019 14:54:39 -0700 Subject: [PATCH 06/37] Address review comments --- cni/network/network.go | 2 +- cni/network/network_linux.go | 4 +- cni/network/network_windows.go | 11 +++--- network/endpoint_windows.go | 39 ++++++++++++------ network/network_windows.go | 30 +++++++++----- network/policy/policy_windows.go | 68 ++++++++++++++------------------ 6 files changed, 88 insertions(+), 66 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index 26cd670614..4748330afb 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -300,7 +300,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { */ epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId) if epInfo != nil { - resultConsAdd, errConsAdd := handleConsecutiveAdd(args.ContainerID, endpointId, args.Netns, nwInfo, nwCfg) + resultConsAdd, errConsAdd := handleConsecutiveAdd(args, endpointId, nwInfo, nwCfg) if errConsAdd != nil { log.Printf("handleConsecutiveAdd failed with error %v", errConsAdd) result = resultConsAdd diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index cdf736944d..dcde056ee6 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -9,6 +9,8 @@ import ( "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network" "github.com/Azure/azure-container-networking/network/policy" + + cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) @@ -19,7 +21,7 @@ const ( ) // handleConsecutiveAdd is a dummy function for Linux platform. -func handleConsecutiveAdd(containerId, endpointId, netNs string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { +func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { return nil, nil } diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index d7ae29624a..fba8f6cbb3 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -14,6 +14,7 @@ import ( "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" + cniSkel "github.com/containernetworking/cni/pkg/skel" cniTypes "github.com/containernetworking/cni/pkg/types" cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) @@ -23,20 +24,20 @@ import ( * We can delete this if statement once they fix it. * Issue link: https://github.com/kubernetes/kubernetes/issues/57253 */ -func handleConsecutiveAdd(containerId, endpointId, netNs string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { +func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { // Return in case of HNSv2 as consecutive add call doesn't need to be handled - if useHnsV2, err := network.UseHnsV2(netNs); useHnsV2 { + if useHnsV2, err := network.UseHnsV2(args.Netns); useHnsV2 { return nil, err } hnsEndpoint, err := hcsshim.GetHNSEndpointByName(endpointId) if hnsEndpoint != nil { log.Printf("[net] Found existing endpoint through hcsshim: %+v", hnsEndpoint) - log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, containerId) + log.Printf("[net] Attaching ep %v to container %v", hnsEndpoint.Id, args.ContainerID) - err := hcsshim.HotAttachEndpoint(containerId, hnsEndpoint.Id) + err := hcsshim.HotAttachEndpoint(args.ContainerID, hnsEndpoint.Id) if err != nil { - log.Printf("[cni-net] Failed to hot attach shared endpoint[%v] to container [%v], err:%v.", hnsEndpoint.Id, containerId, err) + log.Printf("[cni-net] Failed to hot attach shared endpoint[%v] to container [%v], err:%v.", hnsEndpoint.Id, args.ContainerID, err) return nil, err } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 4f40d0980b..700d7a2ad9 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -155,15 +155,8 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo) (*endpoint, error) return ep, nil } -// newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 -func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { - var vlanid int - if epInfo.Data != nil { - if vlanData, ok := epInfo.Data[VlanIDKey]; ok { - vlanid = vlanData.(int) - } - } - +// configureHcnEndpoint configures hcn endpoint for creation +func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeEndpoint, error) { infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) hcnEndpoint := &hcn.HostComputeEndpoint{ @@ -208,6 +201,17 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) hcnEndpoint.IpConfigurations = append(hcnEndpoint.IpConfigurations, ipConfiguration) } + return hcnEndpoint, nil +} + +// newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 +func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { + hcnEndpoint, err := nw.configureHcnEndpoint(epInfo) + if err != nil { + log.Printf("[net] Failed to configure hcn endpoint due to error: %v", err) + return nil, err + } + // Create the HCN endpoint. log.Printf("[net] Creating hcn endpoint: %+v", hcnEndpoint) hnsResponse, err := hcnEndpoint.Create() @@ -235,14 +239,26 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) hnsResponse.Id, namespace.Id, err) } + var vlanid int + if epInfo.Data != nil { + if vlanData, ok := epInfo.Data[VlanIDKey]; ok { + vlanid = vlanData.(int) + } + } + + var gateway net.IP + if len(hnsResponse.Routes) > 0 { + gateway = net.ParseIP(hnsResponse.Routes[0].NextHop) + } + // Create the endpoint object. ep := &endpoint{ - Id: infraEpName, + Id: hcnEndpoint.Name, HnsId: hnsResponse.Id, SandboxKey: epInfo.ContainerID, IfName: epInfo.IfName, IPAddresses: epInfo.IPAddresses, - Gateways: []net.IP{net.ParseIP(hnsResponse.Routes[0].NextHop)}, + Gateways: []net.IP{gateway}, DNS: epInfo.DNS, VlanID: vlanid, EnableSnatOnHost: epInfo.EnableSnatOnHost, @@ -284,6 +300,7 @@ func (nw *network) deleteEndpointImplHnsV1(ep *endpoint) error { func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { var hcnEndpoint *hcn.HostComputeEndpoint var err error + log.Printf("[net] Deleting hcn endpoint with id: %s", ep.HnsId) if hcnEndpoint, err = hcn.GetEndpointByID(ep.HnsId); err != nil { diff --git a/network/network_windows.go b/network/network_windows.go index 99f9f5c1f3..0269ab5d6b 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -131,8 +131,8 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern return nw, nil } -// newNetworkImplHnsV2 creates a new container network for HNSv2. -func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { +// configureHcnEndpoint configures hcn endpoint for creation +func (nm *networkManager) configureHcnNetwork(nwInfo *NetworkInfo, extIf *externalInterface) (*hcn.HostComputeNetwork, error) { // Initialize HNS network. hcnNetwork := &hcn.HostComputeNetwork{ Name: nwInfo.Id, @@ -209,6 +209,17 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern hcnNetwork.Ipams[0].Subnets = append(hcnNetwork.Ipams[0].Subnets, hnsSubnet) } + return hcnNetwork, nil +} + +// newNetworkImplHnsV2 creates a new container network for HNSv2. +func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { + hcnNetwork, err := nm.configureHcnNetwork(nwInfo, extIf) + if err != nil { + log.Printf("[net] Failed to configure hcn network due to error: %v", err) + return nil, err + } + // Create the HNS network. log.Printf("[net] Creating hcn network: %+v", hcnNetwork) hnsResponse, err := hcnNetwork.Create() @@ -219,6 +230,13 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern log.Printf("[net] Successfully created hcn network with response: %+v", hnsResponse) + var vlanid int + opt, _ := nwInfo.Options[genericData].(map[string]interface{}) + if opt != nil && opt[VlanIDKey] != nil { + vlanID, _ := strconv.ParseInt(opt[VlanIDKey].(string), 10, 32) + vlanid = (int)(vlanID) + } + // Create the network object. nw := &network{ Id: nwInfo.Id, @@ -231,14 +249,6 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern NetNs: nwInfo.NetNs, } - globals, err := hcn.GetGlobals() - if err != nil || globals.Version.Major <= hcn.HNSVersion1803.Major { - // err would be not nil for windows 1709 & below - // Sleep for 10 seconds as a workaround for windows 1803 & below - // This is done only when the network is created. - time.Sleep(time.Duration(10) * time.Second) - } - return nw, nil } diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index ee0ab62658..4ad4d3008b 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -16,8 +16,29 @@ const ( // protocolUdp indicates udp protocol id for portmapping protocolUdp = 17 + + // CnetAddressSpace indicates constant for the key string + cnetAddressSpace = "cnetAddressSpace" ) +type KVPairPortMapping struct { + Type CNIPolicyType `json:"Type"` + ExternalPort uint16 `json:"ExternalPort"` + InternalPort uint16 `json:"InternalPort"` + Protocol string `json:"Protocol"` +} + +type KVPairOutBoundNAT struct { + Type CNIPolicyType `json:"Type"` + ExceptionList json.RawMessage `json:"ExceptionList"` +} + +type KVPairRoute struct { + Type CNIPolicyType `json:"Type"` + DestinationPrefix string `json:"DestinationPrefix"` + NeedEncap bool `json:"NeedEncap"` +} + // SerializePolicies serializes policies to json. func SerializePolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}) []json.RawMessage { var jsonPolicies []json.RawMessage @@ -39,12 +60,7 @@ func SerializePolicies(policyType CNIPolicyType, policies []Policy, epInfoData m // GetOutBoundNatExceptionList returns exception list for outbound nat policy func GetOutBoundNatExceptionList(policy Policy) ([]string, error) { - type KVPair struct { - Type CNIPolicyType `json:"Type"` - ExceptionList json.RawMessage `json:"ExceptionList"` - } - - var data KVPair + var data KVPairOutBoundNAT if err := json.Unmarshal(policy.Data, &data); err != nil { return nil, err } @@ -65,11 +81,7 @@ func GetOutBoundNatExceptionList(policy Policy) ([]string, error) { // IsPolicyTypeOutBoundNAT return true if the policy type is OutBoundNAT func IsPolicyTypeOutBoundNAT(policy Policy) bool { if policy.Type == EndpointPolicy { - type KVPair struct { - Type CNIPolicyType `json:"Type"` - ExceptionList json.RawMessage `json:"ExceptionList"` - } - var data KVPair + var data KVPairOutBoundNAT if err := json.Unmarshal(policy.Data, &data); err != nil { return false } @@ -99,8 +111,8 @@ func SerializeOutBoundNATPolicy(policy Policy, epInfoData map[string]interface{} } } - if epInfoData["cnetAddressSpace"] != nil { - if cnetAddressSpace := epInfoData["cnetAddressSpace"].([]string); cnetAddressSpace != nil { + if epInfoData[cnetAddressSpace] != nil { + if cnetAddressSpace := epInfoData[cnetAddressSpace].([]string); cnetAddressSpace != nil { for _, ipAddress := range cnetAddressSpace { outBoundNatPolicy.Exceptions = append(outBoundNatPolicy.Exceptions, ipAddress) } @@ -118,10 +130,6 @@ func SerializeOutBoundNATPolicy(policy Policy, epInfoData map[string]interface{} // GetPolicyType parses the policy and returns the policy type func GetPolicyType(policy Policy) CNIPolicyType { // Check if the type is OutBoundNAT - type KVPairOutBoundNAT struct { - Type CNIPolicyType `json:"Type"` - ExceptionList json.RawMessage `json:"ExceptionList"` - } var dataOutBoundNAT KVPairOutBoundNAT if err := json.Unmarshal(policy.Data, &dataOutBoundNAT); err == nil { if dataOutBoundNAT.Type == OutBoundNatPolicy { @@ -130,11 +138,6 @@ func GetPolicyType(policy Policy) CNIPolicyType { } // Check if the type is Route - type KVPairRoute struct { - Type CNIPolicyType `json:"Type"` - DestinationPrefix string `json:"DestinationPrefix"` - NeedEncap bool `json:"NeedEncap"` - } var dataRoute KVPairRoute if err := json.Unmarshal(policy.Data, &dataRoute); err == nil { if dataRoute.Type == RoutePolicy { @@ -143,13 +146,6 @@ func GetPolicyType(policy Policy) CNIPolicyType { } // Check if the type if Port mapping / NAT - type KVPairPortMapping struct { - Type CNIPolicyType `json:"Type"` - Protocol string - InternalPort uint16 - ExternalPort uint16 - } - var dataPortMapping KVPairPortMapping if err := json.Unmarshal(policy.Data, &dataPortMapping); err == nil { if dataPortMapping.Type == PortMappingPolicy { @@ -177,6 +173,7 @@ func SerializeHcnSubnetVlanPolicy(vlanID uint32) ([]byte, error) { Type: hcn.VLAN, Settings: vlanPolicySettingJSON, } + vlanSubnetPolicyJSON, err := json.Marshal(vlanSubnetPolicy) if err != nil { return nil, err @@ -224,8 +221,8 @@ func GetHcnOutBoundNATPolicy(policy Policy, epInfoData map[string]interface{}) ( } } - if epInfoData["cnetAddressSpace"] != nil { - if cnetAddressSpace := epInfoData["cnetAddressSpace"].([]string); cnetAddressSpace != nil { + if epInfoData[cnetAddressSpace] != nil { + if cnetAddressSpace := epInfoData[cnetAddressSpace].([]string); cnetAddressSpace != nil { for _, ipAddress := range cnetAddressSpace { outBoundNATPolicySetting.Exceptions = append(outBoundNATPolicySetting.Exceptions, ipAddress) } @@ -265,6 +262,7 @@ func GetHcnRoutePolicy(policy Policy) (hcn.EndpointPolicy, error) { if data.Type == RoutePolicy { var destinationPrefix string var needEncap bool + if err := json.Unmarshal(data.DestinationPrefix, &destinationPrefix); err != nil { return routePolicy, err } @@ -277,6 +275,7 @@ func GetHcnRoutePolicy(policy Policy) (hcn.EndpointPolicy, error) { DestinationPrefix: destinationPrefix, NeedEncap: needEncap, } + routePolicySettingJSON, err := json.Marshal(sdnRoutePolicySetting) if err != nil { return routePolicy, err @@ -296,13 +295,6 @@ func GetHcnPortMappingPolicy(policy Policy) (hcn.EndpointPolicy, error) { Type: hcn.PortMapping, } - type KVPairPortMapping struct { - Type CNIPolicyType `json:"Type"` - ExternalPort uint16 `json:"ExternalPort"` - InternalPort uint16 `json:"InternalPort"` - Protocol string `json:"Protocol"` - } - var dataPortMapping KVPairPortMapping if err := json.Unmarshal(policy.Data, &dataPortMapping); err != nil { return portMappingPolicy, From 21f447f0e3664078d3c863e7af7ca05fd52f61e9 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Tue, 7 May 2019 13:30:40 -0700 Subject: [PATCH 07/37] Update the dependencies in vendor --- Gopkg.lock | 128 +++- vendor/github.com/Microsoft/go-winio/ea.go | 274 +++---- .../github.com/Microsoft/hcsshim/.gitignore | 2 +- .../github.com/Microsoft/hcsshim/appveyor.yml | 56 +- .../github.com/Microsoft/hcsshim/hcn/hcn.go | 177 +++++ .../Microsoft/hcsshim/hcn/hcnendpoint.go | 366 +++++++++ .../Microsoft/hcsshim/hcn/hcnerrors.go | 95 +++ .../Microsoft/hcsshim/hcn/hcnglobals.go | 87 +++ .../Microsoft/hcsshim/hcn/hcnloadbalancer.go | 335 ++++++++ .../Microsoft/hcsshim/hcn/hcnnamespace.go | 424 +++++++++++ .../Microsoft/hcsshim/hcn/hcnnetwork.go | 418 ++++++++++ .../Microsoft/hcsshim/hcn/hcnpolicy.go | 217 ++++++ .../Microsoft/hcsshim/hcn/hcnsupport.go | 71 ++ .../Microsoft/hcsshim/hcn/zsyscall_windows.go | 714 ++++++++++++++++++ .../hcsshim/internal/cni/registry.go | 110 +++ .../Microsoft/hcsshim/internal/guid/guid.go | 138 ++-- .../hcsshim/internal/regstate/regstate.go | 287 +++++++ .../internal/regstate/zsyscall_windows.go | 51 ++ .../hcsshim/internal/runhcs/container.go | 71 ++ .../Microsoft/hcsshim/internal/runhcs/util.go | 16 + .../Microsoft/hcsshim/internal/runhcs/vm.go | 43 ++ .../hcsshim/internal/schema1/schema1.go | 490 ++++++------ .../hcsshim/internal/wclayer/layerid.go | 26 +- .../golang.org/x/sys/windows/registry/key.go | 198 +++++ .../x/sys/windows/registry/mksyscall.go | 7 + .../x/sys/windows/registry/syscall.go | 32 + .../x/sys/windows/registry/value.go | 384 ++++++++++ .../sys/windows/registry/zsyscall_windows.go | 120 +++ 28 files changed, 4827 insertions(+), 510 deletions(-) create mode 100644 vendor/github.com/Microsoft/hcsshim/hcn/hcn.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hcn/hcnendpoint.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hcn/hcnerrors.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hcn/hcnloadbalancer.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hcn/hcnnamespace.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hcn/hcnnetwork.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go create mode 100644 vendor/github.com/Microsoft/hcsshim/hcn/zsyscall_windows.go create mode 100644 vendor/github.com/Microsoft/hcsshim/internal/cni/registry.go create mode 100644 vendor/github.com/Microsoft/hcsshim/internal/regstate/regstate.go create mode 100644 vendor/github.com/Microsoft/hcsshim/internal/regstate/zsyscall_windows.go create mode 100644 vendor/github.com/Microsoft/hcsshim/internal/runhcs/container.go create mode 100644 vendor/github.com/Microsoft/hcsshim/internal/runhcs/util.go create mode 100644 vendor/github.com/Microsoft/hcsshim/internal/runhcs/vm.go create mode 100644 vendor/golang.org/x/sys/windows/registry/key.go create mode 100644 vendor/golang.org/x/sys/windows/registry/mksyscall.go create mode 100644 vendor/golang.org/x/sys/windows/registry/syscall.go create mode 100644 vendor/golang.org/x/sys/windows/registry/value.go create mode 100644 vendor/golang.org/x/sys/windows/registry/zsyscall_windows.go diff --git a/Gopkg.lock b/Gopkg.lock index d64e54ad1c..b4f550ab72 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,15 +2,20 @@ [[projects]] + digest = "1:f9ae348e1f793dcf9ed930ed47136a67343dbd6809c5c91391322267f4476892" name = "github.com/Microsoft/go-winio" packages = ["."] + pruneopts = "UT" revision = "1a8911d1ed007260465c3bfbbc785ac6915a0bb8" version = "v0.4.12" [[projects]] + digest = "1:8cf3e59c26d185775c218c4ac3e63ba8dac9c2692d97e99152a353f7344b5a1d" name = "github.com/Microsoft/hcsshim" packages = [ ".", + "hcn", + "internal/cni", "internal/guestrequest", "internal/guid", "internal/hcs", @@ -20,16 +25,20 @@ "internal/logfields", "internal/longpath", "internal/mergemaps", + "internal/regstate", + "internal/runhcs", "internal/safefile", "internal/schema1", "internal/schema2", "internal/timeout", - "internal/wclayer" + "internal/wclayer", ] + pruneopts = "UT" revision = "f92b8fb9c92e17da496af5a69e3ee13fbe9916e1" version = "v0.8.6" [[projects]] + digest = "1:eb56b03a240f4ae8de520edf525f75d4d26ea41b99f5972b449cbd5ba9f0adf6" name = "github.com/containernetworking/cni" packages = [ "libcni", @@ -38,116 +47,146 @@ "pkg/types", "pkg/types/020", "pkg/types/current", - "pkg/version" + "pkg/version", ] + pruneopts = "UT" revision = "fbb95fff8a5239a4295c991efa8a397d43118f7e" [[projects]] + digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" name = "github.com/davecgh/go-spew" packages = ["spew"] + pruneopts = "UT" revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" version = "v1.1.1" [[projects]] + digest = "1:190198fd3c956b1383a31273d6b7e76cb926e9b68f7eaf1f6b9e487b50113d6e" name = "github.com/docker/libnetwork" packages = [ "driverapi", "drivers/remote/api", - "types" + "types", ] + pruneopts = "UT" revision = "1ee720e18fe98dceda6039bdd005ffbcb359d343" version = "v0.5.6" [[projects]] + digest = "1:4d02824a56d268f74a6b6fdd944b20b58a77c3d70e81008b3ee0c4f1a6777340" name = "github.com/gogo/protobuf" packages = [ "proto", - "sortkeys" + "sortkeys", ] + pruneopts = "UT" revision = "ba06b47c162d49f2af050fb4c75bcbc86a159d5c" version = "v1.2.1" [[projects]] + digest = "1:239c4c7fd2159585454003d9be7207167970194216193a8a210b8d29576f19c9" name = "github.com/golang/protobuf" packages = [ "proto", "ptypes", "ptypes/any", "ptypes/duration", - "ptypes/timestamp" + "ptypes/timestamp", ] + pruneopts = "UT" revision = "b5d812f8a3706043e23a9cd5babf2e5423744d30" version = "v1.3.1" [[projects]] + digest = "1:a6181aca1fd5e27103f9a920876f29ac72854df7345a39f3b01e61c8c94cc8af" name = "github.com/google/gofuzz" packages = ["."] + pruneopts = "UT" revision = "f140a6486e521aad38f5917de355cbf147cc0496" version = "v1.0.0" [[projects]] + digest = "1:582b704bebaa06b48c29b0cec224a6058a09c86883aaddabde889cd1a5f73e1b" name = "github.com/google/uuid" packages = ["."] + pruneopts = "UT" revision = "0cd6bf5da1e1c83f8b45653022c74f71af0538a4" version = "v1.1.1" [[projects]] + digest = "1:65c4414eeb350c47b8de71110150d0ea8a281835b1f386eacaa3ad7325929c21" name = "github.com/googleapis/gnostic" packages = [ "OpenAPIv2", "compiler", - "extensions" + "extensions", ] + pruneopts = "UT" revision = "7c663266750e7d82587642f65e60bc4083f1f84e" version = "v0.2.0" [[projects]] + digest = "1:d15ee511aa0f56baacc1eb4c6b922fa1c03b38413b6be18166b996d82a0156ea" name = "github.com/hashicorp/golang-lru" packages = [ ".", - "simplelru" + "simplelru", ] + pruneopts = "UT" revision = "7087cb70de9f7a8bc0a10c375cb0d2280a8edf9c" version = "v0.5.1" [[projects]] + digest = "1:f5a2051c55d05548d2d4fd23d244027b59fbd943217df8aa3b5e170ac2fd6e1b" name = "github.com/json-iterator/go" packages = ["."] + pruneopts = "UT" revision = "0ff49de124c6f76f8494e194af75bde0f1a49a29" version = "v1.1.6" [[projects]] + digest = "1:31e761d97c76151dde79e9d28964a812c46efc5baee4085b86f68f0c654450de" name = "github.com/konsorten/go-windows-terminal-sequences" packages = ["."] + pruneopts = "UT" revision = "f55edac94c9bbba5d6182a4be46d86a2c9b5b50e" version = "v1.0.2" [[projects]] + digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563" name = "github.com/modern-go/concurrent" packages = ["."] + pruneopts = "UT" revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" version = "1.0.3" [[projects]] + digest = "1:e32bdbdb7c377a07a9a46378290059822efdce5c8d96fe71940d87cb4f918855" name = "github.com/modern-go/reflect2" packages = ["."] + pruneopts = "UT" revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" version = "1.0.1" [[projects]] + digest = "1:fd61cf4ae1953d55df708acb6b91492d538f49c305b364a014049914495db426" name = "github.com/sirupsen/logrus" packages = ["."] + pruneopts = "UT" revision = "8bdbc7bcc01dcbb8ec23dc8a28e332258d25251f" version = "v1.4.1" [[projects]] branch = "master" + digest = "1:bbe51412d9915d64ffaa96b51d409e070665efc5194fcf145c4a27d4133107a4" name = "golang.org/x/crypto" packages = ["ssh/terminal"] + pruneopts = "UT" revision = "a29dc8fdc73485234dbef99ebedb95d2eced08de" [[projects]] branch = "master" + digest = "1:b9bf9ddb713916ca0fe4630b135cc937c81a00378a97083a906c9669d56ab23f" name = "golang.org/x/net" packages = [ "context", @@ -155,29 +194,36 @@ "http/httpguts", "http2", "http2/hpack", - "idna" + "idna", ] + pruneopts = "UT" revision = "7f726cade0ab7c929c16ce0b5b25bd201e25f39f" [[projects]] branch = "master" + digest = "1:9927d6aceb89d188e21485f42a7a254e67e6fdcf4260aba375fe18e3c300dfb4" name = "golang.org/x/oauth2" packages = [ ".", - "internal" + "internal", ] + pruneopts = "UT" revision = "9f3314589c9a9136388751d9adae6b0ed400978a" [[projects]] branch = "master" + digest = "1:5a8d9a6dfd3f8a2c75bf3d59fe35dadb8052084d02803ffab046e460feb31145" name = "golang.org/x/sys" packages = [ "unix", - "windows" + "windows", + "windows/registry", ] + pruneopts = "UT" revision = "a43fa875dd822b81eb6d2ad538bc1f4caba169bd" [[projects]] + digest = "1:8d8faad6b12a3a4c819a3f9618cb6ee1fa1cfc33253abeeea8b55336721e3405" name = "golang.org/x/text" packages = [ "collate", @@ -195,18 +241,22 @@ "unicode/bidi", "unicode/cldr", "unicode/norm", - "unicode/rangetable" + "unicode/rangetable", ] + pruneopts = "UT" revision = "342b2e1fbaa52c93f31447ad2c6abc048c63e475" version = "v0.3.2" [[projects]] branch = "master" + digest = "1:9fdc2b55e8e0fafe4b41884091e51e77344f7dc511c5acedcfd98200003bff90" name = "golang.org/x/time" packages = ["rate"] + pruneopts = "UT" revision = "9d24e82272b4f38b78bc8cff74fa936d31ccd8ef" [[projects]] + digest = "1:6eb6e3b6d9fffb62958cf7f7d88dbbe1dd6839436b0802e194c590667a40412a" name = "google.golang.org/appengine" packages = [ "internal", @@ -215,25 +265,31 @@ "internal/log", "internal/remote_api", "internal/urlfetch", - "urlfetch" + "urlfetch", ] + pruneopts = "UT" revision = "54a98f90d1c46b7731eb8fb305d2a321c30ef610" version = "v1.5.0" [[projects]] + digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a" name = "gopkg.in/inf.v0" packages = ["."] + pruneopts = "UT" revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf" version = "v0.9.1" [[projects]] + digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96" name = "gopkg.in/yaml.v2" packages = ["."] + pruneopts = "UT" revision = "51d6538a90f86fe93ac480b35f37b2be17fef232" version = "v2.2.2" [[projects]] branch = "master" + digest = "1:3d206da0c5871a719863cbf3a2d44fa5c0a77670491c19e2274b05799e84879a" name = "k8s.io/api" packages = [ "admissionregistration/v1beta1", @@ -271,11 +327,13 @@ "settings/v1alpha1", "storage/v1", "storage/v1alpha1", - "storage/v1beta1" + "storage/v1beta1", ] + pruneopts = "UT" revision = "61630f889b3c3efb8c3a4ba377da1a421d9ff6ce" [[projects]] + digest = "1:d43bb672b3e811c5ea37b48a0a73bea7fcbcb348a45ce7ba349006145c76418d" name = "k8s.io/apimachinery" packages = [ "pkg/api/errors", @@ -316,11 +374,13 @@ "pkg/util/yaml", "pkg/version", "pkg/watch", - "third_party/forked/golang/reflect" + "third_party/forked/golang/reflect", ] + pruneopts = "UT" revision = "d7deff9243b165ee192f5551710ea4285dcfd615" [[projects]] + digest = "1:a3712b06370e1f92902bc22151b31bbfef00f520945bd51716fe498375923864" name = "k8s.io/client-go" packages = [ "discovery", @@ -462,36 +522,70 @@ "util/connrotation", "util/flowcontrol", "util/keyutil", - "util/retry" + "util/retry", ] + pruneopts = "UT" revision = "6ee68ca5fd8355d024d02f9db0b3b667e8357a0f" version = "v11.0.0" [[projects]] + digest = "1:c696379ad201c1e86591785579e16bf6cf886c362e9a7534e8eb0d1028b20582" name = "k8s.io/klog" packages = ["."] + pruneopts = "UT" revision = "e531227889390a39d9533dde61f590fe9f4b0035" version = "v0.3.0" [[projects]] branch = "master" + digest = "1:14e8a3b53e6d8cb5f44783056b71bb2ca1ac7e333939cc97f3e50b579c920845" name = "k8s.io/utils" packages = [ "buffer", "integer", - "trace" + "trace", ] + pruneopts = "UT" revision = "21c4ce38f2a793ec01e925ddc31216500183b773" [[projects]] + digest = "1:7719608fe0b52a4ece56c2dde37bedd95b938677d1ab0f84b8a7852e4c59f849" name = "sigs.k8s.io/yaml" packages = ["."] + pruneopts = "UT" revision = "fd68e9863619f6ec2fdd8625fe1f02e7c877e480" version = "v1.1.0" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "6355bb8af9b2a11f4dbe507402aefb79e843ca1c0191829cd76c3e5de4b8ef1d" + input-imports = [ + "github.com/Microsoft/go-winio", + "github.com/Microsoft/hcsshim", + "github.com/Microsoft/hcsshim/hcn", + "github.com/containernetworking/cni/libcni", + "github.com/containernetworking/cni/pkg/invoke", + "github.com/containernetworking/cni/pkg/skel", + "github.com/containernetworking/cni/pkg/types", + "github.com/containernetworking/cni/pkg/types/current", + "github.com/containernetworking/cni/pkg/version", + "github.com/docker/libnetwork/driverapi", + "github.com/docker/libnetwork/drivers/remote/api", + "github.com/google/uuid", + "golang.org/x/sys/unix", + "k8s.io/api/core/v1", + "k8s.io/api/networking/v1", + "k8s.io/apimachinery/pkg/apis/meta/v1", + "k8s.io/apimachinery/pkg/types", + "k8s.io/apimachinery/pkg/util/intstr", + "k8s.io/apimachinery/pkg/util/wait", + "k8s.io/apimachinery/pkg/version", + "k8s.io/client-go/informers", + "k8s.io/client-go/informers/core/v1", + "k8s.io/client-go/informers/networking/v1", + "k8s.io/client-go/kubernetes", + "k8s.io/client-go/rest", + "k8s.io/client-go/tools/cache", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/vendor/github.com/Microsoft/go-winio/ea.go b/vendor/github.com/Microsoft/go-winio/ea.go index b37e930d6a..4051c1b33b 100644 --- a/vendor/github.com/Microsoft/go-winio/ea.go +++ b/vendor/github.com/Microsoft/go-winio/ea.go @@ -1,137 +1,137 @@ -package winio - -import ( - "bytes" - "encoding/binary" - "errors" -) - -type fileFullEaInformation struct { - NextEntryOffset uint32 - Flags uint8 - NameLength uint8 - ValueLength uint16 -} - -var ( - fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) - - errInvalidEaBuffer = errors.New("invalid extended attribute buffer") - errEaNameTooLarge = errors.New("extended attribute name too large") - errEaValueTooLarge = errors.New("extended attribute value too large") -) - -// ExtendedAttribute represents a single Windows EA. -type ExtendedAttribute struct { - Name string - Value []byte - Flags uint8 -} - -func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { - var info fileFullEaInformation - err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) - if err != nil { - err = errInvalidEaBuffer - return - } - - nameOffset := fileFullEaInformationSize - nameLen := int(info.NameLength) - valueOffset := nameOffset + int(info.NameLength) + 1 - valueLen := int(info.ValueLength) - nextOffset := int(info.NextEntryOffset) - if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { - err = errInvalidEaBuffer - return - } - - ea.Name = string(b[nameOffset : nameOffset+nameLen]) - ea.Value = b[valueOffset : valueOffset+valueLen] - ea.Flags = info.Flags - if info.NextEntryOffset != 0 { - nb = b[info.NextEntryOffset:] - } - return -} - -// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION -// buffer retrieved from BackupRead, ZwQueryEaFile, etc. -func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { - for len(b) != 0 { - ea, nb, err := parseEa(b) - if err != nil { - return nil, err - } - - eas = append(eas, ea) - b = nb - } - return -} - -func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { - if int(uint8(len(ea.Name))) != len(ea.Name) { - return errEaNameTooLarge - } - if int(uint16(len(ea.Value))) != len(ea.Value) { - return errEaValueTooLarge - } - entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) - withPadding := (entrySize + 3) &^ 3 - nextOffset := uint32(0) - if !last { - nextOffset = withPadding - } - info := fileFullEaInformation{ - NextEntryOffset: nextOffset, - Flags: ea.Flags, - NameLength: uint8(len(ea.Name)), - ValueLength: uint16(len(ea.Value)), - } - - err := binary.Write(buf, binary.LittleEndian, &info) - if err != nil { - return err - } - - _, err = buf.Write([]byte(ea.Name)) - if err != nil { - return err - } - - err = buf.WriteByte(0) - if err != nil { - return err - } - - _, err = buf.Write(ea.Value) - if err != nil { - return err - } - - _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) - if err != nil { - return err - } - - return nil -} - -// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION -// buffer for use with BackupWrite, ZwSetEaFile, etc. -func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { - var buf bytes.Buffer - for i := range eas { - last := false - if i == len(eas)-1 { - last = true - } - - err := writeEa(&buf, &eas[i], last) - if err != nil { - return nil, err - } - } - return buf.Bytes(), nil -} +package winio + +import ( + "bytes" + "encoding/binary" + "errors" +) + +type fileFullEaInformation struct { + NextEntryOffset uint32 + Flags uint8 + NameLength uint8 + ValueLength uint16 +} + +var ( + fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) + + errInvalidEaBuffer = errors.New("invalid extended attribute buffer") + errEaNameTooLarge = errors.New("extended attribute name too large") + errEaValueTooLarge = errors.New("extended attribute value too large") +) + +// ExtendedAttribute represents a single Windows EA. +type ExtendedAttribute struct { + Name string + Value []byte + Flags uint8 +} + +func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { + var info fileFullEaInformation + err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) + if err != nil { + err = errInvalidEaBuffer + return + } + + nameOffset := fileFullEaInformationSize + nameLen := int(info.NameLength) + valueOffset := nameOffset + int(info.NameLength) + 1 + valueLen := int(info.ValueLength) + nextOffset := int(info.NextEntryOffset) + if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { + err = errInvalidEaBuffer + return + } + + ea.Name = string(b[nameOffset : nameOffset+nameLen]) + ea.Value = b[valueOffset : valueOffset+valueLen] + ea.Flags = info.Flags + if info.NextEntryOffset != 0 { + nb = b[info.NextEntryOffset:] + } + return +} + +// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION +// buffer retrieved from BackupRead, ZwQueryEaFile, etc. +func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { + for len(b) != 0 { + ea, nb, err := parseEa(b) + if err != nil { + return nil, err + } + + eas = append(eas, ea) + b = nb + } + return +} + +func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { + if int(uint8(len(ea.Name))) != len(ea.Name) { + return errEaNameTooLarge + } + if int(uint16(len(ea.Value))) != len(ea.Value) { + return errEaValueTooLarge + } + entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) + withPadding := (entrySize + 3) &^ 3 + nextOffset := uint32(0) + if !last { + nextOffset = withPadding + } + info := fileFullEaInformation{ + NextEntryOffset: nextOffset, + Flags: ea.Flags, + NameLength: uint8(len(ea.Name)), + ValueLength: uint16(len(ea.Value)), + } + + err := binary.Write(buf, binary.LittleEndian, &info) + if err != nil { + return err + } + + _, err = buf.Write([]byte(ea.Name)) + if err != nil { + return err + } + + err = buf.WriteByte(0) + if err != nil { + return err + } + + _, err = buf.Write(ea.Value) + if err != nil { + return err + } + + _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) + if err != nil { + return err + } + + return nil +} + +// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION +// buffer for use with BackupWrite, ZwSetEaFile, etc. +func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { + var buf bytes.Buffer + for i := range eas { + last := false + if i == len(eas)-1 { + last = true + } + + err := writeEa(&buf, &eas[i], last) + if err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/.gitignore b/vendor/github.com/Microsoft/hcsshim/.gitignore index 1feae78042..b883f1fdc6 100644 --- a/vendor/github.com/Microsoft/hcsshim/.gitignore +++ b/vendor/github.com/Microsoft/hcsshim/.gitignore @@ -1 +1 @@ -*.exe +*.exe diff --git a/vendor/github.com/Microsoft/hcsshim/appveyor.yml b/vendor/github.com/Microsoft/hcsshim/appveyor.yml index 3969ee169e..a8ec5a5939 100644 --- a/vendor/github.com/Microsoft/hcsshim/appveyor.yml +++ b/vendor/github.com/Microsoft/hcsshim/appveyor.yml @@ -1,29 +1,29 @@ -version: 0.1.{build} - -image: Visual Studio 2017 - -clone_folder: c:\gopath\src\github.com\Microsoft\hcsshim - -environment: - GOPATH: c:\gopath - PATH: C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin;%GOPATH%\bin;C:\gometalinter-2.0.12-windows-amd64;%PATH% - -stack: go 1.11 - -build_script: - - appveyor DownloadFile https://github.com/alecthomas/gometalinter/releases/download/v2.0.12/gometalinter-2.0.12-windows-amd64.zip - - 7z x gometalinter-2.0.12-windows-amd64.zip -y -oC:\ > NUL - - gometalinter.exe --config .gometalinter.json ./... - - go build ./cmd/wclayer - - go build ./cmd/runhcs - - go build ./cmd/tar2ext4 - - go test -v ./... -tags admin - - go test -c ./test/functional/ -tags functional - - go test -c ./test/runhcs/ -tags integration - -artifacts: - - path: 'wclayer.exe' - - path: 'runhcs.exe' - - path: 'tar2ext4.exe' - - path: 'functional.test.exe' +version: 0.1.{build} + +image: Visual Studio 2017 + +clone_folder: c:\gopath\src\github.com\Microsoft\hcsshim + +environment: + GOPATH: c:\gopath + PATH: C:\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev1\mingw64\bin;%GOPATH%\bin;C:\gometalinter-2.0.12-windows-amd64;%PATH% + +stack: go 1.11 + +build_script: + - appveyor DownloadFile https://github.com/alecthomas/gometalinter/releases/download/v2.0.12/gometalinter-2.0.12-windows-amd64.zip + - 7z x gometalinter-2.0.12-windows-amd64.zip -y -oC:\ > NUL + - gometalinter.exe --config .gometalinter.json ./... + - go build ./cmd/wclayer + - go build ./cmd/runhcs + - go build ./cmd/tar2ext4 + - go test -v ./... -tags admin + - go test -c ./test/functional/ -tags functional + - go test -c ./test/runhcs/ -tags integration + +artifacts: + - path: 'wclayer.exe' + - path: 'runhcs.exe' + - path: 'tar2ext4.exe' + - path: 'functional.test.exe' - path: 'runhcs.test.exe' \ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/hcn/hcn.go b/vendor/github.com/Microsoft/hcsshim/hcn/hcn.go new file mode 100644 index 0000000000..8bae5fc0ee --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcn/hcn.go @@ -0,0 +1,177 @@ +// Package hcn is a shim for the Host Compute Networking (HCN) service, which manages networking for Windows Server +// containers and Hyper-V containers. Previous to RS5, HCN was referred to as Host Networking Service (HNS). +package hcn + +import ( + "encoding/json" + "fmt" + "syscall" + + "github.com/Microsoft/hcsshim/internal/guid" +) + +//go:generate go run ../mksyscall_windows.go -output zsyscall_windows.go hcn.go + +/// HNS V1 API + +//sys SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) = iphlpapi.SetCurrentThreadCompartmentId +//sys _hnsCall(method string, path string, object string, response **uint16) (hr error) = vmcompute.HNSCall? + +/// HCN V2 API + +// Network +//sys hcnEnumerateNetworks(query string, networks **uint16, result **uint16) (hr error) = computenetwork.HcnEnumerateNetworks? +//sys hcnCreateNetwork(id *_guid, settings string, network *hcnNetwork, result **uint16) (hr error) = computenetwork.HcnCreateNetwork? +//sys hcnOpenNetwork(id *_guid, network *hcnNetwork, result **uint16) (hr error) = computenetwork.HcnOpenNetwork? +//sys hcnModifyNetwork(network hcnNetwork, settings string, result **uint16) (hr error) = computenetwork.HcnModifyNetwork? +//sys hcnQueryNetworkProperties(network hcnNetwork, query string, properties **uint16, result **uint16) (hr error) = computenetwork.HcnQueryNetworkProperties? +//sys hcnDeleteNetwork(id *_guid, result **uint16) (hr error) = computenetwork.HcnDeleteNetwork? +//sys hcnCloseNetwork(network hcnNetwork) (hr error) = computenetwork.HcnCloseNetwork? + +// Endpoint +//sys hcnEnumerateEndpoints(query string, endpoints **uint16, result **uint16) (hr error) = computenetwork.HcnEnumerateEndpoints? +//sys hcnCreateEndpoint(network hcnNetwork, id *_guid, settings string, endpoint *hcnEndpoint, result **uint16) (hr error) = computenetwork.HcnCreateEndpoint? +//sys hcnOpenEndpoint(id *_guid, endpoint *hcnEndpoint, result **uint16) (hr error) = computenetwork.HcnOpenEndpoint? +//sys hcnModifyEndpoint(endpoint hcnEndpoint, settings string, result **uint16) (hr error) = computenetwork.HcnModifyEndpoint? +//sys hcnQueryEndpointProperties(endpoint hcnEndpoint, query string, properties **uint16, result **uint16) (hr error) = computenetwork.HcnQueryEndpointProperties? +//sys hcnDeleteEndpoint(id *_guid, result **uint16) (hr error) = computenetwork.HcnDeleteEndpoint? +//sys hcnCloseEndpoint(endpoint hcnEndpoint) (hr error) = computenetwork.HcnCloseEndpoint? + +// Namespace +//sys hcnEnumerateNamespaces(query string, namespaces **uint16, result **uint16) (hr error) = computenetwork.HcnEnumerateNamespaces? +//sys hcnCreateNamespace(id *_guid, settings string, namespace *hcnNamespace, result **uint16) (hr error) = computenetwork.HcnCreateNamespace? +//sys hcnOpenNamespace(id *_guid, namespace *hcnNamespace, result **uint16) (hr error) = computenetwork.HcnOpenNamespace? +//sys hcnModifyNamespace(namespace hcnNamespace, settings string, result **uint16) (hr error) = computenetwork.HcnModifyNamespace? +//sys hcnQueryNamespaceProperties(namespace hcnNamespace, query string, properties **uint16, result **uint16) (hr error) = computenetwork.HcnQueryNamespaceProperties? +//sys hcnDeleteNamespace(id *_guid, result **uint16) (hr error) = computenetwork.HcnDeleteNamespace? +//sys hcnCloseNamespace(namespace hcnNamespace) (hr error) = computenetwork.HcnCloseNamespace? + +// LoadBalancer +//sys hcnEnumerateLoadBalancers(query string, loadBalancers **uint16, result **uint16) (hr error) = computenetwork.HcnEnumerateLoadBalancers? +//sys hcnCreateLoadBalancer(id *_guid, settings string, loadBalancer *hcnLoadBalancer, result **uint16) (hr error) = computenetwork.HcnCreateLoadBalancer? +//sys hcnOpenLoadBalancer(id *_guid, loadBalancer *hcnLoadBalancer, result **uint16) (hr error) = computenetwork.HcnOpenLoadBalancer? +//sys hcnModifyLoadBalancer(loadBalancer hcnLoadBalancer, settings string, result **uint16) (hr error) = computenetwork.HcnModifyLoadBalancer? +//sys hcnQueryLoadBalancerProperties(loadBalancer hcnLoadBalancer, query string, properties **uint16, result **uint16) (hr error) = computenetwork.HcnQueryLoadBalancerProperties? +//sys hcnDeleteLoadBalancer(id *_guid, result **uint16) (hr error) = computenetwork.HcnDeleteLoadBalancer? +//sys hcnCloseLoadBalancer(loadBalancer hcnLoadBalancer) (hr error) = computenetwork.HcnCloseLoadBalancer? + +// Service +//sys hcnOpenService(service *hcnService, result **uint16) (hr error) = computenetwork.HcnOpenService? +//sys hcnRegisterServiceCallback(service hcnService, callback int32, context int32, callbackHandle *hcnCallbackHandle) (hr error) = computenetwork.HcnRegisterServiceCallback? +//sys hcnUnregisterServiceCallback(callbackHandle hcnCallbackHandle) (hr error) = computenetwork.HcnUnregisterServiceCallback? +//sys hcnCloseService(service hcnService) (hr error) = computenetwork.HcnCloseService? + +type _guid = guid.GUID + +type hcnNetwork syscall.Handle +type hcnEndpoint syscall.Handle +type hcnNamespace syscall.Handle +type hcnLoadBalancer syscall.Handle +type hcnService syscall.Handle +type hcnCallbackHandle syscall.Handle + +// SchemaVersion for HCN Objects/Queries. +type SchemaVersion = Version // hcnglobals.go + +// HostComputeQueryFlags are passed in to a HostComputeQuery to determine which +// properties of an object are returned. +type HostComputeQueryFlags uint32 + +var ( + // HostComputeQueryFlagsNone returns an object with the standard properties. + HostComputeQueryFlagsNone HostComputeQueryFlags + // HostComputeQueryFlagsDetailed returns an object with all properties. + HostComputeQueryFlagsDetailed HostComputeQueryFlags = 1 +) + +// HostComputeQuery is the format for HCN queries. +type HostComputeQuery struct { + SchemaVersion SchemaVersion `json:""` + Flags HostComputeQueryFlags `json:",omitempty"` + Filter string `json:",omitempty"` +} + +// defaultQuery generates HCN Query. +// Passed into get/enumerate calls to filter results. +func defaultQuery() HostComputeQuery { + query := HostComputeQuery{ + SchemaVersion: SchemaVersion{ + Major: 2, + Minor: 0, + }, + Flags: HostComputeQueryFlagsNone, + } + return query +} + +func defaultQueryJson() string { + query := defaultQuery() + queryJson, err := json.Marshal(query) + if err != nil { + return "" + } + return string(queryJson) +} + +// PlatformDoesNotSupportError happens when users are attempting to use a newer shim on an older OS +func platformDoesNotSupportError(featureName string) error { + return fmt.Errorf("Platform does not support feature %s", featureName) +} + +// V2ApiSupported returns an error if the HCN version does not support the V2 Apis. +func V2ApiSupported() error { + supported := GetSupportedFeatures() + if supported.Api.V2 { + return nil + } + return platformDoesNotSupportError("V2 Api/Schema") +} + +func V2SchemaVersion() SchemaVersion { + return SchemaVersion{ + Major: 2, + Minor: 0, + } +} + +// RemoteSubnetSupported returns an error if the HCN version does not support Remote Subnet policies. +func RemoteSubnetSupported() error { + supported := GetSupportedFeatures() + if supported.RemoteSubnet { + return nil + } + return platformDoesNotSupportError("Remote Subnet") +} + +// HostRouteSupported returns an error if the HCN version does not support Host Route policies. +func HostRouteSupported() error { + supported := GetSupportedFeatures() + if supported.HostRoute { + return nil + } + return platformDoesNotSupportError("Host Route") +} + +// DSRSupported returns an error if the HCN version does not support Direct Server Return. +func DSRSupported() error { + supported := GetSupportedFeatures() + if supported.DSR { + return nil + } + return platformDoesNotSupportError("Direct Server Return (DSR)") +} + +// RequestType are the different operations performed to settings. +// Used to update the settings of Endpoint/Namespace objects. +type RequestType string + +var ( + // RequestTypeAdd adds the provided settings object. + RequestTypeAdd RequestType = "Add" + // RequestTypeRemove removes the provided settings object. + RequestTypeRemove RequestType = "Remove" + // RequestTypeUpdate replaces settings with the ones provided. + RequestTypeUpdate RequestType = "Update" + // RequestTypeRefresh refreshes the settings provided. + RequestTypeRefresh RequestType = "Refresh" +) diff --git a/vendor/github.com/Microsoft/hcsshim/hcn/hcnendpoint.go b/vendor/github.com/Microsoft/hcsshim/hcn/hcnendpoint.go new file mode 100644 index 0000000000..d58335e5c6 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcn/hcnendpoint.go @@ -0,0 +1,366 @@ +package hcn + +import ( + "encoding/json" + + "github.com/Microsoft/hcsshim/internal/guid" + "github.com/Microsoft/hcsshim/internal/interop" + "github.com/sirupsen/logrus" +) + +// IpConfig is assoicated with an endpoint +type IpConfig struct { + IpAddress string `json:",omitempty"` + PrefixLength uint8 `json:",omitempty"` +} + +// EndpointFlags are special settings on an endpoint. +type EndpointFlags uint32 + +var ( + // EndpointFlagsNone is the default. + EndpointFlagsNone EndpointFlags + // EndpointFlagsRemoteEndpoint means that an endpoint is on another host. + EndpointFlagsRemoteEndpoint EndpointFlags = 1 +) + +// HostComputeEndpoint represents a network endpoint +type HostComputeEndpoint struct { + Id string `json:"ID,omitempty"` + Name string `json:",omitempty"` + HostComputeNetwork string `json:",omitempty"` // GUID + HostComputeNamespace string `json:",omitempty"` // GUID + Policies []EndpointPolicy `json:",omitempty"` + IpConfigurations []IpConfig `json:",omitempty"` + Dns Dns `json:",omitempty"` + Routes []Route `json:",omitempty"` + MacAddress string `json:",omitempty"` + Flags EndpointFlags `json:",omitempty"` + SchemaVersion SchemaVersion `json:",omitempty"` +} + +// EndpointResourceType are the two different Endpoint settings resources. +type EndpointResourceType string + +var ( + // EndpointResourceTypePolicy is for Endpoint Policies. Ex: ACL, NAT + EndpointResourceTypePolicy EndpointResourceType = "Policy" + // EndpointResourceTypePort is for Endpoint Port settings. + EndpointResourceTypePort EndpointResourceType = "Port" +) + +// ModifyEndpointSettingRequest is the structure used to send request to modify an endpoint. +// Used to update policy/port on an endpoint. +type ModifyEndpointSettingRequest struct { + ResourceType EndpointResourceType `json:",omitempty"` // Policy, Port + RequestType RequestType `json:",omitempty"` // Add, Remove, Update, Refresh + Settings json.RawMessage `json:",omitempty"` +} + +type PolicyEndpointRequest struct { + Policies []EndpointPolicy `json:",omitempty"` +} + +func getEndpoint(endpointGuid guid.GUID, query string) (*HostComputeEndpoint, error) { + // Open endpoint. + var ( + endpointHandle hcnEndpoint + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + hr := hcnOpenEndpoint(&endpointGuid, &endpointHandle, &resultBuffer) + if err := checkForErrors("hcnOpenEndpoint", hr, resultBuffer); err != nil { + return nil, err + } + // Query endpoint. + hr = hcnQueryEndpointProperties(endpointHandle, query, &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryEndpointProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close endpoint. + hr = hcnCloseEndpoint(endpointHandle) + if err := checkForErrors("hcnCloseEndpoint", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeEndpoint + var outputEndpoint HostComputeEndpoint + if err := json.Unmarshal([]byte(properties), &outputEndpoint); err != nil { + return nil, err + } + return &outputEndpoint, nil +} + +func enumerateEndpoints(query string) ([]HostComputeEndpoint, error) { + // Enumerate all Endpoint Guids + var ( + resultBuffer *uint16 + endpointBuffer *uint16 + ) + hr := hcnEnumerateEndpoints(query, &endpointBuffer, &resultBuffer) + if err := checkForErrors("hcnEnumerateEndpoints", hr, resultBuffer); err != nil { + return nil, err + } + + endpoints := interop.ConvertAndFreeCoTaskMemString(endpointBuffer) + var endpointIds []guid.GUID + err := json.Unmarshal([]byte(endpoints), &endpointIds) + if err != nil { + return nil, err + } + + var outputEndpoints []HostComputeEndpoint + for _, endpointGuid := range endpointIds { + endpoint, err := getEndpoint(endpointGuid, query) + if err != nil { + return nil, err + } + outputEndpoints = append(outputEndpoints, *endpoint) + } + return outputEndpoints, nil +} + +func createEndpoint(networkId string, endpointSettings string) (*HostComputeEndpoint, error) { + networkGuid := guid.FromString(networkId) + // Open network. + var networkHandle hcnNetwork + var resultBuffer *uint16 + hr := hcnOpenNetwork(&networkGuid, &networkHandle, &resultBuffer) + if err := checkForErrors("hcnOpenNetwork", hr, resultBuffer); err != nil { + return nil, err + } + // Create endpoint. + endpointId := guid.GUID{} + var endpointHandle hcnEndpoint + hr = hcnCreateEndpoint(networkHandle, &endpointId, endpointSettings, &endpointHandle, &resultBuffer) + if err := checkForErrors("hcnCreateEndpoint", hr, resultBuffer); err != nil { + return nil, err + } + // Query endpoint. + hcnQuery := defaultQuery() + query, err := json.Marshal(hcnQuery) + if err != nil { + return nil, err + } + var propertiesBuffer *uint16 + hr = hcnQueryEndpointProperties(endpointHandle, string(query), &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryEndpointProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close endpoint. + hr = hcnCloseEndpoint(endpointHandle) + if err := checkForErrors("hcnCloseEndpoint", hr, nil); err != nil { + return nil, err + } + // Close network. + hr = hcnCloseNetwork(networkHandle) + if err := checkForErrors("hcnCloseNetwork", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeEndpoint + var outputEndpoint HostComputeEndpoint + if err := json.Unmarshal([]byte(properties), &outputEndpoint); err != nil { + return nil, err + } + return &outputEndpoint, nil +} + +func modifyEndpoint(endpointId string, settings string) (*HostComputeEndpoint, error) { + endpointGuid := guid.FromString(endpointId) + // Open endpoint + var ( + endpointHandle hcnEndpoint + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + hr := hcnOpenEndpoint(&endpointGuid, &endpointHandle, &resultBuffer) + if err := checkForErrors("hcnOpenEndpoint", hr, resultBuffer); err != nil { + return nil, err + } + // Modify endpoint + hr = hcnModifyEndpoint(endpointHandle, settings, &resultBuffer) + if err := checkForErrors("hcnModifyEndpoint", hr, resultBuffer); err != nil { + return nil, err + } + // Query endpoint. + hcnQuery := defaultQuery() + query, err := json.Marshal(hcnQuery) + if err != nil { + return nil, err + } + hr = hcnQueryEndpointProperties(endpointHandle, string(query), &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryEndpointProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close endpoint. + hr = hcnCloseEndpoint(endpointHandle) + if err := checkForErrors("hcnCloseEndpoint", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeEndpoint + var outputEndpoint HostComputeEndpoint + if err := json.Unmarshal([]byte(properties), &outputEndpoint); err != nil { + return nil, err + } + return &outputEndpoint, nil +} + +func deleteEndpoint(endpointId string) error { + endpointGuid := guid.FromString(endpointId) + var resultBuffer *uint16 + hr := hcnDeleteEndpoint(&endpointGuid, &resultBuffer) + if err := checkForErrors("hcnDeleteEndpoint", hr, resultBuffer); err != nil { + return err + } + return nil +} + +// ListEndpoints makes a call to list all available endpoints. +func ListEndpoints() ([]HostComputeEndpoint, error) { + hcnQuery := defaultQuery() + endpoints, err := ListEndpointsQuery(hcnQuery) + if err != nil { + return nil, err + } + return endpoints, nil +} + +// ListEndpointsQuery makes a call to query the list of available endpoints. +func ListEndpointsQuery(query HostComputeQuery) ([]HostComputeEndpoint, error) { + queryJson, err := json.Marshal(query) + if err != nil { + return nil, err + } + + endpoints, err := enumerateEndpoints(string(queryJson)) + if err != nil { + return nil, err + } + return endpoints, nil +} + +// ListEndpointsOfNetwork queries the list of endpoints on a network. +func ListEndpointsOfNetwork(networkId string) ([]HostComputeEndpoint, error) { + hcnQuery := defaultQuery() + // TODO: Once query can convert schema, change to {HostComputeNetwork:networkId} + mapA := map[string]string{"VirtualNetwork": networkId} + filter, err := json.Marshal(mapA) + if err != nil { + return nil, err + } + hcnQuery.Filter = string(filter) + + return ListEndpointsQuery(hcnQuery) +} + +// GetEndpointByID returns an endpoint specified by Id +func GetEndpointByID(endpointId string) (*HostComputeEndpoint, error) { + hcnQuery := defaultQuery() + mapA := map[string]string{"ID": endpointId} + filter, err := json.Marshal(mapA) + if err != nil { + return nil, err + } + hcnQuery.Filter = string(filter) + + endpoints, err := ListEndpointsQuery(hcnQuery) + if err != nil { + return nil, err + } + if len(endpoints) == 0 { + return nil, EndpointNotFoundError{EndpointID: endpointId} + } + return &endpoints[0], err +} + +// GetEndpointByName returns an endpoint specified by Name +func GetEndpointByName(endpointName string) (*HostComputeEndpoint, error) { + hcnQuery := defaultQuery() + mapA := map[string]string{"Name": endpointName} + filter, err := json.Marshal(mapA) + if err != nil { + return nil, err + } + hcnQuery.Filter = string(filter) + + endpoints, err := ListEndpointsQuery(hcnQuery) + if err != nil { + return nil, err + } + if len(endpoints) == 0 { + return nil, EndpointNotFoundError{EndpointName: endpointName} + } + return &endpoints[0], err +} + +// Create Endpoint. +func (endpoint *HostComputeEndpoint) Create() (*HostComputeEndpoint, error) { + logrus.Debugf("hcn::HostComputeEndpoint::Create id=%s", endpoint.Id) + + jsonString, err := json.Marshal(endpoint) + if err != nil { + return nil, err + } + + logrus.Debugf("hcn::HostComputeEndpoint::Create JSON: %s", jsonString) + endpoint, hcnErr := createEndpoint(endpoint.HostComputeNetwork, string(jsonString)) + if hcnErr != nil { + return nil, hcnErr + } + return endpoint, nil +} + +// Delete Endpoint. +func (endpoint *HostComputeEndpoint) Delete() error { + logrus.Debugf("hcn::HostComputeEndpoint::Delete id=%s", endpoint.Id) + + if err := deleteEndpoint(endpoint.Id); err != nil { + return err + } + return nil +} + +// ModifyEndpointSettings updates the Port/Policy of an Endpoint. +func ModifyEndpointSettings(endpointId string, request *ModifyEndpointSettingRequest) error { + logrus.Debugf("hcn::HostComputeEndpoint::ModifyEndpointSettings id=%s", endpointId) + + endpointSettingsRequest, err := json.Marshal(request) + if err != nil { + return err + } + + _, err = modifyEndpoint(endpointId, string(endpointSettingsRequest)) + if err != nil { + return err + } + return nil +} + +// ApplyPolicy applies a Policy (ex: ACL) on the Endpoint. +func (endpoint *HostComputeEndpoint) ApplyPolicy(endpointPolicy PolicyEndpointRequest) error { + logrus.Debugf("hcn::HostComputeEndpoint::ApplyPolicy id=%s", endpoint.Id) + + settingsJson, err := json.Marshal(endpointPolicy) + if err != nil { + return err + } + requestMessage := &ModifyEndpointSettingRequest{ + ResourceType: EndpointResourceTypePolicy, + RequestType: RequestTypeUpdate, + Settings: settingsJson, + } + + return ModifyEndpointSettings(endpoint.Id, requestMessage) +} + +// NamespaceAttach modifies a Namespace to add an endpoint. +func (endpoint *HostComputeEndpoint) NamespaceAttach(namespaceId string) error { + return AddNamespaceEndpoint(namespaceId, endpoint.Id) +} + +// NamespaceDetach modifies a Namespace to remove an endpoint. +func (endpoint *HostComputeEndpoint) NamespaceDetach(namespaceId string) error { + return RemoveNamespaceEndpoint(namespaceId, endpoint.Id) +} diff --git a/vendor/github.com/Microsoft/hcsshim/hcn/hcnerrors.go b/vendor/github.com/Microsoft/hcsshim/hcn/hcnerrors.go new file mode 100644 index 0000000000..6d46bf8bbc --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcn/hcnerrors.go @@ -0,0 +1,95 @@ +// Package hcn is a shim for the Host Compute Networking (HCN) service, which manages networking for Windows Server +// containers and Hyper-V containers. Previous to RS5, HCN was referred to as Host Networking Service (HNS). +package hcn + +import ( + "fmt" + + "github.com/Microsoft/hcsshim/internal/hcserror" + "github.com/Microsoft/hcsshim/internal/interop" + "github.com/sirupsen/logrus" +) + +func checkForErrors(methodName string, hr error, resultBuffer *uint16) error { + errorFound := false + + if hr != nil { + errorFound = true + } + + result := "" + if resultBuffer != nil { + result = interop.ConvertAndFreeCoTaskMemString(resultBuffer) + if result != "" { + errorFound = true + } + } + + if errorFound { + returnError := hcserror.New(hr, methodName, result) + logrus.Debugf(returnError.Error()) // HCN errors logged for debugging. + return returnError + } + + return nil +} + +// NetworkNotFoundError results from a failed seach for a network by Id or Name +type NetworkNotFoundError struct { + NetworkName string + NetworkID string +} + +func (e NetworkNotFoundError) Error() string { + if e.NetworkName == "" { + return fmt.Sprintf("Network Name %s not found", e.NetworkName) + } + return fmt.Sprintf("Network Id %s not found", e.NetworkID) +} + +// EndpointNotFoundError results from a failed seach for an endpoint by Id or Name +type EndpointNotFoundError struct { + EndpointName string + EndpointID string +} + +func (e EndpointNotFoundError) Error() string { + if e.EndpointName == "" { + return fmt.Sprintf("Endpoint Name %s not found", e.EndpointName) + } + return fmt.Sprintf("Endpoint Id %s not found", e.EndpointID) +} + +// NamespaceNotFoundError results from a failed seach for a namsepace by Id +type NamespaceNotFoundError struct { + NamespaceID string +} + +func (e NamespaceNotFoundError) Error() string { + return fmt.Sprintf("Namespace %s not found", e.NamespaceID) +} + +// LoadBalancerNotFoundError results from a failed seach for a loadbalancer by Id +type LoadBalancerNotFoundError struct { + LoadBalancerId string +} + +func (e LoadBalancerNotFoundError) Error() string { + return fmt.Sprintf("LoadBalancer %s not found", e.LoadBalancerId) +} + +// IsNotFoundError returns a boolean indicating whether the error was caused by +// a resource not being found. +func IsNotFoundError(err error) bool { + switch err.(type) { + case NetworkNotFoundError: + return true + case EndpointNotFoundError: + return true + case NamespaceNotFoundError: + return true + case LoadBalancerNotFoundError: + return true + } + return false +} diff --git a/vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go b/vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go new file mode 100644 index 0000000000..29d13deac9 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcn/hcnglobals.go @@ -0,0 +1,87 @@ +package hcn + +import ( + "encoding/json" + "fmt" + + "github.com/Microsoft/hcsshim/internal/hcserror" + "github.com/Microsoft/hcsshim/internal/interop" + "github.com/sirupsen/logrus" +) + +// Globals are all global properties of the HCN Service. +type Globals struct { + Version Version `json:"Version"` +} + +// Version is the HCN Service version. +type Version struct { + Major int `json:"Major"` + Minor int `json:"Minor"` +} + +var ( + // HNSVersion1803 added ACL functionality. + HNSVersion1803 = Version{Major: 7, Minor: 2} + // V2ApiSupport allows the use of V2 Api calls and V2 Schema. + V2ApiSupport = Version{Major: 9, Minor: 1} + // Remote Subnet allows for Remote Subnet policies on Overlay networks + RemoteSubnetVersion = Version{Major: 9, Minor: 2} + // A Host Route policy allows for local container to local host communication Overlay networks + HostRouteVersion = Version{Major: 9, Minor: 2} + // HNS 10.2 allows for Direct Server Return for loadbalancing + DSRVersion = Version{Major: 10, Minor: 2} +) + +// GetGlobals returns the global properties of the HCN Service. +func GetGlobals() (*Globals, error) { + var version Version + err := hnsCall("GET", "/globals/version", "", &version) + if err != nil { + return nil, err + } + + globals := &Globals{ + Version: version, + } + + return globals, nil +} + +type hnsResponse struct { + Success bool + Error string + Output json.RawMessage +} + +func hnsCall(method, path, request string, returnResponse interface{}) error { + var responseBuffer *uint16 + logrus.Debugf("[%s]=>[%s] Request : %s", method, path, request) + + err := _hnsCall(method, path, request, &responseBuffer) + if err != nil { + return hcserror.New(err, "hnsCall ", "") + } + response := interop.ConvertAndFreeCoTaskMemString(responseBuffer) + + hnsresponse := &hnsResponse{} + if err = json.Unmarshal([]byte(response), &hnsresponse); err != nil { + return err + } + + if !hnsresponse.Success { + return fmt.Errorf("HNS failed with error : %s", hnsresponse.Error) + } + + if len(hnsresponse.Output) == 0 { + return nil + } + + logrus.Debugf("Network Response : %s", hnsresponse.Output) + err = json.Unmarshal(hnsresponse.Output, returnResponse) + if err != nil { + return err + } + + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/hcn/hcnloadbalancer.go b/vendor/github.com/Microsoft/hcsshim/hcn/hcnloadbalancer.go new file mode 100644 index 0000000000..cff68e1350 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcn/hcnloadbalancer.go @@ -0,0 +1,335 @@ +package hcn + +import ( + "encoding/json" + + "github.com/Microsoft/hcsshim/internal/guid" + "github.com/Microsoft/hcsshim/internal/interop" + "github.com/sirupsen/logrus" +) + +// LoadBalancerPortMapping is associated with HostComputeLoadBalancer +type LoadBalancerPortMapping struct { + Protocol uint32 `json:",omitempty"` // EX: TCP = 6, UDP = 17 + InternalPort uint16 `json:",omitempty"` + ExternalPort uint16 `json:",omitempty"` + Flags LoadBalancerPortMappingFlags `json:",omitempty"` +} + +// HostComputeLoadBalancer represents software load balancer. +type HostComputeLoadBalancer struct { + Id string `json:"ID,omitempty"` + HostComputeEndpoints []string `json:",omitempty"` + SourceVIP string `json:",omitempty"` + FrontendVIPs []string `json:",omitempty"` + PortMappings []LoadBalancerPortMapping `json:",omitempty"` + SchemaVersion SchemaVersion `json:",omitempty"` + Flags LoadBalancerFlags `json:",omitempty"` // 0: None, 1: EnableDirectServerReturn +} + +//LoadBalancerFlags modify settings for a loadbalancer. +type LoadBalancerFlags uint32 + +var ( + // LoadBalancerFlagsNone is the default. + LoadBalancerFlagsNone LoadBalancerFlags = 0 + // LoadBalancerFlagsDSR enables Direct Server Return (DSR) + LoadBalancerFlagsDSR LoadBalancerFlags = 1 +) + +// LoadBalancerPortMappingFlags are special settings on a loadbalancer. +type LoadBalancerPortMappingFlags uint32 + +var ( + // LoadBalancerPortMappingFlagsNone is the default. + LoadBalancerPortMappingFlagsNone LoadBalancerPortMappingFlags + // LoadBalancerPortMappingFlagsILB enables internal loadbalancing. + LoadBalancerPortMappingFlagsILB LoadBalancerPortMappingFlags = 1 + // LoadBalancerPortMappingFlagsLocalRoutedVIP enables VIP access from the host. + LoadBalancerPortMappingFlagsLocalRoutedVIP LoadBalancerPortMappingFlags = 2 + // LoadBalancerPortMappingFlagsUseMux enables DSR for NodePort access of VIP. + LoadBalancerPortMappingFlagsUseMux LoadBalancerPortMappingFlags = 4 + // LoadBalancerPortMappingFlagsPreserveDIP delivers packets with destination IP as the VIP. + LoadBalancerPortMappingFlagsPreserveDIP LoadBalancerPortMappingFlags = 8 +) + +func getLoadBalancer(loadBalancerGuid guid.GUID, query string) (*HostComputeLoadBalancer, error) { + // Open loadBalancer. + var ( + loadBalancerHandle hcnLoadBalancer + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + hr := hcnOpenLoadBalancer(&loadBalancerGuid, &loadBalancerHandle, &resultBuffer) + if err := checkForErrors("hcnOpenLoadBalancer", hr, resultBuffer); err != nil { + return nil, err + } + // Query loadBalancer. + hr = hcnQueryLoadBalancerProperties(loadBalancerHandle, query, &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryLoadBalancerProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close loadBalancer. + hr = hcnCloseLoadBalancer(loadBalancerHandle) + if err := checkForErrors("hcnCloseLoadBalancer", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeLoadBalancer + var outputLoadBalancer HostComputeLoadBalancer + if err := json.Unmarshal([]byte(properties), &outputLoadBalancer); err != nil { + return nil, err + } + return &outputLoadBalancer, nil +} + +func enumerateLoadBalancers(query string) ([]HostComputeLoadBalancer, error) { + // Enumerate all LoadBalancer Guids + var ( + resultBuffer *uint16 + loadBalancerBuffer *uint16 + ) + hr := hcnEnumerateLoadBalancers(query, &loadBalancerBuffer, &resultBuffer) + if err := checkForErrors("hcnEnumerateLoadBalancers", hr, resultBuffer); err != nil { + return nil, err + } + + loadBalancers := interop.ConvertAndFreeCoTaskMemString(loadBalancerBuffer) + var loadBalancerIds []guid.GUID + if err := json.Unmarshal([]byte(loadBalancers), &loadBalancerIds); err != nil { + return nil, err + } + + var outputLoadBalancers []HostComputeLoadBalancer + for _, loadBalancerGuid := range loadBalancerIds { + loadBalancer, err := getLoadBalancer(loadBalancerGuid, query) + if err != nil { + return nil, err + } + outputLoadBalancers = append(outputLoadBalancers, *loadBalancer) + } + return outputLoadBalancers, nil +} + +func createLoadBalancer(settings string) (*HostComputeLoadBalancer, error) { + // Create new loadBalancer. + var ( + loadBalancerHandle hcnLoadBalancer + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + loadBalancerGuid := guid.GUID{} + hr := hcnCreateLoadBalancer(&loadBalancerGuid, settings, &loadBalancerHandle, &resultBuffer) + if err := checkForErrors("hcnCreateLoadBalancer", hr, resultBuffer); err != nil { + return nil, err + } + // Query loadBalancer. + hcnQuery := defaultQuery() + query, err := json.Marshal(hcnQuery) + if err != nil { + return nil, err + } + hr = hcnQueryLoadBalancerProperties(loadBalancerHandle, string(query), &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryLoadBalancerProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close loadBalancer. + hr = hcnCloseLoadBalancer(loadBalancerHandle) + if err := checkForErrors("hcnCloseLoadBalancer", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeLoadBalancer + var outputLoadBalancer HostComputeLoadBalancer + if err := json.Unmarshal([]byte(properties), &outputLoadBalancer); err != nil { + return nil, err + } + return &outputLoadBalancer, nil +} + +func modifyLoadBalancer(loadBalancerId string, settings string) (*HostComputeLoadBalancer, error) { + loadBalancerGuid := guid.FromString(loadBalancerId) + // Open loadBalancer. + var ( + loadBalancerHandle hcnLoadBalancer + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + hr := hcnOpenLoadBalancer(&loadBalancerGuid, &loadBalancerHandle, &resultBuffer) + if err := checkForErrors("hcnOpenLoadBalancer", hr, resultBuffer); err != nil { + return nil, err + } + // Modify loadBalancer. + hr = hcnModifyLoadBalancer(loadBalancerHandle, settings, &resultBuffer) + if err := checkForErrors("hcnModifyLoadBalancer", hr, resultBuffer); err != nil { + return nil, err + } + // Query loadBalancer. + hcnQuery := defaultQuery() + query, err := json.Marshal(hcnQuery) + if err != nil { + return nil, err + } + hr = hcnQueryLoadBalancerProperties(loadBalancerHandle, string(query), &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryLoadBalancerProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close loadBalancer. + hr = hcnCloseLoadBalancer(loadBalancerHandle) + if err := checkForErrors("hcnCloseLoadBalancer", hr, nil); err != nil { + return nil, err + } + // Convert output to LoadBalancer + var outputLoadBalancer HostComputeLoadBalancer + if err := json.Unmarshal([]byte(properties), &outputLoadBalancer); err != nil { + return nil, err + } + return &outputLoadBalancer, nil +} + +func deleteLoadBalancer(loadBalancerId string) error { + loadBalancerGuid := guid.FromString(loadBalancerId) + var resultBuffer *uint16 + hr := hcnDeleteLoadBalancer(&loadBalancerGuid, &resultBuffer) + if err := checkForErrors("hcnDeleteLoadBalancer", hr, resultBuffer); err != nil { + return err + } + return nil +} + +// ListLoadBalancers makes a call to list all available loadBalancers. +func ListLoadBalancers() ([]HostComputeLoadBalancer, error) { + hcnQuery := defaultQuery() + loadBalancers, err := ListLoadBalancersQuery(hcnQuery) + if err != nil { + return nil, err + } + return loadBalancers, nil +} + +// ListLoadBalancersQuery makes a call to query the list of available loadBalancers. +func ListLoadBalancersQuery(query HostComputeQuery) ([]HostComputeLoadBalancer, error) { + queryJson, err := json.Marshal(query) + if err != nil { + return nil, err + } + + loadBalancers, err := enumerateLoadBalancers(string(queryJson)) + if err != nil { + return nil, err + } + return loadBalancers, nil +} + +// GetLoadBalancerByID returns the LoadBalancer specified by Id. +func GetLoadBalancerByID(loadBalancerId string) (*HostComputeLoadBalancer, error) { + hcnQuery := defaultQuery() + mapA := map[string]string{"ID": loadBalancerId} + filter, err := json.Marshal(mapA) + if err != nil { + return nil, err + } + hcnQuery.Filter = string(filter) + + loadBalancers, err := ListLoadBalancersQuery(hcnQuery) + if err != nil { + return nil, err + } + if len(loadBalancers) == 0 { + return nil, LoadBalancerNotFoundError{LoadBalancerId: loadBalancerId} + } + return &loadBalancers[0], err +} + +// Create LoadBalancer. +func (loadBalancer *HostComputeLoadBalancer) Create() (*HostComputeLoadBalancer, error) { + logrus.Debugf("hcn::HostComputeLoadBalancer::Create id=%s", loadBalancer.Id) + + jsonString, err := json.Marshal(loadBalancer) + if err != nil { + return nil, err + } + + logrus.Debugf("hcn::HostComputeLoadBalancer::Create JSON: %s", jsonString) + loadBalancer, hcnErr := createLoadBalancer(string(jsonString)) + if hcnErr != nil { + return nil, hcnErr + } + return loadBalancer, nil +} + +// Delete LoadBalancer. +func (loadBalancer *HostComputeLoadBalancer) Delete() error { + logrus.Debugf("hcn::HostComputeLoadBalancer::Delete id=%s", loadBalancer.Id) + + if err := deleteLoadBalancer(loadBalancer.Id); err != nil { + return err + } + return nil +} + +// AddEndpoint add an endpoint to a LoadBalancer +func (loadBalancer *HostComputeLoadBalancer) AddEndpoint(endpoint *HostComputeEndpoint) (*HostComputeLoadBalancer, error) { + logrus.Debugf("hcn::HostComputeLoadBalancer::AddEndpoint loadBalancer=%s endpoint=%s", loadBalancer.Id, endpoint.Id) + + err := loadBalancer.Delete() + if err != nil { + return nil, err + } + + // Add Endpoint to the Existing List + loadBalancer.HostComputeEndpoints = append(loadBalancer.HostComputeEndpoints, endpoint.Id) + + return loadBalancer.Create() +} + +// RemoveEndpoint removes an endpoint from a LoadBalancer +func (loadBalancer *HostComputeLoadBalancer) RemoveEndpoint(endpoint *HostComputeEndpoint) (*HostComputeLoadBalancer, error) { + logrus.Debugf("hcn::HostComputeLoadBalancer::RemoveEndpoint loadBalancer=%s endpoint=%s", loadBalancer.Id, endpoint.Id) + + err := loadBalancer.Delete() + if err != nil { + return nil, err + } + + // Create a list of all the endpoints besides the one being removed + var endpoints []string + for _, endpointReference := range loadBalancer.HostComputeEndpoints { + if endpointReference == endpoint.Id { + continue + } + endpoints = append(endpoints, endpointReference) + } + loadBalancer.HostComputeEndpoints = endpoints + return loadBalancer.Create() +} + +// AddLoadBalancer for the specified endpoints +func AddLoadBalancer(endpoints []HostComputeEndpoint, flags LoadBalancerFlags, portMappingFlags LoadBalancerPortMappingFlags, sourceVIP string, frontendVIPs []string, protocol uint16, internalPort uint16, externalPort uint16) (*HostComputeLoadBalancer, error) { + logrus.Debugf("hcn::HostComputeLoadBalancer::AddLoadBalancer endpointId=%v, LoadBalancerFlags=%v, LoadBalancerPortMappingFlags=%v, sourceVIP=%s, frontendVIPs=%v, protocol=%v, internalPort=%v, externalPort=%v", endpoints, flags, portMappingFlags, sourceVIP, frontendVIPs, protocol, internalPort, externalPort) + + loadBalancer := &HostComputeLoadBalancer{ + SourceVIP: sourceVIP, + PortMappings: []LoadBalancerPortMapping{ + { + Protocol: uint32(protocol), + InternalPort: internalPort, + ExternalPort: externalPort, + Flags: portMappingFlags, + }, + }, + FrontendVIPs: frontendVIPs, + SchemaVersion: SchemaVersion{ + Major: 2, + Minor: 0, + }, + Flags: flags, + } + + for _, endpoint := range endpoints { + loadBalancer.HostComputeEndpoints = append(loadBalancer.HostComputeEndpoints, endpoint.Id) + } + + return loadBalancer.Create() +} diff --git a/vendor/github.com/Microsoft/hcsshim/hcn/hcnnamespace.go b/vendor/github.com/Microsoft/hcsshim/hcn/hcnnamespace.go new file mode 100644 index 0000000000..6dbef4f254 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcn/hcnnamespace.go @@ -0,0 +1,424 @@ +package hcn + +import ( + "encoding/json" + "os" + "syscall" + + icni "github.com/Microsoft/hcsshim/internal/cni" + "github.com/Microsoft/hcsshim/internal/guid" + "github.com/Microsoft/hcsshim/internal/interop" + "github.com/Microsoft/hcsshim/internal/regstate" + "github.com/Microsoft/hcsshim/internal/runhcs" + "github.com/sirupsen/logrus" +) + +// NamespaceResourceEndpoint represents an Endpoint attached to a Namespace. +type NamespaceResourceEndpoint struct { + Id string `json:"ID,"` +} + +// NamespaceResourceContainer represents a Container attached to a Namespace. +type NamespaceResourceContainer struct { + Id string `json:"ID,"` +} + +// NamespaceResourceType determines whether the Namespace resource is a Container or Endpoint. +type NamespaceResourceType string + +var ( + // NamespaceResourceTypeContainer are contianers associated with a Namespace. + NamespaceResourceTypeContainer NamespaceResourceType = "Container" + // NamespaceResourceTypeEndpoint are endpoints associated with a Namespace. + NamespaceResourceTypeEndpoint NamespaceResourceType = "Endpoint" +) + +// NamespaceResource is associated with a namespace +type NamespaceResource struct { + Type NamespaceResourceType `json:","` // Container, Endpoint + Data json.RawMessage `json:","` +} + +// NamespaceType determines whether the Namespace is for a Host or Guest +type NamespaceType string + +var ( + // NamespaceTypeHost are host namespaces. + NamespaceTypeHost NamespaceType = "Host" + // NamespaceTypeHostDefault are host namespaces in the default compartment. + NamespaceTypeHostDefault NamespaceType = "HostDefault" + // NamespaceTypeGuest are guest namespaces. + NamespaceTypeGuest NamespaceType = "Guest" + // NamespaceTypeGuestDefault are guest namespaces in the default compartment. + NamespaceTypeGuestDefault NamespaceType = "GuestDefault" +) + +// HostComputeNamespace represents a namespace (AKA compartment) in +type HostComputeNamespace struct { + Id string `json:"ID,omitempty"` + NamespaceId uint32 `json:",omitempty"` + Type NamespaceType `json:",omitempty"` // Host, HostDefault, Guest, GuestDefault + Resources []NamespaceResource `json:",omitempty"` + SchemaVersion SchemaVersion `json:",omitempty"` +} + +// ModifyNamespaceSettingRequest is the structure used to send request to modify a namespace. +// Used to Add/Remove an endpoints and containers to/from a namespace. +type ModifyNamespaceSettingRequest struct { + ResourceType NamespaceResourceType `json:",omitempty"` // Container, Endpoint + RequestType RequestType `json:",omitempty"` // Add, Remove, Update, Refresh + Settings json.RawMessage `json:",omitempty"` +} + +func getNamespace(namespaceGuid guid.GUID, query string) (*HostComputeNamespace, error) { + // Open namespace. + var ( + namespaceHandle hcnNamespace + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + hr := hcnOpenNamespace(&namespaceGuid, &namespaceHandle, &resultBuffer) + if err := checkForErrors("hcnOpenNamespace", hr, resultBuffer); err != nil { + return nil, err + } + // Query namespace. + hr = hcnQueryNamespaceProperties(namespaceHandle, query, &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryNamespaceProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close namespace. + hr = hcnCloseNamespace(namespaceHandle) + if err := checkForErrors("hcnCloseNamespace", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeNamespace + var outputNamespace HostComputeNamespace + if err := json.Unmarshal([]byte(properties), &outputNamespace); err != nil { + return nil, err + } + return &outputNamespace, nil +} + +func enumerateNamespaces(query string) ([]HostComputeNamespace, error) { + // Enumerate all Namespace Guids + var ( + resultBuffer *uint16 + namespaceBuffer *uint16 + ) + hr := hcnEnumerateNamespaces(query, &namespaceBuffer, &resultBuffer) + if err := checkForErrors("hcnEnumerateNamespaces", hr, resultBuffer); err != nil { + return nil, err + } + + namespaces := interop.ConvertAndFreeCoTaskMemString(namespaceBuffer) + var namespaceIds []guid.GUID + if err := json.Unmarshal([]byte(namespaces), &namespaceIds); err != nil { + return nil, err + } + + var outputNamespaces []HostComputeNamespace + for _, namespaceGuid := range namespaceIds { + namespace, err := getNamespace(namespaceGuid, query) + if err != nil { + return nil, err + } + outputNamespaces = append(outputNamespaces, *namespace) + } + return outputNamespaces, nil +} + +func createNamespace(settings string) (*HostComputeNamespace, error) { + // Create new namespace. + var ( + namespaceHandle hcnNamespace + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + namespaceGuid := guid.GUID{} + hr := hcnCreateNamespace(&namespaceGuid, settings, &namespaceHandle, &resultBuffer) + if err := checkForErrors("hcnCreateNamespace", hr, resultBuffer); err != nil { + return nil, err + } + // Query namespace. + hcnQuery := defaultQuery() + query, err := json.Marshal(hcnQuery) + if err != nil { + return nil, err + } + hr = hcnQueryNamespaceProperties(namespaceHandle, string(query), &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryNamespaceProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close namespace. + hr = hcnCloseNamespace(namespaceHandle) + if err := checkForErrors("hcnCloseNamespace", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeNamespace + var outputNamespace HostComputeNamespace + if err := json.Unmarshal([]byte(properties), &outputNamespace); err != nil { + return nil, err + } + return &outputNamespace, nil +} + +func modifyNamespace(namespaceId string, settings string) (*HostComputeNamespace, error) { + namespaceGuid := guid.FromString(namespaceId) + // Open namespace. + var ( + namespaceHandle hcnNamespace + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + hr := hcnOpenNamespace(&namespaceGuid, &namespaceHandle, &resultBuffer) + if err := checkForErrors("hcnOpenNamespace", hr, resultBuffer); err != nil { + return nil, err + } + // Modify namespace. + hr = hcnModifyNamespace(namespaceHandle, settings, &resultBuffer) + if err := checkForErrors("hcnModifyNamespace", hr, resultBuffer); err != nil { + return nil, err + } + // Query namespace. + hcnQuery := defaultQuery() + query, err := json.Marshal(hcnQuery) + if err != nil { + return nil, err + } + hr = hcnQueryNamespaceProperties(namespaceHandle, string(query), &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryNamespaceProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close namespace. + hr = hcnCloseNamespace(namespaceHandle) + if err := checkForErrors("hcnCloseNamespace", hr, nil); err != nil { + return nil, err + } + // Convert output to Namespace + var outputNamespace HostComputeNamespace + if err := json.Unmarshal([]byte(properties), &outputNamespace); err != nil { + return nil, err + } + return &outputNamespace, nil +} + +func deleteNamespace(namespaceId string) error { + namespaceGuid := guid.FromString(namespaceId) + var resultBuffer *uint16 + hr := hcnDeleteNamespace(&namespaceGuid, &resultBuffer) + if err := checkForErrors("hcnDeleteNamespace", hr, resultBuffer); err != nil { + return err + } + return nil +} + +// ListNamespaces makes a call to list all available namespaces. +func ListNamespaces() ([]HostComputeNamespace, error) { + hcnQuery := defaultQuery() + namespaces, err := ListNamespacesQuery(hcnQuery) + if err != nil { + return nil, err + } + return namespaces, nil +} + +// ListNamespacesQuery makes a call to query the list of available namespaces. +func ListNamespacesQuery(query HostComputeQuery) ([]HostComputeNamespace, error) { + queryJson, err := json.Marshal(query) + if err != nil { + return nil, err + } + + namespaces, err := enumerateNamespaces(string(queryJson)) + if err != nil { + return nil, err + } + return namespaces, nil +} + +// GetNamespaceByID returns the Namespace specified by Id. +func GetNamespaceByID(namespaceId string) (*HostComputeNamespace, error) { + return getNamespace(guid.FromString(namespaceId), defaultQueryJson()) +} + +// GetNamespaceEndpointIds returns the endpoints of the Namespace specified by Id. +func GetNamespaceEndpointIds(namespaceId string) ([]string, error) { + namespace, err := GetNamespaceByID(namespaceId) + if err != nil { + return nil, err + } + var endpointsIds []string + for _, resource := range namespace.Resources { + if resource.Type == "Endpoint" { + var endpointResource NamespaceResourceEndpoint + if err := json.Unmarshal([]byte(resource.Data), &endpointResource); err != nil { + return nil, err + } + endpointsIds = append(endpointsIds, endpointResource.Id) + } + } + return endpointsIds, nil +} + +// GetNamespaceContainerIds returns the containers of the Namespace specified by Id. +func GetNamespaceContainerIds(namespaceId string) ([]string, error) { + namespace, err := GetNamespaceByID(namespaceId) + if err != nil { + return nil, err + } + var containerIds []string + for _, resource := range namespace.Resources { + if resource.Type == "Container" { + var contaienrResource NamespaceResourceContainer + if err := json.Unmarshal([]byte(resource.Data), &contaienrResource); err != nil { + return nil, err + } + containerIds = append(containerIds, contaienrResource.Id) + } + } + return containerIds, nil +} + +// NewNamespace creates a new Namespace object +func NewNamespace(nsType NamespaceType) *HostComputeNamespace { + return &HostComputeNamespace{ + Type: nsType, + SchemaVersion: V2SchemaVersion(), + } +} + +// Create Namespace. +func (namespace *HostComputeNamespace) Create() (*HostComputeNamespace, error) { + logrus.Debugf("hcn::HostComputeNamespace::Create id=%s", namespace.Id) + + jsonString, err := json.Marshal(namespace) + if err != nil { + return nil, err + } + + logrus.Debugf("hcn::HostComputeNamespace::Create JSON: %s", jsonString) + namespace, hcnErr := createNamespace(string(jsonString)) + if hcnErr != nil { + return nil, hcnErr + } + return namespace, nil +} + +// Delete Namespace. +func (namespace *HostComputeNamespace) Delete() error { + logrus.Debugf("hcn::HostComputeNamespace::Delete id=%s", namespace.Id) + + if err := deleteNamespace(namespace.Id); err != nil { + return err + } + return nil +} + +// Sync Namespace endpoints with the appropriate sandbox container holding the +// network namespace open. If no sandbox container is found for this namespace +// this method is determined to be a success and will not return an error in +// this case. If the sandbox container is found and a sync is initiated any +// failures will be returned via this method. +// +// This call initiates a sync between endpoints and the matching UtilityVM +// hosting those endpoints. It is safe to call for any `NamespaceType` but +// `NamespaceTypeGuest` is the only case when a sync will actually occur. For +// `NamespaceTypeHost` the process container will be automatically synchronized +// when the the endpoint is added via `AddNamespaceEndpoint`. +// +// Note: This method sync's both additions and removals of endpoints from a +// `NamespaceTypeGuest` namespace. +func (namespace *HostComputeNamespace) Sync() error { + logrus.WithField("id", namespace.Id).Debugf("hcs::HostComputeNamespace::Sync") + + // We only attempt a sync for namespace guest. + if namespace.Type != NamespaceTypeGuest { + return nil + } + + // Look in the registry for the key to map from namespace id to pod-id + cfg, err := icni.LoadPersistedNamespaceConfig(namespace.Id) + if err != nil { + if regstate.IsNotFoundError(err) { + return nil + } + return err + } + req := runhcs.VMRequest{ + ID: cfg.ContainerID, + Op: runhcs.OpSyncNamespace, + } + shimPath := runhcs.VMPipePath(cfg.HostUniqueID) + if err := runhcs.IssueVMRequest(shimPath, &req); err != nil { + // The shim is likey gone. Simply ignore the sync as if it didn't exist. + if perr, ok := err.(*os.PathError); ok && perr.Err == syscall.ERROR_FILE_NOT_FOUND { + // Remove the reg key there is no point to try again + cfg.Remove() + return nil + } + f := map[string]interface{}{ + "id": namespace.Id, + "container-id": cfg.ContainerID, + } + logrus.WithFields(f). + WithError(err). + Debugf("hcs::HostComputeNamespace::Sync failed to connect to shim pipe: '%s'", shimPath) + return err + } + return nil +} + +// ModifyNamespaceSettings updates the Endpoints/Containers of a Namespace. +func ModifyNamespaceSettings(namespaceId string, request *ModifyNamespaceSettingRequest) error { + logrus.Debugf("hcn::HostComputeNamespace::ModifyNamespaceSettings id=%s", namespaceId) + + namespaceSettings, err := json.Marshal(request) + if err != nil { + return err + } + + _, err = modifyNamespace(namespaceId, string(namespaceSettings)) + if err != nil { + return err + } + return nil +} + +// AddNamespaceEndpoint adds an endpoint to a Namespace. +func AddNamespaceEndpoint(namespaceId string, endpointId string) error { + logrus.Debugf("hcn::HostComputeEndpoint::AddNamespaceEndpoint id=%s", endpointId) + + mapA := map[string]string{"EndpointId": endpointId} + settingsJson, err := json.Marshal(mapA) + if err != nil { + return err + } + requestMessage := &ModifyNamespaceSettingRequest{ + ResourceType: NamespaceResourceTypeEndpoint, + RequestType: RequestTypeAdd, + Settings: settingsJson, + } + + return ModifyNamespaceSettings(namespaceId, requestMessage) +} + +// RemoveNamespaceEndpoint removes an endpoint from a Namespace. +func RemoveNamespaceEndpoint(namespaceId string, endpointId string) error { + logrus.Debugf("hcn::HostComputeNamespace::RemoveNamespaceEndpoint id=%s", endpointId) + + mapA := map[string]string{"EndpointId": endpointId} + settingsJson, err := json.Marshal(mapA) + if err != nil { + return err + } + requestMessage := &ModifyNamespaceSettingRequest{ + ResourceType: NamespaceResourceTypeEndpoint, + RequestType: RequestTypeRemove, + Settings: settingsJson, + } + + return ModifyNamespaceSettings(namespaceId, requestMessage) +} diff --git a/vendor/github.com/Microsoft/hcsshim/hcn/hcnnetwork.go b/vendor/github.com/Microsoft/hcsshim/hcn/hcnnetwork.go new file mode 100644 index 0000000000..b5f1db8b22 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcn/hcnnetwork.go @@ -0,0 +1,418 @@ +package hcn + +import ( + "encoding/json" + + "github.com/Microsoft/hcsshim/internal/guid" + "github.com/Microsoft/hcsshim/internal/interop" + "github.com/sirupsen/logrus" +) + +// Route is assoicated with a subnet. +type Route struct { + NextHop string `json:",omitempty"` + DestinationPrefix string `json:",omitempty"` + Metric uint16 `json:",omitempty"` +} + +// Subnet is assoicated with a Ipam. +type Subnet struct { + IpAddressPrefix string `json:",omitempty"` + Policies []json.RawMessage `json:",omitempty"` + Routes []Route `json:",omitempty"` +} + +// Ipam (Internet Protocol Addres Management) is assoicated with a network +// and represents the address space(s) of a network. +type Ipam struct { + Type string `json:",omitempty"` // Ex: Static, DHCP + Subnets []Subnet `json:",omitempty"` +} + +// MacRange is associated with MacPool and respresents the start and end addresses. +type MacRange struct { + StartMacAddress string `json:",omitempty"` + EndMacAddress string `json:",omitempty"` +} + +// MacPool is assoicated with a network and represents pool of MacRanges. +type MacPool struct { + Ranges []MacRange `json:",omitempty"` +} + +// Dns (Domain Name System is associated with a network. +type Dns struct { + Domain string `json:",omitempty"` + Search []string `json:",omitempty"` + ServerList []string `json:",omitempty"` + Options []string `json:",omitempty"` +} + +// NetworkType are various networks. +type NetworkType string + +// NetworkType const +const ( + NAT NetworkType = "NAT" + Transparent NetworkType = "Transparent" + L2Bridge NetworkType = "L2Bridge" + L2Tunnel NetworkType = "L2Tunnel" + ICS NetworkType = "ICS" + Private NetworkType = "Private" + Overlay NetworkType = "Overlay" +) + +// NetworkFlags are various network flags. +type NetworkFlags uint32 + +// NetworkFlags const +const ( + None NetworkFlags = 0 + EnableNonPersistent NetworkFlags = 8 +) + +// HostComputeNetwork represents a network +type HostComputeNetwork struct { + Id string `json:"ID,omitempty"` + Name string `json:",omitempty"` + Type NetworkType `json:",omitempty"` + Policies []NetworkPolicy `json:",omitempty"` + MacPool MacPool `json:",omitempty"` + Dns Dns `json:",omitempty"` + Ipams []Ipam `json:",omitempty"` + Flags NetworkFlags `json:",omitempty"` // 0: None + SchemaVersion SchemaVersion `json:",omitempty"` +} + +// NetworkResourceType are the 3 different Network settings resources. +type NetworkResourceType string + +var ( + // NetworkResourceTypePolicy is for Network's policies. Ex: RemoteSubnet + NetworkResourceTypePolicy NetworkResourceType = "Policy" + // NetworkResourceTypeDNS is for Network's DNS settings. + NetworkResourceTypeDNS NetworkResourceType = "DNS" + // NetworkResourceTypeExtension is for Network's extension settings. + NetworkResourceTypeExtension NetworkResourceType = "Extension" +) + +// ModifyNetworkSettingRequest is the structure used to send request to modify an network. +// Used to update DNS/extension/policy on an network. +type ModifyNetworkSettingRequest struct { + ResourceType NetworkResourceType `json:",omitempty"` // Policy, DNS, Extension + RequestType RequestType `json:",omitempty"` // Add, Remove, Update, Refresh + Settings json.RawMessage `json:",omitempty"` +} + +type PolicyNetworkRequest struct { + Policies []NetworkPolicy `json:",omitempty"` +} + +func getNetwork(networkGuid guid.GUID, query string) (*HostComputeNetwork, error) { + // Open network. + var ( + networkHandle hcnNetwork + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + hr := hcnOpenNetwork(&networkGuid, &networkHandle, &resultBuffer) + if err := checkForErrors("hcnOpenNetwork", hr, resultBuffer); err != nil { + return nil, err + } + // Query network. + hr = hcnQueryNetworkProperties(networkHandle, query, &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryNetworkProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close network. + hr = hcnCloseNetwork(networkHandle) + if err := checkForErrors("hcnCloseNetwork", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeNetwork + var outputNetwork HostComputeNetwork + if err := json.Unmarshal([]byte(properties), &outputNetwork); err != nil { + return nil, err + } + return &outputNetwork, nil +} + +func enumerateNetworks(query string) ([]HostComputeNetwork, error) { + // Enumerate all Network Guids + var ( + resultBuffer *uint16 + networkBuffer *uint16 + ) + hr := hcnEnumerateNetworks(query, &networkBuffer, &resultBuffer) + if err := checkForErrors("hcnEnumerateNetworks", hr, resultBuffer); err != nil { + return nil, err + } + + networks := interop.ConvertAndFreeCoTaskMemString(networkBuffer) + var networkIds []guid.GUID + if err := json.Unmarshal([]byte(networks), &networkIds); err != nil { + return nil, err + } + + var outputNetworks []HostComputeNetwork + for _, networkGuid := range networkIds { + network, err := getNetwork(networkGuid, query) + if err != nil { + return nil, err + } + outputNetworks = append(outputNetworks, *network) + } + return outputNetworks, nil +} + +func createNetwork(settings string) (*HostComputeNetwork, error) { + // Create new network. + var ( + networkHandle hcnNetwork + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + networkGuid := guid.GUID{} + hr := hcnCreateNetwork(&networkGuid, settings, &networkHandle, &resultBuffer) + if err := checkForErrors("hcnCreateNetwork", hr, resultBuffer); err != nil { + return nil, err + } + // Query network. + hcnQuery := defaultQuery() + query, err := json.Marshal(hcnQuery) + if err != nil { + return nil, err + } + hr = hcnQueryNetworkProperties(networkHandle, string(query), &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryNetworkProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close network. + hr = hcnCloseNetwork(networkHandle) + if err := checkForErrors("hcnCloseNetwork", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeNetwork + var outputNetwork HostComputeNetwork + if err := json.Unmarshal([]byte(properties), &outputNetwork); err != nil { + return nil, err + } + return &outputNetwork, nil +} + +func modifyNetwork(networkId string, settings string) (*HostComputeNetwork, error) { + networkGuid := guid.FromString(networkId) + // Open Network + var ( + networkHandle hcnNetwork + resultBuffer *uint16 + propertiesBuffer *uint16 + ) + hr := hcnOpenNetwork(&networkGuid, &networkHandle, &resultBuffer) + if err := checkForErrors("hcnOpenNetwork", hr, resultBuffer); err != nil { + return nil, err + } + // Modify Network + hr = hcnModifyNetwork(networkHandle, settings, &resultBuffer) + if err := checkForErrors("hcnModifyNetwork", hr, resultBuffer); err != nil { + return nil, err + } + // Query network. + hcnQuery := defaultQuery() + query, err := json.Marshal(hcnQuery) + if err != nil { + return nil, err + } + hr = hcnQueryNetworkProperties(networkHandle, string(query), &propertiesBuffer, &resultBuffer) + if err := checkForErrors("hcnQueryNetworkProperties", hr, resultBuffer); err != nil { + return nil, err + } + properties := interop.ConvertAndFreeCoTaskMemString(propertiesBuffer) + // Close network. + hr = hcnCloseNetwork(networkHandle) + if err := checkForErrors("hcnCloseNetwork", hr, nil); err != nil { + return nil, err + } + // Convert output to HostComputeNetwork + var outputNetwork HostComputeNetwork + if err := json.Unmarshal([]byte(properties), &outputNetwork); err != nil { + return nil, err + } + return &outputNetwork, nil +} + +func deleteNetwork(networkId string) error { + networkGuid := guid.FromString(networkId) + var resultBuffer *uint16 + hr := hcnDeleteNetwork(&networkGuid, &resultBuffer) + if err := checkForErrors("hcnDeleteNetwork", hr, resultBuffer); err != nil { + return err + } + return nil +} + +// ListNetworks makes a call to list all available networks. +func ListNetworks() ([]HostComputeNetwork, error) { + hcnQuery := defaultQuery() + networks, err := ListNetworksQuery(hcnQuery) + if err != nil { + return nil, err + } + return networks, nil +} + +// ListNetworksQuery makes a call to query the list of available networks. +func ListNetworksQuery(query HostComputeQuery) ([]HostComputeNetwork, error) { + queryJson, err := json.Marshal(query) + if err != nil { + return nil, err + } + + networks, err := enumerateNetworks(string(queryJson)) + if err != nil { + return nil, err + } + return networks, nil +} + +// GetNetworkByID returns the network specified by Id. +func GetNetworkByID(networkID string) (*HostComputeNetwork, error) { + hcnQuery := defaultQuery() + mapA := map[string]string{"ID": networkID} + filter, err := json.Marshal(mapA) + if err != nil { + return nil, err + } + hcnQuery.Filter = string(filter) + + networks, err := ListNetworksQuery(hcnQuery) + if err != nil { + return nil, err + } + if len(networks) == 0 { + return nil, NetworkNotFoundError{NetworkID: networkID} + } + return &networks[0], err +} + +// GetNetworkByName returns the network specified by Name. +func GetNetworkByName(networkName string) (*HostComputeNetwork, error) { + hcnQuery := defaultQuery() + mapA := map[string]string{"Name": networkName} + filter, err := json.Marshal(mapA) + if err != nil { + return nil, err + } + hcnQuery.Filter = string(filter) + + networks, err := ListNetworksQuery(hcnQuery) + if err != nil { + return nil, err + } + if len(networks) == 0 { + return nil, NetworkNotFoundError{NetworkName: networkName} + } + return &networks[0], err +} + +// Create Network. +func (network *HostComputeNetwork) Create() (*HostComputeNetwork, error) { + logrus.Debugf("hcn::HostComputeNetwork::Create id=%s", network.Id) + + jsonString, err := json.Marshal(network) + if err != nil { + return nil, err + } + + logrus.Debugf("hcn::HostComputeNetwork::Create JSON: %s", jsonString) + network, hcnErr := createNetwork(string(jsonString)) + if hcnErr != nil { + return nil, hcnErr + } + return network, nil +} + +// Delete Network. +func (network *HostComputeNetwork) Delete() error { + logrus.Debugf("hcn::HostComputeNetwork::Delete id=%s", network.Id) + + if err := deleteNetwork(network.Id); err != nil { + return err + } + return nil +} + +// ModifyNetworkSettings updates the Policy for a network. +func (network *HostComputeNetwork) ModifyNetworkSettings(request *ModifyNetworkSettingRequest) error { + logrus.Debugf("hcn::HostComputeNetwork::ModifyNetworkSettings id=%s", network.Id) + + networkSettingsRequest, err := json.Marshal(request) + if err != nil { + return err + } + + _, err = modifyNetwork(network.Id, string(networkSettingsRequest)) + if err != nil { + return err + } + return nil +} + +// AddPolicy applies a Policy (ex: RemoteSubnet) on the Network. +func (network *HostComputeNetwork) AddPolicy(networkPolicy PolicyNetworkRequest) error { + logrus.Debugf("hcn::HostComputeNetwork::AddPolicy id=%s", network.Id) + + settingsJson, err := json.Marshal(networkPolicy) + if err != nil { + return err + } + requestMessage := &ModifyNetworkSettingRequest{ + ResourceType: NetworkResourceTypePolicy, + RequestType: RequestTypeAdd, + Settings: settingsJson, + } + + return network.ModifyNetworkSettings(requestMessage) +} + +// RemovePolicy removes a Policy (ex: RemoteSubnet) from the Network. +func (network *HostComputeNetwork) RemovePolicy(networkPolicy PolicyNetworkRequest) error { + logrus.Debugf("hcn::HostComputeNetwork::RemovePolicy id=%s", network.Id) + + settingsJson, err := json.Marshal(networkPolicy) + if err != nil { + return err + } + requestMessage := &ModifyNetworkSettingRequest{ + ResourceType: NetworkResourceTypePolicy, + RequestType: RequestTypeRemove, + Settings: settingsJson, + } + + return network.ModifyNetworkSettings(requestMessage) +} + +// CreateEndpoint creates an endpoint on the Network. +func (network *HostComputeNetwork) CreateEndpoint(endpoint *HostComputeEndpoint) (*HostComputeEndpoint, error) { + isRemote := endpoint.Flags&EndpointFlagsRemoteEndpoint != 0 + logrus.Debugf("hcn::HostComputeNetwork::CreatEndpoint, networkId=%s remote=%t", network.Id, isRemote) + + endpoint.HostComputeNetwork = network.Id + endpointSettings, err := json.Marshal(endpoint) + if err != nil { + return nil, err + } + newEndpoint, err := createEndpoint(network.Id, string(endpointSettings)) + if err != nil { + return nil, err + } + return newEndpoint, nil +} + +// CreateRemoteEndpoint creates a remote endpoint on the Network. +func (network *HostComputeNetwork) CreateRemoteEndpoint(endpoint *HostComputeEndpoint) (*HostComputeEndpoint, error) { + endpoint.Flags = EndpointFlagsRemoteEndpoint | endpoint.Flags + return network.CreateEndpoint(endpoint) +} diff --git a/vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go b/vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go new file mode 100644 index 0000000000..6b12d73c60 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcn/hcnpolicy.go @@ -0,0 +1,217 @@ +package hcn + +import "encoding/json" + +// EndpointPolicyType are the potential Policies that apply to Endpoints. +type EndpointPolicyType string + +// EndpointPolicyType const +const ( + PortMapping EndpointPolicyType = "PortMapping" + ACL EndpointPolicyType = "ACL" + QOS EndpointPolicyType = "QOS" + L2Driver EndpointPolicyType = "L2Driver" + OutBoundNAT EndpointPolicyType = "OutBoundNAT" + SDNRoute EndpointPolicyType = "SDNRoute" + L4Proxy EndpointPolicyType = "L4Proxy" + PortName EndpointPolicyType = "PortName" + EncapOverhead EndpointPolicyType = "EncapOverhead" + // Endpoint and Network have InterfaceConstraint and ProviderAddress + NetworkProviderAddress EndpointPolicyType = "ProviderAddress" + NetworkInterfaceConstraint EndpointPolicyType = "InterfaceConstraint" +) + +// EndpointPolicy is a collection of Policy settings for an Endpoint. +type EndpointPolicy struct { + Type EndpointPolicyType `json:""` + Settings json.RawMessage `json:",omitempty"` +} + +// NetworkPolicyType are the potential Policies that apply to Networks. +type NetworkPolicyType string + +// NetworkPolicyType const +const ( + SourceMacAddress NetworkPolicyType = "SourceMacAddress" + NetAdapterName NetworkPolicyType = "NetAdapterName" + VSwitchExtension NetworkPolicyType = "VSwitchExtension" + DrMacAddress NetworkPolicyType = "DrMacAddress" + AutomaticDNS NetworkPolicyType = "AutomaticDNS" + InterfaceConstraint NetworkPolicyType = "InterfaceConstraint" + ProviderAddress NetworkPolicyType = "ProviderAddress" + RemoteSubnetRoute NetworkPolicyType = "RemoteSubnetRoute" + HostRoute NetworkPolicyType = "HostRoute" +) + +// NetworkPolicy is a collection of Policy settings for a Network. +type NetworkPolicy struct { + Type NetworkPolicyType `json:""` + Settings json.RawMessage `json:",omitempty"` +} + +// SubnetPolicyType are the potential Policies that apply to Subnets. +type SubnetPolicyType string + +// SubnetPolicyType const +const ( + VLAN SubnetPolicyType = "VLAN" + VSID SubnetPolicyType = "VSID" +) + +// SubnetPolicy is a collection of Policy settings for a Subnet. +type SubnetPolicy struct { + Type SubnetPolicyType `json:""` + Settings json.RawMessage `json:",omitempty"` +} + +/// Endpoint Policy objects + +// PortMappingPolicySetting defines Port Mapping (NAT) +type PortMappingPolicySetting struct { + Protocol uint32 `json:",omitempty"` // EX: TCP = 6, UDP = 17 + InternalPort uint16 `json:",omitempty"` + ExternalPort uint16 `json:",omitempty"` + VIP string `json:",omitempty"` +} + +// ActionType associated with ACLs. Value is either Allow or Block. +type ActionType string + +// DirectionType associated with ACLs. Value is either In or Out. +type DirectionType string + +// RuleType associated with ACLs. Value is either Host (WFP) or Switch (VFP). +type RuleType string + +const ( + // Allow traffic + ActionTypeAllow ActionType = "Allow" + // Block traffic + ActionTypeBlock ActionType = "Block" + + // In is traffic coming to the Endpoint + DirectionTypeIn DirectionType = "In" + // Out is traffic leaving the Endpoint + DirectionTypeOut DirectionType = "Out" + + // Host creates WFP (Windows Firewall) rules + RuleTypeHost RuleType = "Host" + // Switch creates VFP (Virtual Filter Platform) rules + RuleTypeSwitch RuleType = "Switch" +) + +// AclPolicySetting creates firewall rules on an endpoint +type AclPolicySetting struct { + Protocols string `json:",omitempty"` // EX: 6 (TCP), 17 (UDP), 1 (ICMPv4), 58 (ICMPv6), 2 (IGMP) + Action ActionType `json:","` + Direction DirectionType `json:","` + LocalAddresses string `json:",omitempty"` + RemoteAddresses string `json:",omitempty"` + LocalPorts string `json:",omitempty"` + RemotePorts string `json:",omitempty"` + RuleType RuleType `json:",omitempty"` + Priority uint16 `json:",omitempty"` +} + +// QosPolicySetting sets Quality of Service bandwidth caps on an Endpoint. +type QosPolicySetting struct { + MaximumOutgoingBandwidthInBytes uint64 +} + +// OutboundNatPolicySetting sets outbound Network Address Translation on an Endpoint. +type OutboundNatPolicySetting struct { + VirtualIP string `json:",omitempty"` + Exceptions []string `json:",omitempty"` +} + +// SDNRoutePolicySetting sets SDN Route on an Endpoint. +type SDNRoutePolicySetting struct { + DestinationPrefix string `json:",omitempty"` + NextHop string `json:",omitempty"` + NeedEncap bool `json:",omitempty"` +} + +// L4ProxyPolicySetting sets Layer-4 Proxy on an endpoint. +type L4ProxyPolicySetting struct { + IP string `json:",omitempty"` + Port string `json:",omitempty"` + Protocol uint32 `json:",omitempty"` // EX: TCP = 6, UDP = 17 + ExceptionList []string `json:",omitempty"` + Destination string `json:","` + OutboundNat bool `json:",omitempty"` +} + +// PortnameEndpointPolicySetting sets the port name for an endpoint. +type PortnameEndpointPolicySetting struct { + Name string `json:",omitempty"` +} + +// EncapOverheadEndpointPolicySetting sets the encap overhead for an endpoint. +type EncapOverheadEndpointPolicySetting struct { + Overhead uint16 `json:",omitempty"` +} + +/// Endpoint and Network Policy objects + +// ProviderAddressEndpointPolicySetting sets the PA for an endpoint. +type ProviderAddressEndpointPolicySetting struct { + ProviderAddress string `json:",omitempty"` +} + +// InterfaceConstraintPolicySetting limits an Endpoint or Network to a specific Nic. +type InterfaceConstraintPolicySetting struct { + InterfaceGuid string `json:",omitempty"` + InterfaceLuid uint64 `json:",omitempty"` + InterfaceIndex uint32 `json:",omitempty"` + InterfaceMediaType uint32 `json:",omitempty"` + InterfaceAlias string `json:",omitempty"` + InterfaceDescription string `json:",omitempty"` +} + +/// Network Policy objects + +// SourceMacAddressNetworkPolicySetting sets source MAC for a network. +type SourceMacAddressNetworkPolicySetting struct { + SourceMacAddress string `json:",omitempty"` +} + +// NetAdapterNameNetworkPolicySetting sets network adapter of a network. +type NetAdapterNameNetworkPolicySetting struct { + NetworkAdapterName string `json:",omitempty"` +} + +// VSwitchExtensionNetworkPolicySetting enables/disabled VSwitch extensions for a network. +type VSwitchExtensionNetworkPolicySetting struct { + ExtensionID string `json:",omitempty"` + Enable bool `json:",omitempty"` +} + +// DrMacAddressNetworkPolicySetting sets the DR MAC for a network. +type DrMacAddressNetworkPolicySetting struct { + Address string `json:",omitempty"` +} + +// AutomaticDNSNetworkPolicySetting enables/disables automatic DNS on a network. +type AutomaticDNSNetworkPolicySetting struct { + Enable bool `json:",omitempty"` +} + +/// Subnet Policy objects + +// VlanPolicySetting isolates a subnet with VLAN tagging. +type VlanPolicySetting struct { + IsolationId uint32 `json:","` +} + +// VsidPolicySetting isolates a subnet with VSID tagging. +type VsidPolicySetting struct { + IsolationId uint32 `json:","` +} + +// RemoteSubnetRoutePolicySetting creates remote subnet route rules on a network +type RemoteSubnetRoutePolicySetting struct { + DestinationPrefix string + IsolationId uint16 + ProviderAddress string + DistributedRouterMacAddress string +} diff --git a/vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go b/vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go new file mode 100644 index 0000000000..9b5df20301 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcn/hcnsupport.go @@ -0,0 +1,71 @@ +package hcn + +import ( + "github.com/sirupsen/logrus" +) + +// SupportedFeatures are the features provided by the Service. +type SupportedFeatures struct { + Acl AclFeatures `json:"ACL"` + Api ApiSupport `json:"API"` + RemoteSubnet bool `json:"RemoteSubnet"` + HostRoute bool `json:"HostRoute"` + DSR bool `json:"DSR"` +} + +// AclFeatures are the supported ACL possibilities. +type AclFeatures struct { + AclAddressLists bool `json:"AclAddressLists"` + AclNoHostRulePriority bool `json:"AclHostRulePriority"` + AclPortRanges bool `json:"AclPortRanges"` + AclRuleId bool `json:"AclRuleId"` +} + +// ApiSupport lists the supported API versions. +type ApiSupport struct { + V1 bool `json:"V1"` + V2 bool `json:"V2"` +} + +// GetSupportedFeatures returns the features supported by the Service. +func GetSupportedFeatures() SupportedFeatures { + var features SupportedFeatures + + globals, err := GetGlobals() + if err != nil { + // Expected on pre-1803 builds, all features will be false/unsupported + logrus.Debugf("Unable to obtain globals: %s", err) + return features + } + + features.Acl = AclFeatures{ + AclAddressLists: isFeatureSupported(globals.Version, HNSVersion1803), + AclNoHostRulePriority: isFeatureSupported(globals.Version, HNSVersion1803), + AclPortRanges: isFeatureSupported(globals.Version, HNSVersion1803), + AclRuleId: isFeatureSupported(globals.Version, HNSVersion1803), + } + + features.Api = ApiSupport{ + V2: isFeatureSupported(globals.Version, V2ApiSupport), + V1: true, // HNSCall is still available. + } + + features.RemoteSubnet = isFeatureSupported(globals.Version, RemoteSubnetVersion) + features.HostRoute = isFeatureSupported(globals.Version, HostRouteVersion) + features.DSR = isFeatureSupported(globals.Version, DSRVersion) + + return features +} + +func isFeatureSupported(currentVersion Version, minVersionSupported Version) bool { + if currentVersion.Major < minVersionSupported.Major { + return false + } + if currentVersion.Major > minVersionSupported.Major { + return true + } + if currentVersion.Minor < minVersionSupported.Minor { + return false + } + return true +} diff --git a/vendor/github.com/Microsoft/hcsshim/hcn/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/hcn/zsyscall_windows.go new file mode 100644 index 0000000000..856b2c1408 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/hcn/zsyscall_windows.go @@ -0,0 +1,714 @@ +// Code generated mksyscall_windows.exe DO NOT EDIT + +package hcn + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") + modvmcompute = windows.NewLazySystemDLL("vmcompute.dll") + modcomputenetwork = windows.NewLazySystemDLL("computenetwork.dll") + + procSetCurrentThreadCompartmentId = modiphlpapi.NewProc("SetCurrentThreadCompartmentId") + procHNSCall = modvmcompute.NewProc("HNSCall") + procHcnEnumerateNetworks = modcomputenetwork.NewProc("HcnEnumerateNetworks") + procHcnCreateNetwork = modcomputenetwork.NewProc("HcnCreateNetwork") + procHcnOpenNetwork = modcomputenetwork.NewProc("HcnOpenNetwork") + procHcnModifyNetwork = modcomputenetwork.NewProc("HcnModifyNetwork") + procHcnQueryNetworkProperties = modcomputenetwork.NewProc("HcnQueryNetworkProperties") + procHcnDeleteNetwork = modcomputenetwork.NewProc("HcnDeleteNetwork") + procHcnCloseNetwork = modcomputenetwork.NewProc("HcnCloseNetwork") + procHcnEnumerateEndpoints = modcomputenetwork.NewProc("HcnEnumerateEndpoints") + procHcnCreateEndpoint = modcomputenetwork.NewProc("HcnCreateEndpoint") + procHcnOpenEndpoint = modcomputenetwork.NewProc("HcnOpenEndpoint") + procHcnModifyEndpoint = modcomputenetwork.NewProc("HcnModifyEndpoint") + procHcnQueryEndpointProperties = modcomputenetwork.NewProc("HcnQueryEndpointProperties") + procHcnDeleteEndpoint = modcomputenetwork.NewProc("HcnDeleteEndpoint") + procHcnCloseEndpoint = modcomputenetwork.NewProc("HcnCloseEndpoint") + procHcnEnumerateNamespaces = modcomputenetwork.NewProc("HcnEnumerateNamespaces") + procHcnCreateNamespace = modcomputenetwork.NewProc("HcnCreateNamespace") + procHcnOpenNamespace = modcomputenetwork.NewProc("HcnOpenNamespace") + procHcnModifyNamespace = modcomputenetwork.NewProc("HcnModifyNamespace") + procHcnQueryNamespaceProperties = modcomputenetwork.NewProc("HcnQueryNamespaceProperties") + procHcnDeleteNamespace = modcomputenetwork.NewProc("HcnDeleteNamespace") + procHcnCloseNamespace = modcomputenetwork.NewProc("HcnCloseNamespace") + procHcnEnumerateLoadBalancers = modcomputenetwork.NewProc("HcnEnumerateLoadBalancers") + procHcnCreateLoadBalancer = modcomputenetwork.NewProc("HcnCreateLoadBalancer") + procHcnOpenLoadBalancer = modcomputenetwork.NewProc("HcnOpenLoadBalancer") + procHcnModifyLoadBalancer = modcomputenetwork.NewProc("HcnModifyLoadBalancer") + procHcnQueryLoadBalancerProperties = modcomputenetwork.NewProc("HcnQueryLoadBalancerProperties") + procHcnDeleteLoadBalancer = modcomputenetwork.NewProc("HcnDeleteLoadBalancer") + procHcnCloseLoadBalancer = modcomputenetwork.NewProc("HcnCloseLoadBalancer") + procHcnOpenService = modcomputenetwork.NewProc("HcnOpenService") + procHcnRegisterServiceCallback = modcomputenetwork.NewProc("HcnRegisterServiceCallback") + procHcnUnregisterServiceCallback = modcomputenetwork.NewProc("HcnUnregisterServiceCallback") + procHcnCloseService = modcomputenetwork.NewProc("HcnCloseService") +) + +func SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) { + r0, _, _ := syscall.Syscall(procSetCurrentThreadCompartmentId.Addr(), 1, uintptr(compartmentId), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func _hnsCall(method string, path string, object string, response **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(method) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + var _p2 *uint16 + _p2, hr = syscall.UTF16PtrFromString(object) + if hr != nil { + return + } + return __hnsCall(_p0, _p1, _p2, response) +} + +func __hnsCall(method *uint16, path *uint16, object *uint16, response **uint16) (hr error) { + if hr = procHNSCall.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHNSCall.Addr(), 4, uintptr(unsafe.Pointer(method)), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(object)), uintptr(unsafe.Pointer(response)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnEnumerateNetworks(query string, networks **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(query) + if hr != nil { + return + } + return _hcnEnumerateNetworks(_p0, networks, result) +} + +func _hcnEnumerateNetworks(query *uint16, networks **uint16, result **uint16) (hr error) { + if hr = procHcnEnumerateNetworks.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnEnumerateNetworks.Addr(), 3, uintptr(unsafe.Pointer(query)), uintptr(unsafe.Pointer(networks)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnCreateNetwork(id *_guid, settings string, network *hcnNetwork, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(settings) + if hr != nil { + return + } + return _hcnCreateNetwork(id, _p0, network, result) +} + +func _hcnCreateNetwork(id *_guid, settings *uint16, network *hcnNetwork, result **uint16) (hr error) { + if hr = procHcnCreateNetwork.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcnCreateNetwork.Addr(), 4, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(network)), uintptr(unsafe.Pointer(result)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnOpenNetwork(id *_guid, network *hcnNetwork, result **uint16) (hr error) { + if hr = procHcnOpenNetwork.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnOpenNetwork.Addr(), 3, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(network)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnModifyNetwork(network hcnNetwork, settings string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(settings) + if hr != nil { + return + } + return _hcnModifyNetwork(network, _p0, result) +} + +func _hcnModifyNetwork(network hcnNetwork, settings *uint16, result **uint16) (hr error) { + if hr = procHcnModifyNetwork.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnModifyNetwork.Addr(), 3, uintptr(network), uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnQueryNetworkProperties(network hcnNetwork, query string, properties **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(query) + if hr != nil { + return + } + return _hcnQueryNetworkProperties(network, _p0, properties, result) +} + +func _hcnQueryNetworkProperties(network hcnNetwork, query *uint16, properties **uint16, result **uint16) (hr error) { + if hr = procHcnQueryNetworkProperties.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcnQueryNetworkProperties.Addr(), 4, uintptr(network), uintptr(unsafe.Pointer(query)), uintptr(unsafe.Pointer(properties)), uintptr(unsafe.Pointer(result)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnDeleteNetwork(id *_guid, result **uint16) (hr error) { + if hr = procHcnDeleteNetwork.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnDeleteNetwork.Addr(), 2, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(result)), 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnCloseNetwork(network hcnNetwork) (hr error) { + if hr = procHcnCloseNetwork.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnCloseNetwork.Addr(), 1, uintptr(network), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnEnumerateEndpoints(query string, endpoints **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(query) + if hr != nil { + return + } + return _hcnEnumerateEndpoints(_p0, endpoints, result) +} + +func _hcnEnumerateEndpoints(query *uint16, endpoints **uint16, result **uint16) (hr error) { + if hr = procHcnEnumerateEndpoints.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnEnumerateEndpoints.Addr(), 3, uintptr(unsafe.Pointer(query)), uintptr(unsafe.Pointer(endpoints)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnCreateEndpoint(network hcnNetwork, id *_guid, settings string, endpoint *hcnEndpoint, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(settings) + if hr != nil { + return + } + return _hcnCreateEndpoint(network, id, _p0, endpoint, result) +} + +func _hcnCreateEndpoint(network hcnNetwork, id *_guid, settings *uint16, endpoint *hcnEndpoint, result **uint16) (hr error) { + if hr = procHcnCreateEndpoint.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcnCreateEndpoint.Addr(), 5, uintptr(network), uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(endpoint)), uintptr(unsafe.Pointer(result)), 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnOpenEndpoint(id *_guid, endpoint *hcnEndpoint, result **uint16) (hr error) { + if hr = procHcnOpenEndpoint.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnOpenEndpoint.Addr(), 3, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(endpoint)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnModifyEndpoint(endpoint hcnEndpoint, settings string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(settings) + if hr != nil { + return + } + return _hcnModifyEndpoint(endpoint, _p0, result) +} + +func _hcnModifyEndpoint(endpoint hcnEndpoint, settings *uint16, result **uint16) (hr error) { + if hr = procHcnModifyEndpoint.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnModifyEndpoint.Addr(), 3, uintptr(endpoint), uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnQueryEndpointProperties(endpoint hcnEndpoint, query string, properties **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(query) + if hr != nil { + return + } + return _hcnQueryEndpointProperties(endpoint, _p0, properties, result) +} + +func _hcnQueryEndpointProperties(endpoint hcnEndpoint, query *uint16, properties **uint16, result **uint16) (hr error) { + if hr = procHcnQueryEndpointProperties.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcnQueryEndpointProperties.Addr(), 4, uintptr(endpoint), uintptr(unsafe.Pointer(query)), uintptr(unsafe.Pointer(properties)), uintptr(unsafe.Pointer(result)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnDeleteEndpoint(id *_guid, result **uint16) (hr error) { + if hr = procHcnDeleteEndpoint.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnDeleteEndpoint.Addr(), 2, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(result)), 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnCloseEndpoint(endpoint hcnEndpoint) (hr error) { + if hr = procHcnCloseEndpoint.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnCloseEndpoint.Addr(), 1, uintptr(endpoint), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnEnumerateNamespaces(query string, namespaces **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(query) + if hr != nil { + return + } + return _hcnEnumerateNamespaces(_p0, namespaces, result) +} + +func _hcnEnumerateNamespaces(query *uint16, namespaces **uint16, result **uint16) (hr error) { + if hr = procHcnEnumerateNamespaces.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnEnumerateNamespaces.Addr(), 3, uintptr(unsafe.Pointer(query)), uintptr(unsafe.Pointer(namespaces)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnCreateNamespace(id *_guid, settings string, namespace *hcnNamespace, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(settings) + if hr != nil { + return + } + return _hcnCreateNamespace(id, _p0, namespace, result) +} + +func _hcnCreateNamespace(id *_guid, settings *uint16, namespace *hcnNamespace, result **uint16) (hr error) { + if hr = procHcnCreateNamespace.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcnCreateNamespace.Addr(), 4, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(namespace)), uintptr(unsafe.Pointer(result)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnOpenNamespace(id *_guid, namespace *hcnNamespace, result **uint16) (hr error) { + if hr = procHcnOpenNamespace.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnOpenNamespace.Addr(), 3, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(namespace)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnModifyNamespace(namespace hcnNamespace, settings string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(settings) + if hr != nil { + return + } + return _hcnModifyNamespace(namespace, _p0, result) +} + +func _hcnModifyNamespace(namespace hcnNamespace, settings *uint16, result **uint16) (hr error) { + if hr = procHcnModifyNamespace.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnModifyNamespace.Addr(), 3, uintptr(namespace), uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnQueryNamespaceProperties(namespace hcnNamespace, query string, properties **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(query) + if hr != nil { + return + } + return _hcnQueryNamespaceProperties(namespace, _p0, properties, result) +} + +func _hcnQueryNamespaceProperties(namespace hcnNamespace, query *uint16, properties **uint16, result **uint16) (hr error) { + if hr = procHcnQueryNamespaceProperties.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcnQueryNamespaceProperties.Addr(), 4, uintptr(namespace), uintptr(unsafe.Pointer(query)), uintptr(unsafe.Pointer(properties)), uintptr(unsafe.Pointer(result)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnDeleteNamespace(id *_guid, result **uint16) (hr error) { + if hr = procHcnDeleteNamespace.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnDeleteNamespace.Addr(), 2, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(result)), 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnCloseNamespace(namespace hcnNamespace) (hr error) { + if hr = procHcnCloseNamespace.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnCloseNamespace.Addr(), 1, uintptr(namespace), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnEnumerateLoadBalancers(query string, loadBalancers **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(query) + if hr != nil { + return + } + return _hcnEnumerateLoadBalancers(_p0, loadBalancers, result) +} + +func _hcnEnumerateLoadBalancers(query *uint16, loadBalancers **uint16, result **uint16) (hr error) { + if hr = procHcnEnumerateLoadBalancers.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnEnumerateLoadBalancers.Addr(), 3, uintptr(unsafe.Pointer(query)), uintptr(unsafe.Pointer(loadBalancers)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnCreateLoadBalancer(id *_guid, settings string, loadBalancer *hcnLoadBalancer, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(settings) + if hr != nil { + return + } + return _hcnCreateLoadBalancer(id, _p0, loadBalancer, result) +} + +func _hcnCreateLoadBalancer(id *_guid, settings *uint16, loadBalancer *hcnLoadBalancer, result **uint16) (hr error) { + if hr = procHcnCreateLoadBalancer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcnCreateLoadBalancer.Addr(), 4, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(loadBalancer)), uintptr(unsafe.Pointer(result)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnOpenLoadBalancer(id *_guid, loadBalancer *hcnLoadBalancer, result **uint16) (hr error) { + if hr = procHcnOpenLoadBalancer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnOpenLoadBalancer.Addr(), 3, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(loadBalancer)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnModifyLoadBalancer(loadBalancer hcnLoadBalancer, settings string, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(settings) + if hr != nil { + return + } + return _hcnModifyLoadBalancer(loadBalancer, _p0, result) +} + +func _hcnModifyLoadBalancer(loadBalancer hcnLoadBalancer, settings *uint16, result **uint16) (hr error) { + if hr = procHcnModifyLoadBalancer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnModifyLoadBalancer.Addr(), 3, uintptr(loadBalancer), uintptr(unsafe.Pointer(settings)), uintptr(unsafe.Pointer(result))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnQueryLoadBalancerProperties(loadBalancer hcnLoadBalancer, query string, properties **uint16, result **uint16) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(query) + if hr != nil { + return + } + return _hcnQueryLoadBalancerProperties(loadBalancer, _p0, properties, result) +} + +func _hcnQueryLoadBalancerProperties(loadBalancer hcnLoadBalancer, query *uint16, properties **uint16, result **uint16) (hr error) { + if hr = procHcnQueryLoadBalancerProperties.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcnQueryLoadBalancerProperties.Addr(), 4, uintptr(loadBalancer), uintptr(unsafe.Pointer(query)), uintptr(unsafe.Pointer(properties)), uintptr(unsafe.Pointer(result)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnDeleteLoadBalancer(id *_guid, result **uint16) (hr error) { + if hr = procHcnDeleteLoadBalancer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnDeleteLoadBalancer.Addr(), 2, uintptr(unsafe.Pointer(id)), uintptr(unsafe.Pointer(result)), 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnCloseLoadBalancer(loadBalancer hcnLoadBalancer) (hr error) { + if hr = procHcnCloseLoadBalancer.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnCloseLoadBalancer.Addr(), 1, uintptr(loadBalancer), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnOpenService(service *hcnService, result **uint16) (hr error) { + if hr = procHcnOpenService.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnOpenService.Addr(), 2, uintptr(unsafe.Pointer(service)), uintptr(unsafe.Pointer(result)), 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnRegisterServiceCallback(service hcnService, callback int32, context int32, callbackHandle *hcnCallbackHandle) (hr error) { + if hr = procHcnRegisterServiceCallback.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall6(procHcnRegisterServiceCallback.Addr(), 4, uintptr(service), uintptr(callback), uintptr(context), uintptr(unsafe.Pointer(callbackHandle)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnUnregisterServiceCallback(callbackHandle hcnCallbackHandle) (hr error) { + if hr = procHcnUnregisterServiceCallback.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnUnregisterServiceCallback.Addr(), 1, uintptr(callbackHandle), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func hcnCloseService(service hcnService) (hr error) { + if hr = procHcnCloseService.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall(procHcnCloseService.Addr(), 1, uintptr(service), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/cni/registry.go b/vendor/github.com/Microsoft/hcsshim/internal/cni/registry.go new file mode 100644 index 0000000000..842ee1f7af --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/cni/registry.go @@ -0,0 +1,110 @@ +package cni + +import ( + "errors" + + "github.com/Microsoft/hcsshim/internal/guid" + "github.com/Microsoft/hcsshim/internal/regstate" +) + +const ( + cniRoot = "cni" + cniKey = "cfg" +) + +// PersistedNamespaceConfig is the registry version of the `NamespaceID` to UVM +// map. +type PersistedNamespaceConfig struct { + namespaceID string + stored bool + + ContainerID string + HostUniqueID guid.GUID +} + +// NewPersistedNamespaceConfig creates an in-memory namespace config that can be +// persisted to the registry. +func NewPersistedNamespaceConfig(namespaceID, containerID string, containerHostUniqueID guid.GUID) *PersistedNamespaceConfig { + return &PersistedNamespaceConfig{ + namespaceID: namespaceID, + ContainerID: containerID, + HostUniqueID: containerHostUniqueID, + } +} + +// LoadPersistedNamespaceConfig loads a persisted config from the registry that matches +// `namespaceID`. If not found returns `regstate.NotFoundError` +func LoadPersistedNamespaceConfig(namespaceID string) (*PersistedNamespaceConfig, error) { + sk, err := regstate.Open(cniRoot, false) + if err != nil { + return nil, err + } + defer sk.Close() + + pnc := PersistedNamespaceConfig{ + namespaceID: namespaceID, + stored: true, + } + if err := sk.Get(namespaceID, cniKey, &pnc); err != nil { + return nil, err + } + return &pnc, nil +} + +// Store stores or updates the in-memory config to its registry state. If the +// store failes returns the store error. +func (pnc *PersistedNamespaceConfig) Store() error { + if pnc.namespaceID == "" { + return errors.New("invalid namespaceID ''") + } + if pnc.ContainerID == "" { + return errors.New("invalid containerID ''") + } + empty := guid.GUID{} + if pnc.HostUniqueID == empty { + return errors.New("invalid containerHostUniqueID 'empy'") + } + sk, err := regstate.Open(cniRoot, false) + if err != nil { + return err + } + defer sk.Close() + + if pnc.stored { + if err := sk.Set(pnc.namespaceID, cniKey, pnc); err != nil { + return err + } + } else { + if err := sk.Create(pnc.namespaceID, cniKey, pnc); err != nil { + return err + } + } + pnc.stored = true + return nil +} + +// Remove removes any persisted state associated with this config. If the config +// is not found in the registery `Remove` returns no error. +func (pnc *PersistedNamespaceConfig) Remove() error { + if pnc.stored { + sk, err := regstate.Open(cniRoot, false) + if err != nil { + if regstate.IsNotFoundError(err) { + pnc.stored = false + return nil + } + return err + } + defer sk.Close() + + if err := sk.Remove(pnc.namespaceID); err != nil { + if regstate.IsNotFoundError(err) { + pnc.stored = false + return nil + } + return err + } + } + pnc.stored = false + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go b/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go index 24808559bb..e9e45c0306 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/guid/guid.go @@ -1,69 +1,69 @@ -package guid - -import ( - "crypto/rand" - "encoding/json" - "fmt" - "io" - "strconv" - "strings" -) - -var _ = (json.Marshaler)(&GUID{}) -var _ = (json.Unmarshaler)(&GUID{}) - -type GUID [16]byte - -func New() GUID { - g := GUID{} - _, err := io.ReadFull(rand.Reader, g[:]) - if err != nil { - panic(err) - } - return g -} - -func (g GUID) String() string { - return fmt.Sprintf("%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x-%02x", g[3], g[2], g[1], g[0], g[5], g[4], g[7], g[6], g[8:10], g[10:]) -} - -func FromString(s string) GUID { - if len(s) != 36 { - panic(fmt.Sprintf("invalid GUID length: %d", len(s))) - } - if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { - panic("invalid GUID format") - } - indexOrder := [16]int{ - 0, 2, 4, 6, - 9, 11, - 14, 16, - 19, 21, - 24, 26, 28, 30, 32, 34, - } - byteOrder := [16]int{ - 3, 2, 1, 0, - 5, 4, - 7, 6, - 8, 9, - 10, 11, 12, 13, 14, 15, - } - var g GUID - for i, x := range indexOrder { - b, err := strconv.ParseInt(s[x:x+2], 16, 16) - if err != nil { - panic(err) - } - g[byteOrder[i]] = byte(b) - } - return g -} - -func (g GUID) MarshalJSON() ([]byte, error) { - return json.Marshal(g.String()) -} - -func (g *GUID) UnmarshalJSON(data []byte) error { - *g = FromString(strings.Trim(string(data), "\"")) - return nil -} +package guid + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" +) + +var _ = (json.Marshaler)(&GUID{}) +var _ = (json.Unmarshaler)(&GUID{}) + +type GUID [16]byte + +func New() GUID { + g := GUID{} + _, err := io.ReadFull(rand.Reader, g[:]) + if err != nil { + panic(err) + } + return g +} + +func (g GUID) String() string { + return fmt.Sprintf("%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x-%02x", g[3], g[2], g[1], g[0], g[5], g[4], g[7], g[6], g[8:10], g[10:]) +} + +func FromString(s string) GUID { + if len(s) != 36 { + panic(fmt.Sprintf("invalid GUID length: %d", len(s))) + } + if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { + panic("invalid GUID format") + } + indexOrder := [16]int{ + 0, 2, 4, 6, + 9, 11, + 14, 16, + 19, 21, + 24, 26, 28, 30, 32, 34, + } + byteOrder := [16]int{ + 3, 2, 1, 0, + 5, 4, + 7, 6, + 8, 9, + 10, 11, 12, 13, 14, 15, + } + var g GUID + for i, x := range indexOrder { + b, err := strconv.ParseInt(s[x:x+2], 16, 16) + if err != nil { + panic(err) + } + g[byteOrder[i]] = byte(b) + } + return g +} + +func (g GUID) MarshalJSON() ([]byte, error) { + return json.Marshal(g.String()) +} + +func (g *GUID) UnmarshalJSON(data []byte) error { + *g = FromString(strings.Trim(string(data), "\"")) + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/regstate/regstate.go b/vendor/github.com/Microsoft/hcsshim/internal/regstate/regstate.go new file mode 100644 index 0000000000..6c4a641561 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/regstate/regstate.go @@ -0,0 +1,287 @@ +package regstate + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "reflect" + "syscall" + + "golang.org/x/sys/windows/registry" +) + +//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go regstate.go + +//sys regCreateKeyEx(key syscall.Handle, subkey *uint16, reserved uint32, class *uint16, options uint32, desired uint32, sa *syscall.SecurityAttributes, result *syscall.Handle, disposition *uint32) (regerrno error) = advapi32.RegCreateKeyExW + +const ( + _REG_OPTION_VOLATILE = 1 + + _REG_CREATED_NEW_KEY = 1 + _REG_OPENED_EXISTING_KEY = 2 +) + +type Key struct { + registry.Key + Name string +} + +var localMachine = &Key{registry.LOCAL_MACHINE, "HKEY_LOCAL_MACHINE"} +var localUser = &Key{registry.CURRENT_USER, "HKEY_CURRENT_USER"} + +var rootPath = `SOFTWARE\Microsoft\runhcs` + +type NotFoundError struct { + Id string +} + +func (err *NotFoundError) Error() string { + return fmt.Sprintf("ID '%s' was not found", err.Id) +} + +func IsNotFoundError(err error) bool { + _, ok := err.(*NotFoundError) + return ok +} + +type NoStateError struct { + ID string + Key string +} + +func (err *NoStateError) Error() string { + return fmt.Sprintf("state '%s' is not present for ID '%s'", err.Key, err.ID) +} + +func createVolatileKey(k *Key, path string, access uint32) (newk *Key, openedExisting bool, err error) { + var ( + h syscall.Handle + d uint32 + ) + fullpath := filepath.Join(k.Name, path) + err = regCreateKeyEx(syscall.Handle(k.Key), syscall.StringToUTF16Ptr(path), 0, nil, _REG_OPTION_VOLATILE, access, nil, &h, &d) + if err != nil { + return nil, false, &os.PathError{Op: "RegCreateKeyEx", Path: fullpath, Err: err} + } + return &Key{registry.Key(h), fullpath}, d == _REG_OPENED_EXISTING_KEY, nil +} + +func hive(perUser bool) *Key { + r := localMachine + if perUser { + r = localUser + } + return r +} + +func Open(root string, perUser bool) (*Key, error) { + k, _, err := createVolatileKey(hive(perUser), rootPath, registry.ALL_ACCESS) + if err != nil { + return nil, err + } + defer k.Close() + + k2, _, err := createVolatileKey(k, url.PathEscape(root), registry.ALL_ACCESS) + if err != nil { + return nil, err + } + return k2, nil +} + +func RemoveAll(root string, perUser bool) error { + k, err := hive(perUser).open(rootPath) + if err != nil { + return err + } + defer k.Close() + r, err := k.open(url.PathEscape(root)) + if err != nil { + return err + } + defer r.Close() + ids, err := r.Enumerate() + if err != nil { + return err + } + for _, id := range ids { + err = r.Remove(id) + if err != nil { + return err + } + } + r.Close() + return k.Remove(root) +} + +func (k *Key) Close() error { + err := k.Key.Close() + k.Key = 0 + return err +} + +func (k *Key) Enumerate() ([]string, error) { + escapedIDs, err := k.ReadSubKeyNames(0) + if err != nil { + return nil, err + } + var ids []string + for _, e := range escapedIDs { + id, err := url.PathUnescape(e) + if err == nil { + ids = append(ids, id) + } + } + return ids, nil +} + +func (k *Key) open(name string) (*Key, error) { + fullpath := filepath.Join(k.Name, name) + nk, err := registry.OpenKey(k.Key, name, registry.ALL_ACCESS) + if err != nil { + return nil, &os.PathError{Op: "RegOpenKey", Path: fullpath, Err: err} + } + return &Key{nk, fullpath}, nil +} + +func (k *Key) openid(id string) (*Key, error) { + escaped := url.PathEscape(id) + fullpath := filepath.Join(k.Name, escaped) + nk, err := k.open(escaped) + if perr, ok := err.(*os.PathError); ok && perr.Err == syscall.ERROR_FILE_NOT_FOUND { + return nil, &NotFoundError{id} + } + if err != nil { + return nil, &os.PathError{Op: "RegOpenKey", Path: fullpath, Err: err} + } + return nk, nil +} + +func (k *Key) Remove(id string) error { + escaped := url.PathEscape(id) + err := registry.DeleteKey(k.Key, escaped) + if err != nil { + if err == syscall.ERROR_FILE_NOT_FOUND { + return &NotFoundError{id} + } + return &os.PathError{Op: "RegDeleteKey", Path: filepath.Join(k.Name, escaped), Err: err} + } + return nil +} + +func (k *Key) set(id string, create bool, key string, state interface{}) error { + var sk *Key + var err error + if create { + var existing bool + eid := url.PathEscape(id) + sk, existing, err = createVolatileKey(k, eid, registry.ALL_ACCESS) + if err != nil { + return err + } + defer sk.Close() + if existing { + sk.Close() + return fmt.Errorf("container %s already exists", id) + } + } else { + sk, err = k.openid(id) + if err != nil { + return err + } + defer sk.Close() + } + switch reflect.TypeOf(state).Kind() { + case reflect.Bool: + v := uint32(0) + if state.(bool) { + v = 1 + } + err = sk.SetDWordValue(key, v) + case reflect.Int: + err = sk.SetQWordValue(key, uint64(state.(int))) + case reflect.String: + err = sk.SetStringValue(key, state.(string)) + default: + var js []byte + js, err = json.Marshal(state) + if err != nil { + return err + } + err = sk.SetBinaryValue(key, js) + } + if err != nil { + if err == syscall.ERROR_FILE_NOT_FOUND { + return &NoStateError{id, key} + } + return &os.PathError{Op: "RegSetValueEx", Path: sk.Name + ":" + key, Err: err} + } + return nil +} + +func (k *Key) Create(id, key string, state interface{}) error { + return k.set(id, true, key, state) +} + +func (k *Key) Set(id, key string, state interface{}) error { + return k.set(id, false, key, state) +} + +func (k *Key) Clear(id, key string) error { + sk, err := k.openid(id) + if err != nil { + return err + } + defer sk.Close() + err = sk.DeleteValue(key) + if err != nil { + if err == syscall.ERROR_FILE_NOT_FOUND { + return &NoStateError{id, key} + } + return &os.PathError{Op: "RegDeleteValue", Path: sk.Name + ":" + key, Err: err} + } + return nil +} + +func (k *Key) Get(id, key string, state interface{}) error { + sk, err := k.openid(id) + if err != nil { + return err + } + defer sk.Close() + + var js []byte + switch reflect.TypeOf(state).Elem().Kind() { + case reflect.Bool: + var v uint64 + v, _, err = sk.GetIntegerValue(key) + if err == nil { + *state.(*bool) = v != 0 + } + case reflect.Int: + var v uint64 + v, _, err = sk.GetIntegerValue(key) + if err == nil { + *state.(*int) = int(v) + } + case reflect.String: + var v string + v, _, err = sk.GetStringValue(key) + if err == nil { + *state.(*string) = string(v) + } + default: + js, _, err = sk.GetBinaryValue(key) + } + if err != nil { + if err == syscall.ERROR_FILE_NOT_FOUND { + return &NoStateError{id, key} + } + return &os.PathError{Op: "RegQueryValueEx", Path: sk.Name + ":" + key, Err: err} + } + if js != nil { + err = json.Unmarshal(js, state) + } + return err +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/regstate/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/regstate/zsyscall_windows.go new file mode 100644 index 0000000000..4e349ad498 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/regstate/zsyscall_windows.go @@ -0,0 +1,51 @@ +// Code generated by 'go generate'; DO NOT EDIT. + +package regstate + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") + + procRegCreateKeyExW = modadvapi32.NewProc("RegCreateKeyExW") +) + +func regCreateKeyEx(key syscall.Handle, subkey *uint16, reserved uint32, class *uint16, options uint32, desired uint32, sa *syscall.SecurityAttributes, result *syscall.Handle, disposition *uint32) (regerrno error) { + r0, _, _ := syscall.Syscall9(procRegCreateKeyExW.Addr(), 9, uintptr(key), uintptr(unsafe.Pointer(subkey)), uintptr(reserved), uintptr(unsafe.Pointer(class)), uintptr(options), uintptr(desired), uintptr(unsafe.Pointer(sa)), uintptr(unsafe.Pointer(result)), uintptr(unsafe.Pointer(disposition))) + if r0 != 0 { + regerrno = syscall.Errno(r0) + } + return +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/runhcs/container.go b/vendor/github.com/Microsoft/hcsshim/internal/runhcs/container.go new file mode 100644 index 0000000000..3015c3640e --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/runhcs/container.go @@ -0,0 +1,71 @@ +package runhcs + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "syscall" + "time" + + "github.com/Microsoft/hcsshim/internal/guid" +) + +// ContainerState represents the platform agnostic pieces relating to a +// running container's status and state +type ContainerState struct { + // Version is the OCI version for the container + Version string `json:"ociVersion"` + // ID is the container ID + ID string `json:"id"` + // InitProcessPid is the init process id in the parent namespace + InitProcessPid int `json:"pid"` + // Status is the current status of the container, running, paused, ... + Status string `json:"status"` + // Bundle is the path on the filesystem to the bundle + Bundle string `json:"bundle"` + // Rootfs is a path to a directory containing the container's root filesystem. + Rootfs string `json:"rootfs"` + // Created is the unix timestamp for the creation time of the container in UTC + Created time.Time `json:"created"` + // Annotations is the user defined annotations added to the config. + Annotations map[string]string `json:"annotations,omitempty"` + // The owner of the state directory (the owner of the container). + Owner string `json:"owner"` +} + +// GetErrorFromPipe returns reads from `pipe` and verifies if the operation +// returned success or error. If error converts that to an error and returns. If +// `p` is not nill will issue a `Kill` and `Wait` for exit. +func GetErrorFromPipe(pipe io.Reader, p *os.Process) error { + serr, err := ioutil.ReadAll(pipe) + if err != nil { + return err + } + + if bytes.Equal(serr, ShimSuccess) { + return nil + } + + extra := "" + if p != nil { + p.Kill() + state, err := p.Wait() + if err != nil { + panic(err) + } + extra = fmt.Sprintf(", exit code %d", state.Sys().(syscall.WaitStatus).ExitCode) + } + if len(serr) == 0 { + return fmt.Errorf("unknown shim failure%s", extra) + } + + return errors.New(string(serr)) +} + +// VMPipePath returns the named pipe path for the vm shim. +func VMPipePath(hostUniqueID guid.GUID) string { + return SafePipePath("runhcs-vm-" + hostUniqueID.String()) +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/runhcs/util.go b/vendor/github.com/Microsoft/hcsshim/internal/runhcs/util.go new file mode 100644 index 0000000000..dcbb1903b8 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/runhcs/util.go @@ -0,0 +1,16 @@ +package runhcs + +import "net/url" + +const ( + SafePipePrefix = `\\.\pipe\ProtectedPrefix\Administrators\` +) + +// ShimSuccess is the byte stream returned on a successful operation. +var ShimSuccess = []byte{0, 'O', 'K', 0} + +func SafePipePath(name string) string { + // Use a pipe in the Administrators protected prefixed to prevent malicious + // squatting. + return SafePipePrefix + url.PathEscape(name) +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/runhcs/vm.go b/vendor/github.com/Microsoft/hcsshim/internal/runhcs/vm.go new file mode 100644 index 0000000000..2c8957b88d --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/runhcs/vm.go @@ -0,0 +1,43 @@ +package runhcs + +import ( + "encoding/json" + + "github.com/Microsoft/go-winio" +) + +// VMRequestOp is an operation that can be issued to a VM shim. +type VMRequestOp string + +const ( + // OpCreateContainer is a create container request. + OpCreateContainer VMRequestOp = "create" + // OpSyncNamespace is a `cni.NamespaceTypeGuest` sync request with the UVM. + OpSyncNamespace VMRequestOp = "sync" + // OpUnmountContainer is a container unmount request. + OpUnmountContainer VMRequestOp = "unmount" + // OpUnmountContainerDiskOnly is a container unmount disk request. + OpUnmountContainerDiskOnly VMRequestOp = "unmount-disk" +) + +// VMRequest is an operation request that is issued to a VM shim. +type VMRequest struct { + ID string + Op VMRequestOp +} + +// IssueVMRequest issues a request to a shim at the given pipe. +func IssueVMRequest(pipepath string, req *VMRequest) error { + pipe, err := winio.DialPipe(pipepath, nil) + if err != nil { + return err + } + defer pipe.Close() + if err := json.NewEncoder(pipe).Encode(req); err != nil { + return err + } + if err := GetErrorFromPipe(pipe, nil); err != nil { + return err + } + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go b/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go index f0e9447000..995433ace6 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go @@ -1,245 +1,245 @@ -package schema1 - -import ( - "encoding/json" - "time" - - "github.com/Microsoft/hcsshim/internal/schema2" -) - -// ProcessConfig is used as both the input of Container.CreateProcess -// and to convert the parameters to JSON for passing onto the HCS -type ProcessConfig struct { - ApplicationName string `json:",omitempty"` - CommandLine string `json:",omitempty"` - CommandArgs []string `json:",omitempty"` // Used by Linux Containers on Windows - User string `json:",omitempty"` - WorkingDirectory string `json:",omitempty"` - Environment map[string]string `json:",omitempty"` - EmulateConsole bool `json:",omitempty"` - CreateStdInPipe bool `json:",omitempty"` - CreateStdOutPipe bool `json:",omitempty"` - CreateStdErrPipe bool `json:",omitempty"` - ConsoleSize [2]uint `json:",omitempty"` - CreateInUtilityVm bool `json:",omitempty"` // Used by Linux Containers on Windows - OCISpecification *json.RawMessage `json:",omitempty"` // Used by Linux Containers on Windows -} - -type Layer struct { - ID string - Path string -} - -type MappedDir struct { - HostPath string - ContainerPath string - ReadOnly bool - BandwidthMaximum uint64 - IOPSMaximum uint64 - CreateInUtilityVM bool - // LinuxMetadata - Support added in 1803/RS4+. - LinuxMetadata bool `json:",omitempty"` -} - -type MappedPipe struct { - HostPath string - ContainerPipeName string -} - -type HvRuntime struct { - ImagePath string `json:",omitempty"` - SkipTemplate bool `json:",omitempty"` - LinuxInitrdFile string `json:",omitempty"` // File under ImagePath on host containing an initrd image for starting a Linux utility VM - LinuxKernelFile string `json:",omitempty"` // File under ImagePath on host containing a kernel for starting a Linux utility VM - LinuxBootParameters string `json:",omitempty"` // Additional boot parameters for starting a Linux Utility VM in initrd mode - BootSource string `json:",omitempty"` // "Vhd" for Linux Utility VM booting from VHD - WritableBootSource bool `json:",omitempty"` // Linux Utility VM booting from VHD -} - -type MappedVirtualDisk struct { - HostPath string `json:",omitempty"` // Path to VHD on the host - ContainerPath string // Platform-specific mount point path in the container - CreateInUtilityVM bool `json:",omitempty"` - ReadOnly bool `json:",omitempty"` - Cache string `json:",omitempty"` // "" (Unspecified); "Disabled"; "Enabled"; "Private"; "PrivateAllowSharing" - AttachOnly bool `json:",omitempty:` -} - -// AssignedDevice represents a device that has been directly assigned to a container -// -// NOTE: Support added in RS5 -type AssignedDevice struct { - // InterfaceClassGUID of the device to assign to container. - InterfaceClassGUID string `json:"InterfaceClassGuid,omitempty"` -} - -// ContainerConfig is used as both the input of CreateContainer -// and to convert the parameters to JSON for passing onto the HCS -type ContainerConfig struct { - SystemType string // HCS requires this to be hard-coded to "Container" - Name string // Name of the container. We use the docker ID. - Owner string `json:",omitempty"` // The management platform that created this container - VolumePath string `json:",omitempty"` // Windows volume path for scratch space. Used by Windows Server Containers only. Format \\?\\Volume{GUID} - IgnoreFlushesDuringBoot bool `json:",omitempty"` // Optimization hint for container startup in Windows - LayerFolderPath string `json:",omitempty"` // Where the layer folders are located. Used by Windows Server Containers only. Format %root%\windowsfilter\containerID - Layers []Layer // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID - Credentials string `json:",omitempty"` // Credentials information - ProcessorCount uint32 `json:",omitempty"` // Number of processors to assign to the container. - ProcessorWeight uint64 `json:",omitempty"` // CPU shares (relative weight to other containers with cpu shares). Range is from 1 to 10000. A value of 0 results in default shares. - ProcessorMaximum int64 `json:",omitempty"` // Specifies the portion of processor cycles that this container can use as a percentage times 100. Range is from 1 to 10000. A value of 0 results in no limit. - StorageIOPSMaximum uint64 `json:",omitempty"` // Maximum Storage IOPS - StorageBandwidthMaximum uint64 `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second - StorageSandboxSize uint64 `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller - MemoryMaximumInMB int64 `json:",omitempty"` // Maximum memory available to the container in Megabytes - HostName string `json:",omitempty"` // Hostname - MappedDirectories []MappedDir `json:",omitempty"` // List of mapped directories (volumes/mounts) - MappedPipes []MappedPipe `json:",omitempty"` // List of mapped Windows named pipes - HvPartition bool // True if it a Hyper-V Container - NetworkSharedContainerName string `json:",omitempty"` // Name (ID) of the container that we will share the network stack with. - EndpointList []string `json:",omitempty"` // List of networking endpoints to be attached to container - HvRuntime *HvRuntime `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM - Servicing bool `json:",omitempty"` // True if this container is for servicing - AllowUnqualifiedDNSQuery bool `json:",omitempty"` // True to allow unqualified DNS name resolution - DNSSearchList string `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution - ContainerType string `json:",omitempty"` // "Linux" for Linux containers on Windows. Omitted otherwise. - TerminateOnLastHandleClosed bool `json:",omitempty"` // Should HCS terminate the container once all handles have been closed - MappedVirtualDisks []MappedVirtualDisk `json:",omitempty"` // Array of virtual disks to mount at start - AssignedDevices []AssignedDevice `json:",omitempty"` // Array of devices to assign. NOTE: Support added in RS5 -} - -type ComputeSystemQuery struct { - IDs []string `json:"Ids,omitempty"` - Types []string `json:",omitempty"` - Names []string `json:",omitempty"` - Owners []string `json:",omitempty"` -} - -type PropertyType string - -const ( - PropertyTypeStatistics PropertyType = "Statistics" // V1 and V2 - PropertyTypeProcessList = "ProcessList" // V1 and V2 - PropertyTypeMappedVirtualDisk = "MappedVirtualDisk" // Not supported in V2 schema call - PropertyTypeGuestConnection = "GuestConnection" // V1 and V2. Nil return from HCS before RS5 -) - -type PropertyQuery struct { - PropertyTypes []PropertyType `json:",omitempty"` -} - -// ContainerProperties holds the properties for a container and the processes running in that container -type ContainerProperties struct { - ID string `json:"Id"` - State string - Name string - SystemType string - Owner string - SiloGUID string `json:"SiloGuid,omitempty"` - RuntimeID string `json:"RuntimeId,omitempty"` - IsRuntimeTemplate bool `json:",omitempty"` - RuntimeImagePath string `json:",omitempty"` - Stopped bool `json:",omitempty"` - ExitType string `json:",omitempty"` - AreUpdatesPending bool `json:",omitempty"` - ObRoot string `json:",omitempty"` - Statistics Statistics `json:",omitempty"` - ProcessList []ProcessListItem `json:",omitempty"` - MappedVirtualDiskControllers map[int]MappedVirtualDiskController `json:",omitempty"` - GuestConnectionInfo GuestConnectionInfo `json:",omitempty"` -} - -// MemoryStats holds the memory statistics for a container -type MemoryStats struct { - UsageCommitBytes uint64 `json:"MemoryUsageCommitBytes,omitempty"` - UsageCommitPeakBytes uint64 `json:"MemoryUsageCommitPeakBytes,omitempty"` - UsagePrivateWorkingSetBytes uint64 `json:"MemoryUsagePrivateWorkingSetBytes,omitempty"` -} - -// ProcessorStats holds the processor statistics for a container -type ProcessorStats struct { - TotalRuntime100ns uint64 `json:",omitempty"` - RuntimeUser100ns uint64 `json:",omitempty"` - RuntimeKernel100ns uint64 `json:",omitempty"` -} - -// StorageStats holds the storage statistics for a container -type StorageStats struct { - ReadCountNormalized uint64 `json:",omitempty"` - ReadSizeBytes uint64 `json:",omitempty"` - WriteCountNormalized uint64 `json:",omitempty"` - WriteSizeBytes uint64 `json:",omitempty"` -} - -// NetworkStats holds the network statistics for a container -type NetworkStats struct { - BytesReceived uint64 `json:",omitempty"` - BytesSent uint64 `json:",omitempty"` - PacketsReceived uint64 `json:",omitempty"` - PacketsSent uint64 `json:",omitempty"` - DroppedPacketsIncoming uint64 `json:",omitempty"` - DroppedPacketsOutgoing uint64 `json:",omitempty"` - EndpointId string `json:",omitempty"` - InstanceId string `json:",omitempty"` -} - -// Statistics is the structure returned by a statistics call on a container -type Statistics struct { - Timestamp time.Time `json:",omitempty"` - ContainerStartTime time.Time `json:",omitempty"` - Uptime100ns uint64 `json:",omitempty"` - Memory MemoryStats `json:",omitempty"` - Processor ProcessorStats `json:",omitempty"` - Storage StorageStats `json:",omitempty"` - Network []NetworkStats `json:",omitempty"` -} - -// ProcessList is the structure of an item returned by a ProcessList call on a container -type ProcessListItem struct { - CreateTimestamp time.Time `json:",omitempty"` - ImageName string `json:",omitempty"` - KernelTime100ns uint64 `json:",omitempty"` - MemoryCommitBytes uint64 `json:",omitempty"` - MemoryWorkingSetPrivateBytes uint64 `json:",omitempty"` - MemoryWorkingSetSharedBytes uint64 `json:",omitempty"` - ProcessId uint32 `json:",omitempty"` - UserTime100ns uint64 `json:",omitempty"` -} - -// MappedVirtualDiskController is the structure of an item returned by a MappedVirtualDiskList call on a container -type MappedVirtualDiskController struct { - MappedVirtualDisks map[int]MappedVirtualDisk `json:",omitempty"` -} - -// GuestDefinedCapabilities is part of the GuestConnectionInfo returned by a GuestConnection call on a utility VM -type GuestDefinedCapabilities struct { - NamespaceAddRequestSupported bool `json:",omitempty"` - SignalProcessSupported bool `json:",omitempty"` -} - -// GuestConnectionInfo is the structure of an iterm return by a GuestConnection call on a utility VM -type GuestConnectionInfo struct { - SupportedSchemaVersions []hcsschema.Version `json:",omitempty"` - ProtocolVersion uint32 `json:",omitempty"` - GuestDefinedCapabilities GuestDefinedCapabilities `json:",omitempty"` -} - -// Type of Request Support in ModifySystem -type RequestType string - -// Type of Resource Support in ModifySystem -type ResourceType string - -// RequestType const -const ( - Add RequestType = "Add" - Remove RequestType = "Remove" - Network ResourceType = "Network" -) - -// ResourceModificationRequestResponse is the structure used to send request to the container to modify the system -// Supported resource types are Network and Request Types are Add/Remove -type ResourceModificationRequestResponse struct { - Resource ResourceType `json:"ResourceType"` - Data interface{} `json:"Settings"` - Request RequestType `json:"RequestType,omitempty"` -} +package schema1 + +import ( + "encoding/json" + "time" + + "github.com/Microsoft/hcsshim/internal/schema2" +) + +// ProcessConfig is used as both the input of Container.CreateProcess +// and to convert the parameters to JSON for passing onto the HCS +type ProcessConfig struct { + ApplicationName string `json:",omitempty"` + CommandLine string `json:",omitempty"` + CommandArgs []string `json:",omitempty"` // Used by Linux Containers on Windows + User string `json:",omitempty"` + WorkingDirectory string `json:",omitempty"` + Environment map[string]string `json:",omitempty"` + EmulateConsole bool `json:",omitempty"` + CreateStdInPipe bool `json:",omitempty"` + CreateStdOutPipe bool `json:",omitempty"` + CreateStdErrPipe bool `json:",omitempty"` + ConsoleSize [2]uint `json:",omitempty"` + CreateInUtilityVm bool `json:",omitempty"` // Used by Linux Containers on Windows + OCISpecification *json.RawMessage `json:",omitempty"` // Used by Linux Containers on Windows +} + +type Layer struct { + ID string + Path string +} + +type MappedDir struct { + HostPath string + ContainerPath string + ReadOnly bool + BandwidthMaximum uint64 + IOPSMaximum uint64 + CreateInUtilityVM bool + // LinuxMetadata - Support added in 1803/RS4+. + LinuxMetadata bool `json:",omitempty"` +} + +type MappedPipe struct { + HostPath string + ContainerPipeName string +} + +type HvRuntime struct { + ImagePath string `json:",omitempty"` + SkipTemplate bool `json:",omitempty"` + LinuxInitrdFile string `json:",omitempty"` // File under ImagePath on host containing an initrd image for starting a Linux utility VM + LinuxKernelFile string `json:",omitempty"` // File under ImagePath on host containing a kernel for starting a Linux utility VM + LinuxBootParameters string `json:",omitempty"` // Additional boot parameters for starting a Linux Utility VM in initrd mode + BootSource string `json:",omitempty"` // "Vhd" for Linux Utility VM booting from VHD + WritableBootSource bool `json:",omitempty"` // Linux Utility VM booting from VHD +} + +type MappedVirtualDisk struct { + HostPath string `json:",omitempty"` // Path to VHD on the host + ContainerPath string // Platform-specific mount point path in the container + CreateInUtilityVM bool `json:",omitempty"` + ReadOnly bool `json:",omitempty"` + Cache string `json:",omitempty"` // "" (Unspecified); "Disabled"; "Enabled"; "Private"; "PrivateAllowSharing" + AttachOnly bool `json:",omitempty:` +} + +// AssignedDevice represents a device that has been directly assigned to a container +// +// NOTE: Support added in RS5 +type AssignedDevice struct { + // InterfaceClassGUID of the device to assign to container. + InterfaceClassGUID string `json:"InterfaceClassGuid,omitempty"` +} + +// ContainerConfig is used as both the input of CreateContainer +// and to convert the parameters to JSON for passing onto the HCS +type ContainerConfig struct { + SystemType string // HCS requires this to be hard-coded to "Container" + Name string // Name of the container. We use the docker ID. + Owner string `json:",omitempty"` // The management platform that created this container + VolumePath string `json:",omitempty"` // Windows volume path for scratch space. Used by Windows Server Containers only. Format \\?\\Volume{GUID} + IgnoreFlushesDuringBoot bool `json:",omitempty"` // Optimization hint for container startup in Windows + LayerFolderPath string `json:",omitempty"` // Where the layer folders are located. Used by Windows Server Containers only. Format %root%\windowsfilter\containerID + Layers []Layer // List of storage layers. Required for Windows Server and Hyper-V Containers. Format ID=GUID;Path=%root%\windowsfilter\layerID + Credentials string `json:",omitempty"` // Credentials information + ProcessorCount uint32 `json:",omitempty"` // Number of processors to assign to the container. + ProcessorWeight uint64 `json:",omitempty"` // CPU shares (relative weight to other containers with cpu shares). Range is from 1 to 10000. A value of 0 results in default shares. + ProcessorMaximum int64 `json:",omitempty"` // Specifies the portion of processor cycles that this container can use as a percentage times 100. Range is from 1 to 10000. A value of 0 results in no limit. + StorageIOPSMaximum uint64 `json:",omitempty"` // Maximum Storage IOPS + StorageBandwidthMaximum uint64 `json:",omitempty"` // Maximum Storage Bandwidth in bytes per second + StorageSandboxSize uint64 `json:",omitempty"` // Size in bytes that the container system drive should be expanded to if smaller + MemoryMaximumInMB int64 `json:",omitempty"` // Maximum memory available to the container in Megabytes + HostName string `json:",omitempty"` // Hostname + MappedDirectories []MappedDir `json:",omitempty"` // List of mapped directories (volumes/mounts) + MappedPipes []MappedPipe `json:",omitempty"` // List of mapped Windows named pipes + HvPartition bool // True if it a Hyper-V Container + NetworkSharedContainerName string `json:",omitempty"` // Name (ID) of the container that we will share the network stack with. + EndpointList []string `json:",omitempty"` // List of networking endpoints to be attached to container + HvRuntime *HvRuntime `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM + Servicing bool `json:",omitempty"` // True if this container is for servicing + AllowUnqualifiedDNSQuery bool `json:",omitempty"` // True to allow unqualified DNS name resolution + DNSSearchList string `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution + ContainerType string `json:",omitempty"` // "Linux" for Linux containers on Windows. Omitted otherwise. + TerminateOnLastHandleClosed bool `json:",omitempty"` // Should HCS terminate the container once all handles have been closed + MappedVirtualDisks []MappedVirtualDisk `json:",omitempty"` // Array of virtual disks to mount at start + AssignedDevices []AssignedDevice `json:",omitempty"` // Array of devices to assign. NOTE: Support added in RS5 +} + +type ComputeSystemQuery struct { + IDs []string `json:"Ids,omitempty"` + Types []string `json:",omitempty"` + Names []string `json:",omitempty"` + Owners []string `json:",omitempty"` +} + +type PropertyType string + +const ( + PropertyTypeStatistics PropertyType = "Statistics" // V1 and V2 + PropertyTypeProcessList = "ProcessList" // V1 and V2 + PropertyTypeMappedVirtualDisk = "MappedVirtualDisk" // Not supported in V2 schema call + PropertyTypeGuestConnection = "GuestConnection" // V1 and V2. Nil return from HCS before RS5 +) + +type PropertyQuery struct { + PropertyTypes []PropertyType `json:",omitempty"` +} + +// ContainerProperties holds the properties for a container and the processes running in that container +type ContainerProperties struct { + ID string `json:"Id"` + State string + Name string + SystemType string + Owner string + SiloGUID string `json:"SiloGuid,omitempty"` + RuntimeID string `json:"RuntimeId,omitempty"` + IsRuntimeTemplate bool `json:",omitempty"` + RuntimeImagePath string `json:",omitempty"` + Stopped bool `json:",omitempty"` + ExitType string `json:",omitempty"` + AreUpdatesPending bool `json:",omitempty"` + ObRoot string `json:",omitempty"` + Statistics Statistics `json:",omitempty"` + ProcessList []ProcessListItem `json:",omitempty"` + MappedVirtualDiskControllers map[int]MappedVirtualDiskController `json:",omitempty"` + GuestConnectionInfo GuestConnectionInfo `json:",omitempty"` +} + +// MemoryStats holds the memory statistics for a container +type MemoryStats struct { + UsageCommitBytes uint64 `json:"MemoryUsageCommitBytes,omitempty"` + UsageCommitPeakBytes uint64 `json:"MemoryUsageCommitPeakBytes,omitempty"` + UsagePrivateWorkingSetBytes uint64 `json:"MemoryUsagePrivateWorkingSetBytes,omitempty"` +} + +// ProcessorStats holds the processor statistics for a container +type ProcessorStats struct { + TotalRuntime100ns uint64 `json:",omitempty"` + RuntimeUser100ns uint64 `json:",omitempty"` + RuntimeKernel100ns uint64 `json:",omitempty"` +} + +// StorageStats holds the storage statistics for a container +type StorageStats struct { + ReadCountNormalized uint64 `json:",omitempty"` + ReadSizeBytes uint64 `json:",omitempty"` + WriteCountNormalized uint64 `json:",omitempty"` + WriteSizeBytes uint64 `json:",omitempty"` +} + +// NetworkStats holds the network statistics for a container +type NetworkStats struct { + BytesReceived uint64 `json:",omitempty"` + BytesSent uint64 `json:",omitempty"` + PacketsReceived uint64 `json:",omitempty"` + PacketsSent uint64 `json:",omitempty"` + DroppedPacketsIncoming uint64 `json:",omitempty"` + DroppedPacketsOutgoing uint64 `json:",omitempty"` + EndpointId string `json:",omitempty"` + InstanceId string `json:",omitempty"` +} + +// Statistics is the structure returned by a statistics call on a container +type Statistics struct { + Timestamp time.Time `json:",omitempty"` + ContainerStartTime time.Time `json:",omitempty"` + Uptime100ns uint64 `json:",omitempty"` + Memory MemoryStats `json:",omitempty"` + Processor ProcessorStats `json:",omitempty"` + Storage StorageStats `json:",omitempty"` + Network []NetworkStats `json:",omitempty"` +} + +// ProcessList is the structure of an item returned by a ProcessList call on a container +type ProcessListItem struct { + CreateTimestamp time.Time `json:",omitempty"` + ImageName string `json:",omitempty"` + KernelTime100ns uint64 `json:",omitempty"` + MemoryCommitBytes uint64 `json:",omitempty"` + MemoryWorkingSetPrivateBytes uint64 `json:",omitempty"` + MemoryWorkingSetSharedBytes uint64 `json:",omitempty"` + ProcessId uint32 `json:",omitempty"` + UserTime100ns uint64 `json:",omitempty"` +} + +// MappedVirtualDiskController is the structure of an item returned by a MappedVirtualDiskList call on a container +type MappedVirtualDiskController struct { + MappedVirtualDisks map[int]MappedVirtualDisk `json:",omitempty"` +} + +// GuestDefinedCapabilities is part of the GuestConnectionInfo returned by a GuestConnection call on a utility VM +type GuestDefinedCapabilities struct { + NamespaceAddRequestSupported bool `json:",omitempty"` + SignalProcessSupported bool `json:",omitempty"` +} + +// GuestConnectionInfo is the structure of an iterm return by a GuestConnection call on a utility VM +type GuestConnectionInfo struct { + SupportedSchemaVersions []hcsschema.Version `json:",omitempty"` + ProtocolVersion uint32 `json:",omitempty"` + GuestDefinedCapabilities GuestDefinedCapabilities `json:",omitempty"` +} + +// Type of Request Support in ModifySystem +type RequestType string + +// Type of Resource Support in ModifySystem +type ResourceType string + +// RequestType const +const ( + Add RequestType = "Add" + Remove RequestType = "Remove" + Network ResourceType = "Network" +) + +// ResourceModificationRequestResponse is the structure used to send request to the container to modify the system +// Supported resource types are Network and Request Types are Add/Remove +type ResourceModificationRequestResponse struct { + Resource ResourceType `json:"ResourceType"` + Data interface{} `json:"Settings"` + Request RequestType `json:"RequestType,omitempty"` +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerid.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerid.go index 7a6e2efa58..90df3bedce 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerid.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerid.go @@ -1,13 +1,13 @@ -package wclayer - -import ( - "path/filepath" - - "github.com/Microsoft/hcsshim/internal/guid" -) - -// LayerID returns the layer ID of a layer on disk. -func LayerID(path string) (guid.GUID, error) { - _, file := filepath.Split(path) - return NameToGuid(file) -} +package wclayer + +import ( + "path/filepath" + + "github.com/Microsoft/hcsshim/internal/guid" +) + +// LayerID returns the layer ID of a layer on disk. +func LayerID(path string) (guid.GUID, error) { + _, file := filepath.Split(path) + return NameToGuid(file) +} diff --git a/vendor/golang.org/x/sys/windows/registry/key.go b/vendor/golang.org/x/sys/windows/registry/key.go new file mode 100644 index 0000000000..c256483434 --- /dev/null +++ b/vendor/golang.org/x/sys/windows/registry/key.go @@ -0,0 +1,198 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +// Package registry provides access to the Windows registry. +// +// Here is a simple example, opening a registry key and reading a string value from it. +// +// k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE) +// if err != nil { +// log.Fatal(err) +// } +// defer k.Close() +// +// s, _, err := k.GetStringValue("SystemRoot") +// if err != nil { +// log.Fatal(err) +// } +// fmt.Printf("Windows system root is %q\n", s) +// +package registry + +import ( + "io" + "syscall" + "time" +) + +const ( + // Registry key security and access rights. + // See https://msdn.microsoft.com/en-us/library/windows/desktop/ms724878.aspx + // for details. + ALL_ACCESS = 0xf003f + CREATE_LINK = 0x00020 + CREATE_SUB_KEY = 0x00004 + ENUMERATE_SUB_KEYS = 0x00008 + EXECUTE = 0x20019 + NOTIFY = 0x00010 + QUERY_VALUE = 0x00001 + READ = 0x20019 + SET_VALUE = 0x00002 + WOW64_32KEY = 0x00200 + WOW64_64KEY = 0x00100 + WRITE = 0x20006 +) + +// Key is a handle to an open Windows registry key. +// Keys can be obtained by calling OpenKey; there are +// also some predefined root keys such as CURRENT_USER. +// Keys can be used directly in the Windows API. +type Key syscall.Handle + +const ( + // Windows defines some predefined root keys that are always open. + // An application can use these keys as entry points to the registry. + // Normally these keys are used in OpenKey to open new keys, + // but they can also be used anywhere a Key is required. + CLASSES_ROOT = Key(syscall.HKEY_CLASSES_ROOT) + CURRENT_USER = Key(syscall.HKEY_CURRENT_USER) + LOCAL_MACHINE = Key(syscall.HKEY_LOCAL_MACHINE) + USERS = Key(syscall.HKEY_USERS) + CURRENT_CONFIG = Key(syscall.HKEY_CURRENT_CONFIG) + PERFORMANCE_DATA = Key(syscall.HKEY_PERFORMANCE_DATA) +) + +// Close closes open key k. +func (k Key) Close() error { + return syscall.RegCloseKey(syscall.Handle(k)) +} + +// OpenKey opens a new key with path name relative to key k. +// It accepts any open key, including CURRENT_USER and others, +// and returns the new key and an error. +// The access parameter specifies desired access rights to the +// key to be opened. +func OpenKey(k Key, path string, access uint32) (Key, error) { + p, err := syscall.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + var subkey syscall.Handle + err = syscall.RegOpenKeyEx(syscall.Handle(k), p, 0, access, &subkey) + if err != nil { + return 0, err + } + return Key(subkey), nil +} + +// OpenRemoteKey opens a predefined registry key on another +// computer pcname. The key to be opened is specified by k, but +// can only be one of LOCAL_MACHINE, PERFORMANCE_DATA or USERS. +// If pcname is "", OpenRemoteKey returns local computer key. +func OpenRemoteKey(pcname string, k Key) (Key, error) { + var err error + var p *uint16 + if pcname != "" { + p, err = syscall.UTF16PtrFromString(`\\` + pcname) + if err != nil { + return 0, err + } + } + var remoteKey syscall.Handle + err = regConnectRegistry(p, syscall.Handle(k), &remoteKey) + if err != nil { + return 0, err + } + return Key(remoteKey), nil +} + +// ReadSubKeyNames returns the names of subkeys of key k. +// The parameter n controls the number of returned names, +// analogous to the way os.File.Readdirnames works. +func (k Key) ReadSubKeyNames(n int) ([]string, error) { + names := make([]string, 0) + // Registry key size limit is 255 bytes and described there: + // https://msdn.microsoft.com/library/windows/desktop/ms724872.aspx + buf := make([]uint16, 256) //plus extra room for terminating zero byte +loopItems: + for i := uint32(0); ; i++ { + if n > 0 { + if len(names) == n { + return names, nil + } + } + l := uint32(len(buf)) + for { + err := syscall.RegEnumKeyEx(syscall.Handle(k), i, &buf[0], &l, nil, nil, nil, nil) + if err == nil { + break + } + if err == syscall.ERROR_MORE_DATA { + // Double buffer size and try again. + l = uint32(2 * len(buf)) + buf = make([]uint16, l) + continue + } + if err == _ERROR_NO_MORE_ITEMS { + break loopItems + } + return names, err + } + names = append(names, syscall.UTF16ToString(buf[:l])) + } + if n > len(names) { + return names, io.EOF + } + return names, nil +} + +// CreateKey creates a key named path under open key k. +// CreateKey returns the new key and a boolean flag that reports +// whether the key already existed. +// The access parameter specifies the access rights for the key +// to be created. +func CreateKey(k Key, path string, access uint32) (newk Key, openedExisting bool, err error) { + var h syscall.Handle + var d uint32 + err = regCreateKeyEx(syscall.Handle(k), syscall.StringToUTF16Ptr(path), + 0, nil, _REG_OPTION_NON_VOLATILE, access, nil, &h, &d) + if err != nil { + return 0, false, err + } + return Key(h), d == _REG_OPENED_EXISTING_KEY, nil +} + +// DeleteKey deletes the subkey path of key k and its values. +func DeleteKey(k Key, path string) error { + return regDeleteKey(syscall.Handle(k), syscall.StringToUTF16Ptr(path)) +} + +// A KeyInfo describes the statistics of a key. It is returned by Stat. +type KeyInfo struct { + SubKeyCount uint32 + MaxSubKeyLen uint32 // size of the key's subkey with the longest name, in Unicode characters, not including the terminating zero byte + ValueCount uint32 + MaxValueNameLen uint32 // size of the key's longest value name, in Unicode characters, not including the terminating zero byte + MaxValueLen uint32 // longest data component among the key's values, in bytes + lastWriteTime syscall.Filetime +} + +// ModTime returns the key's last write time. +func (ki *KeyInfo) ModTime() time.Time { + return time.Unix(0, ki.lastWriteTime.Nanoseconds()) +} + +// Stat retrieves information about the open key k. +func (k Key) Stat() (*KeyInfo, error) { + var ki KeyInfo + err := syscall.RegQueryInfoKey(syscall.Handle(k), nil, nil, nil, + &ki.SubKeyCount, &ki.MaxSubKeyLen, nil, &ki.ValueCount, + &ki.MaxValueNameLen, &ki.MaxValueLen, nil, &ki.lastWriteTime) + if err != nil { + return nil, err + } + return &ki, nil +} diff --git a/vendor/golang.org/x/sys/windows/registry/mksyscall.go b/vendor/golang.org/x/sys/windows/registry/mksyscall.go new file mode 100644 index 0000000000..0ac95ffe73 --- /dev/null +++ b/vendor/golang.org/x/sys/windows/registry/mksyscall.go @@ -0,0 +1,7 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package registry + +//go:generate go run $GOROOT/src/syscall/mksyscall_windows.go -output zsyscall_windows.go syscall.go diff --git a/vendor/golang.org/x/sys/windows/registry/syscall.go b/vendor/golang.org/x/sys/windows/registry/syscall.go new file mode 100644 index 0000000000..e66643cbaa --- /dev/null +++ b/vendor/golang.org/x/sys/windows/registry/syscall.go @@ -0,0 +1,32 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package registry + +import "syscall" + +const ( + _REG_OPTION_NON_VOLATILE = 0 + + _REG_CREATED_NEW_KEY = 1 + _REG_OPENED_EXISTING_KEY = 2 + + _ERROR_NO_MORE_ITEMS syscall.Errno = 259 +) + +func LoadRegLoadMUIString() error { + return procRegLoadMUIStringW.Find() +} + +//sys regCreateKeyEx(key syscall.Handle, subkey *uint16, reserved uint32, class *uint16, options uint32, desired uint32, sa *syscall.SecurityAttributes, result *syscall.Handle, disposition *uint32) (regerrno error) = advapi32.RegCreateKeyExW +//sys regDeleteKey(key syscall.Handle, subkey *uint16) (regerrno error) = advapi32.RegDeleteKeyW +//sys regSetValueEx(key syscall.Handle, valueName *uint16, reserved uint32, vtype uint32, buf *byte, bufsize uint32) (regerrno error) = advapi32.RegSetValueExW +//sys regEnumValue(key syscall.Handle, index uint32, name *uint16, nameLen *uint32, reserved *uint32, valtype *uint32, buf *byte, buflen *uint32) (regerrno error) = advapi32.RegEnumValueW +//sys regDeleteValue(key syscall.Handle, name *uint16) (regerrno error) = advapi32.RegDeleteValueW +//sys regLoadMUIString(key syscall.Handle, name *uint16, buf *uint16, buflen uint32, buflenCopied *uint32, flags uint32, dir *uint16) (regerrno error) = advapi32.RegLoadMUIStringW +//sys regConnectRegistry(machinename *uint16, key syscall.Handle, result *syscall.Handle) (regerrno error) = advapi32.RegConnectRegistryW + +//sys expandEnvironmentStrings(src *uint16, dst *uint16, size uint32) (n uint32, err error) = kernel32.ExpandEnvironmentStringsW diff --git a/vendor/golang.org/x/sys/windows/registry/value.go b/vendor/golang.org/x/sys/windows/registry/value.go new file mode 100644 index 0000000000..71d4e15bab --- /dev/null +++ b/vendor/golang.org/x/sys/windows/registry/value.go @@ -0,0 +1,384 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package registry + +import ( + "errors" + "io" + "syscall" + "unicode/utf16" + "unsafe" +) + +const ( + // Registry value types. + NONE = 0 + SZ = 1 + EXPAND_SZ = 2 + BINARY = 3 + DWORD = 4 + DWORD_BIG_ENDIAN = 5 + LINK = 6 + MULTI_SZ = 7 + RESOURCE_LIST = 8 + FULL_RESOURCE_DESCRIPTOR = 9 + RESOURCE_REQUIREMENTS_LIST = 10 + QWORD = 11 +) + +var ( + // ErrShortBuffer is returned when the buffer was too short for the operation. + ErrShortBuffer = syscall.ERROR_MORE_DATA + + // ErrNotExist is returned when a registry key or value does not exist. + ErrNotExist = syscall.ERROR_FILE_NOT_FOUND + + // ErrUnexpectedType is returned by Get*Value when the value's type was unexpected. + ErrUnexpectedType = errors.New("unexpected key value type") +) + +// GetValue retrieves the type and data for the specified value associated +// with an open key k. It fills up buffer buf and returns the retrieved +// byte count n. If buf is too small to fit the stored value it returns +// ErrShortBuffer error along with the required buffer size n. +// If no buffer is provided, it returns true and actual buffer size n. +// If no buffer is provided, GetValue returns the value's type only. +// If the value does not exist, the error returned is ErrNotExist. +// +// GetValue is a low level function. If value's type is known, use the appropriate +// Get*Value function instead. +func (k Key) GetValue(name string, buf []byte) (n int, valtype uint32, err error) { + pname, err := syscall.UTF16PtrFromString(name) + if err != nil { + return 0, 0, err + } + var pbuf *byte + if len(buf) > 0 { + pbuf = (*byte)(unsafe.Pointer(&buf[0])) + } + l := uint32(len(buf)) + err = syscall.RegQueryValueEx(syscall.Handle(k), pname, nil, &valtype, pbuf, &l) + if err != nil { + return int(l), valtype, err + } + return int(l), valtype, nil +} + +func (k Key) getValue(name string, buf []byte) (date []byte, valtype uint32, err error) { + p, err := syscall.UTF16PtrFromString(name) + if err != nil { + return nil, 0, err + } + var t uint32 + n := uint32(len(buf)) + for { + err = syscall.RegQueryValueEx(syscall.Handle(k), p, nil, &t, (*byte)(unsafe.Pointer(&buf[0])), &n) + if err == nil { + return buf[:n], t, nil + } + if err != syscall.ERROR_MORE_DATA { + return nil, 0, err + } + if n <= uint32(len(buf)) { + return nil, 0, err + } + buf = make([]byte, n) + } +} + +// GetStringValue retrieves the string value for the specified +// value name associated with an open key k. It also returns the value's type. +// If value does not exist, GetStringValue returns ErrNotExist. +// If value is not SZ or EXPAND_SZ, it will return the correct value +// type and ErrUnexpectedType. +func (k Key) GetStringValue(name string) (val string, valtype uint32, err error) { + data, typ, err2 := k.getValue(name, make([]byte, 64)) + if err2 != nil { + return "", typ, err2 + } + switch typ { + case SZ, EXPAND_SZ: + default: + return "", typ, ErrUnexpectedType + } + if len(data) == 0 { + return "", typ, nil + } + u := (*[1 << 29]uint16)(unsafe.Pointer(&data[0]))[:] + return syscall.UTF16ToString(u), typ, nil +} + +// GetMUIStringValue retrieves the localized string value for +// the specified value name associated with an open key k. +// If the value name doesn't exist or the localized string value +// can't be resolved, GetMUIStringValue returns ErrNotExist. +// GetMUIStringValue panics if the system doesn't support +// regLoadMUIString; use LoadRegLoadMUIString to check if +// regLoadMUIString is supported before calling this function. +func (k Key) GetMUIStringValue(name string) (string, error) { + pname, err := syscall.UTF16PtrFromString(name) + if err != nil { + return "", err + } + + buf := make([]uint16, 1024) + var buflen uint32 + var pdir *uint16 + + err = regLoadMUIString(syscall.Handle(k), pname, &buf[0], uint32(len(buf)), &buflen, 0, pdir) + if err == syscall.ERROR_FILE_NOT_FOUND { // Try fallback path + + // Try to resolve the string value using the system directory as + // a DLL search path; this assumes the string value is of the form + // @[path]\dllname,-strID but with no path given, e.g. @tzres.dll,-320. + + // This approach works with tzres.dll but may have to be revised + // in the future to allow callers to provide custom search paths. + + var s string + s, err = ExpandString("%SystemRoot%\\system32\\") + if err != nil { + return "", err + } + pdir, err = syscall.UTF16PtrFromString(s) + if err != nil { + return "", err + } + + err = regLoadMUIString(syscall.Handle(k), pname, &buf[0], uint32(len(buf)), &buflen, 0, pdir) + } + + for err == syscall.ERROR_MORE_DATA { // Grow buffer if needed + if buflen <= uint32(len(buf)) { + break // Buffer not growing, assume race; break + } + buf = make([]uint16, buflen) + err = regLoadMUIString(syscall.Handle(k), pname, &buf[0], uint32(len(buf)), &buflen, 0, pdir) + } + + if err != nil { + return "", err + } + + return syscall.UTF16ToString(buf), nil +} + +// ExpandString expands environment-variable strings and replaces +// them with the values defined for the current user. +// Use ExpandString to expand EXPAND_SZ strings. +func ExpandString(value string) (string, error) { + if value == "" { + return "", nil + } + p, err := syscall.UTF16PtrFromString(value) + if err != nil { + return "", err + } + r := make([]uint16, 100) + for { + n, err := expandEnvironmentStrings(p, &r[0], uint32(len(r))) + if err != nil { + return "", err + } + if n <= uint32(len(r)) { + u := (*[1 << 29]uint16)(unsafe.Pointer(&r[0]))[:] + return syscall.UTF16ToString(u), nil + } + r = make([]uint16, n) + } +} + +// GetStringsValue retrieves the []string value for the specified +// value name associated with an open key k. It also returns the value's type. +// If value does not exist, GetStringsValue returns ErrNotExist. +// If value is not MULTI_SZ, it will return the correct value +// type and ErrUnexpectedType. +func (k Key) GetStringsValue(name string) (val []string, valtype uint32, err error) { + data, typ, err2 := k.getValue(name, make([]byte, 64)) + if err2 != nil { + return nil, typ, err2 + } + if typ != MULTI_SZ { + return nil, typ, ErrUnexpectedType + } + if len(data) == 0 { + return nil, typ, nil + } + p := (*[1 << 29]uint16)(unsafe.Pointer(&data[0]))[:len(data)/2] + if len(p) == 0 { + return nil, typ, nil + } + if p[len(p)-1] == 0 { + p = p[:len(p)-1] // remove terminating null + } + val = make([]string, 0, 5) + from := 0 + for i, c := range p { + if c == 0 { + val = append(val, string(utf16.Decode(p[from:i]))) + from = i + 1 + } + } + return val, typ, nil +} + +// GetIntegerValue retrieves the integer value for the specified +// value name associated with an open key k. It also returns the value's type. +// If value does not exist, GetIntegerValue returns ErrNotExist. +// If value is not DWORD or QWORD, it will return the correct value +// type and ErrUnexpectedType. +func (k Key) GetIntegerValue(name string) (val uint64, valtype uint32, err error) { + data, typ, err2 := k.getValue(name, make([]byte, 8)) + if err2 != nil { + return 0, typ, err2 + } + switch typ { + case DWORD: + if len(data) != 4 { + return 0, typ, errors.New("DWORD value is not 4 bytes long") + } + return uint64(*(*uint32)(unsafe.Pointer(&data[0]))), DWORD, nil + case QWORD: + if len(data) != 8 { + return 0, typ, errors.New("QWORD value is not 8 bytes long") + } + return uint64(*(*uint64)(unsafe.Pointer(&data[0]))), QWORD, nil + default: + return 0, typ, ErrUnexpectedType + } +} + +// GetBinaryValue retrieves the binary value for the specified +// value name associated with an open key k. It also returns the value's type. +// If value does not exist, GetBinaryValue returns ErrNotExist. +// If value is not BINARY, it will return the correct value +// type and ErrUnexpectedType. +func (k Key) GetBinaryValue(name string) (val []byte, valtype uint32, err error) { + data, typ, err2 := k.getValue(name, make([]byte, 64)) + if err2 != nil { + return nil, typ, err2 + } + if typ != BINARY { + return nil, typ, ErrUnexpectedType + } + return data, typ, nil +} + +func (k Key) setValue(name string, valtype uint32, data []byte) error { + p, err := syscall.UTF16PtrFromString(name) + if err != nil { + return err + } + if len(data) == 0 { + return regSetValueEx(syscall.Handle(k), p, 0, valtype, nil, 0) + } + return regSetValueEx(syscall.Handle(k), p, 0, valtype, &data[0], uint32(len(data))) +} + +// SetDWordValue sets the data and type of a name value +// under key k to value and DWORD. +func (k Key) SetDWordValue(name string, value uint32) error { + return k.setValue(name, DWORD, (*[4]byte)(unsafe.Pointer(&value))[:]) +} + +// SetQWordValue sets the data and type of a name value +// under key k to value and QWORD. +func (k Key) SetQWordValue(name string, value uint64) error { + return k.setValue(name, QWORD, (*[8]byte)(unsafe.Pointer(&value))[:]) +} + +func (k Key) setStringValue(name string, valtype uint32, value string) error { + v, err := syscall.UTF16FromString(value) + if err != nil { + return err + } + buf := (*[1 << 29]byte)(unsafe.Pointer(&v[0]))[:len(v)*2] + return k.setValue(name, valtype, buf) +} + +// SetStringValue sets the data and type of a name value +// under key k to value and SZ. The value must not contain a zero byte. +func (k Key) SetStringValue(name, value string) error { + return k.setStringValue(name, SZ, value) +} + +// SetExpandStringValue sets the data and type of a name value +// under key k to value and EXPAND_SZ. The value must not contain a zero byte. +func (k Key) SetExpandStringValue(name, value string) error { + return k.setStringValue(name, EXPAND_SZ, value) +} + +// SetStringsValue sets the data and type of a name value +// under key k to value and MULTI_SZ. The value strings +// must not contain a zero byte. +func (k Key) SetStringsValue(name string, value []string) error { + ss := "" + for _, s := range value { + for i := 0; i < len(s); i++ { + if s[i] == 0 { + return errors.New("string cannot have 0 inside") + } + } + ss += s + "\x00" + } + v := utf16.Encode([]rune(ss + "\x00")) + buf := (*[1 << 29]byte)(unsafe.Pointer(&v[0]))[:len(v)*2] + return k.setValue(name, MULTI_SZ, buf) +} + +// SetBinaryValue sets the data and type of a name value +// under key k to value and BINARY. +func (k Key) SetBinaryValue(name string, value []byte) error { + return k.setValue(name, BINARY, value) +} + +// DeleteValue removes a named value from the key k. +func (k Key) DeleteValue(name string) error { + return regDeleteValue(syscall.Handle(k), syscall.StringToUTF16Ptr(name)) +} + +// ReadValueNames returns the value names of key k. +// The parameter n controls the number of returned names, +// analogous to the way os.File.Readdirnames works. +func (k Key) ReadValueNames(n int) ([]string, error) { + ki, err := k.Stat() + if err != nil { + return nil, err + } + names := make([]string, 0, ki.ValueCount) + buf := make([]uint16, ki.MaxValueNameLen+1) // extra room for terminating null character +loopItems: + for i := uint32(0); ; i++ { + if n > 0 { + if len(names) == n { + return names, nil + } + } + l := uint32(len(buf)) + for { + err := regEnumValue(syscall.Handle(k), i, &buf[0], &l, nil, nil, nil, nil) + if err == nil { + break + } + if err == syscall.ERROR_MORE_DATA { + // Double buffer size and try again. + l = uint32(2 * len(buf)) + buf = make([]uint16, l) + continue + } + if err == _ERROR_NO_MORE_ITEMS { + break loopItems + } + return names, err + } + names = append(names, syscall.UTF16ToString(buf[:l])) + } + if n > len(names) { + return names, io.EOF + } + return names, nil +} diff --git a/vendor/golang.org/x/sys/windows/registry/zsyscall_windows.go b/vendor/golang.org/x/sys/windows/registry/zsyscall_windows.go new file mode 100644 index 0000000000..3778075da0 --- /dev/null +++ b/vendor/golang.org/x/sys/windows/registry/zsyscall_windows.go @@ -0,0 +1,120 @@ +// Code generated by 'go generate'; DO NOT EDIT. + +package registry + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + + procRegCreateKeyExW = modadvapi32.NewProc("RegCreateKeyExW") + procRegDeleteKeyW = modadvapi32.NewProc("RegDeleteKeyW") + procRegSetValueExW = modadvapi32.NewProc("RegSetValueExW") + procRegEnumValueW = modadvapi32.NewProc("RegEnumValueW") + procRegDeleteValueW = modadvapi32.NewProc("RegDeleteValueW") + procRegLoadMUIStringW = modadvapi32.NewProc("RegLoadMUIStringW") + procRegConnectRegistryW = modadvapi32.NewProc("RegConnectRegistryW") + procExpandEnvironmentStringsW = modkernel32.NewProc("ExpandEnvironmentStringsW") +) + +func regCreateKeyEx(key syscall.Handle, subkey *uint16, reserved uint32, class *uint16, options uint32, desired uint32, sa *syscall.SecurityAttributes, result *syscall.Handle, disposition *uint32) (regerrno error) { + r0, _, _ := syscall.Syscall9(procRegCreateKeyExW.Addr(), 9, uintptr(key), uintptr(unsafe.Pointer(subkey)), uintptr(reserved), uintptr(unsafe.Pointer(class)), uintptr(options), uintptr(desired), uintptr(unsafe.Pointer(sa)), uintptr(unsafe.Pointer(result)), uintptr(unsafe.Pointer(disposition))) + if r0 != 0 { + regerrno = syscall.Errno(r0) + } + return +} + +func regDeleteKey(key syscall.Handle, subkey *uint16) (regerrno error) { + r0, _, _ := syscall.Syscall(procRegDeleteKeyW.Addr(), 2, uintptr(key), uintptr(unsafe.Pointer(subkey)), 0) + if r0 != 0 { + regerrno = syscall.Errno(r0) + } + return +} + +func regSetValueEx(key syscall.Handle, valueName *uint16, reserved uint32, vtype uint32, buf *byte, bufsize uint32) (regerrno error) { + r0, _, _ := syscall.Syscall6(procRegSetValueExW.Addr(), 6, uintptr(key), uintptr(unsafe.Pointer(valueName)), uintptr(reserved), uintptr(vtype), uintptr(unsafe.Pointer(buf)), uintptr(bufsize)) + if r0 != 0 { + regerrno = syscall.Errno(r0) + } + return +} + +func regEnumValue(key syscall.Handle, index uint32, name *uint16, nameLen *uint32, reserved *uint32, valtype *uint32, buf *byte, buflen *uint32) (regerrno error) { + r0, _, _ := syscall.Syscall9(procRegEnumValueW.Addr(), 8, uintptr(key), uintptr(index), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(nameLen)), uintptr(unsafe.Pointer(reserved)), uintptr(unsafe.Pointer(valtype)), uintptr(unsafe.Pointer(buf)), uintptr(unsafe.Pointer(buflen)), 0) + if r0 != 0 { + regerrno = syscall.Errno(r0) + } + return +} + +func regDeleteValue(key syscall.Handle, name *uint16) (regerrno error) { + r0, _, _ := syscall.Syscall(procRegDeleteValueW.Addr(), 2, uintptr(key), uintptr(unsafe.Pointer(name)), 0) + if r0 != 0 { + regerrno = syscall.Errno(r0) + } + return +} + +func regLoadMUIString(key syscall.Handle, name *uint16, buf *uint16, buflen uint32, buflenCopied *uint32, flags uint32, dir *uint16) (regerrno error) { + r0, _, _ := syscall.Syscall9(procRegLoadMUIStringW.Addr(), 7, uintptr(key), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buf)), uintptr(buflen), uintptr(unsafe.Pointer(buflenCopied)), uintptr(flags), uintptr(unsafe.Pointer(dir)), 0, 0) + if r0 != 0 { + regerrno = syscall.Errno(r0) + } + return +} + +func regConnectRegistry(machinename *uint16, key syscall.Handle, result *syscall.Handle) (regerrno error) { + r0, _, _ := syscall.Syscall(procRegConnectRegistryW.Addr(), 3, uintptr(unsafe.Pointer(machinename)), uintptr(key), uintptr(unsafe.Pointer(result))) + if r0 != 0 { + regerrno = syscall.Errno(r0) + } + return +} + +func expandEnvironmentStrings(src *uint16, dst *uint16, size uint32) (n uint32, err error) { + r0, _, e1 := syscall.Syscall(procExpandEnvironmentStringsW.Addr(), 3, uintptr(unsafe.Pointer(src)), uintptr(unsafe.Pointer(dst)), uintptr(size)) + n = uint32(r0) + if n == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} From 62b0c8d1b8b3201ccd36d007d61745389b526683 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Wed, 15 May 2019 16:29:37 -0700 Subject: [PATCH 08/37] WIP - VM to container connectivity --- Makefile | 13 +++- network/endpoint_windows.go | 99 +++++++++++++++++++++++++++ network/network_windows.go | 129 ++++++++++++++++++++++++++++++++++++ 3 files changed, 240 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 42a8056279..01a0e7ff4a 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,10 @@ NPMFILES = \ $(wildcard npm/plugin/*.go) \ $(COREFILES) +DUMMYFILES = \ + $(wildcard dummy/*.go) \ + $(COREFILES) + # Build defaults. GOOS ?= linux GOARCH ?= amd64 @@ -71,6 +75,8 @@ CNI_BUILD_DIR = $(BUILD_DIR)/cni CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy CNS_BUILD_DIR = $(BUILD_DIR)/cns NPM_BUILD_DIR = $(BUILD_DIR)/npm +DUMMY_DIR = dummy +DUMMY_BUILD_DIR = $(BUILD_DIR)/dummy # Containerized build parameters. BUILD_CONTAINER_IMAGE = acn-build @@ -117,6 +123,7 @@ azure-vnet-ipam: $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) azure-cni-plugin: azure-vnet azure-vnet-ipam azure-vnet-telemetry cni-archive azure-cns: $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT) cns-archive azure-vnet-telemetry: $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) +dummy: $(DUMMY_BUILD_DIR)/dummy$(EXE_EXT) # Azure-NPM only supports Linux for now. ifeq ($(GOOS),linux) @@ -126,7 +133,7 @@ endif ifeq ($(GOOS),linux) all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns azure-npm else -all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns +all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns dummy endif ifeq ($(GOOS),linux) @@ -165,6 +172,10 @@ $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT): $(CNSFILES) $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT): $(NPMFILES) go build -v -o $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(NPM_DIR)/*.go +# Build Dummy. +$(DUMMY_BUILD_DIR)/dummy$(EXE_EXT): $(DUMMYFILES) + go build -v -o $(DUMMY_BUILD_DIR)/dummy$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(DUMMY_DIR)/*.go + # Build all binaries in a container. .PHONY: all-containerized all-containerized: diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 700d7a2ad9..8ca92b31e5 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -204,6 +204,103 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE return hcnEndpoint, nil } +// configureTempHcnEndpoint configures hcn endpoint for creation +func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeEndpoint, error) { + //infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) + var hcnNetwork *hcn.HostComputeNetwork + var err error + if hcnNetwork, err = hcn.GetNetworkByName("secondary-nw"); err != nil { + log.Printf("[net] Failed to get temp nw due to error: %v", err) + return nil, fmt.Errorf("Failed to get hcn network with id: %s due to err: %v", nw.HnsId, err) + } + + hcnEndpoint := &hcn.HostComputeEndpoint{ + Name: "secondary-ep", + HostComputeNetwork: hcnNetwork.Id, /* + Dns: hcn.Dns{ + Domain: epInfo.DNS.Suffix, + ServerList: epInfo.DNS.Servers, + },*/ + SchemaVersion: hcn.SchemaVersion{ + Major: hcnSchemaVersionMajor, + Minor: hcnSchemaVersionMinor, + }, + //MacAddress: epInfo.MacAddress.String(), + } + + /* + if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { + for _, epPolicy := range endpointPolicies { + hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) + } + } else { + log.Printf("[net] Failed to get endpoint policies due to error: %v", err) + return nil, err + } + */ + + for _, route := range epInfo.Routes { + hcnRoute := hcn.Route{ + //NextHop: "169.254.0.1", + NextHop: "172.21.11.1", + DestinationPrefix: route.Dst.String(), + } + + hcnEndpoint.Routes = append(hcnEndpoint.Routes, hcnRoute) + } + + for _, ipAddress := range epInfo.IPAddresses { + prefixLength, _ := ipAddress.Mask.Size() + ipConfiguration := hcn.IpConfig{ + //IpAddress: "169.254.0.6", + IpAddress: "172.21.11.5", + PrefixLength: uint8(prefixLength), + } + + hcnEndpoint.IpConfigurations = append(hcnEndpoint.IpConfigurations, ipConfiguration) + } + + return hcnEndpoint, nil +} + +// newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 +func (nw *network) createTempEp(epInfo *EndpointInfo) (*endpoint, error) { + hcnEndpoint, err := nw.configureTempHcnEndpoint(epInfo) + if err != nil { + log.Printf("[net] Failed to configure hcn endpoint due to error: %v", err) + return nil, err + } + + // Create the HCN endpoint. + log.Printf("[net] Creating temp hcn endpoint: %+v", hcnEndpoint) + hnsResponse, err := hcnEndpoint.Create() + if err != nil { + return nil, fmt.Errorf("Failed to create endpoint: %s due to error: %v", hcnEndpoint.Name, err) + } + + log.Printf("[net] Successfully created temp hcn endpoint with response: %+v", hnsResponse) + + defer func() { + if err != nil { + log.Printf("[net] Deleting hcn endpoint with id: %s", hnsResponse.Id) + err = hnsResponse.Delete() + log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", hnsResponse.Id, err) + } + }() + + var namespace *hcn.HostComputeNamespace + if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { + return nil, fmt.Errorf("Failed to get hcn namespace: %s due to error: %v", epInfo.NetNsPath, err) + } + + if err = hcn.AddNamespaceEndpoint(namespace.Id, hnsResponse.Id); err != nil { + return nil, fmt.Errorf("[net] Failed to add endpoint: %s to hcn namespace: %s due to error: %v", + hnsResponse.Id, namespace.Id, err) + } + + return nil, nil +} + // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { hcnEndpoint, err := nw.configureHcnEndpoint(epInfo) @@ -251,6 +348,8 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) gateway = net.ParseIP(hnsResponse.Routes[0].NextHop) } + nw.createTempEp(epInfo) + // Create the endpoint object. ep := &endpoint{ Id: hcnEndpoint.Name, diff --git a/network/network_windows.go b/network/network_windows.go index 0269ab5d6b..66aeabf4b9 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -237,6 +237,8 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern vlanid = (int)(vlanID) } + nm.createTempNw(nwInfo, extIf) + // Create the network object. nw := &network{ Id: nwInfo.Id, @@ -252,6 +254,133 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern return nw, nil } +// configureHcnEndpoint configures hcn endpoint for creation +func (nm *networkManager) configureTempHcnNetwork(nwInfo *NetworkInfo, extIf *externalInterface) (*hcn.HostComputeNetwork, error) { + // Initialize HNS network. + hcnNetwork := &hcn.HostComputeNetwork{ + Name: "secondary-nw", + Dns: hcn.Dns{ + Domain: nwInfo.DNS.Suffix, + ServerList: nwInfo.DNS.Servers, + }, + Ipams: []hcn.Ipam{ + hcn.Ipam{ + Type: hcnIpamTypeStatic, + }, + }, + SchemaVersion: hcn.SchemaVersion{ + Major: hcnSchemaVersionMajor, + Minor: hcnSchemaVersionMinor, + }, + } + + // Set hcn network adaptor name policy + // FixMe: Find a better way to check if a nic that is selected is not part of a vSwitch + /* + if !strings.HasPrefix(extIf.Name, "vEthernet") { + netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy(extIf.Name) + if err != nil { + log.Printf("[net] Failed to serialize network adapter policy due to error: %v", err) + return nil, err + } + + hcnNetwork.Policies = append(hcnNetwork.Policies, netAdapterNamePolicy) + }*/ + + // Set hcn subnet policy + var vlanid int + vlanid = 0 + var subnetPolicy []byte + /* + opt, _ := nwInfo.Options[genericData].(map[string]interface{}) + if opt != nil && opt[VlanIDKey] != nil { + var err error + vlanID, _ := strconv.ParseUint(opt[VlanIDKey].(string), 10, 32) + subnetPolicy, err = policy.SerializeHcnSubnetVlanPolicy((uint32)(vlanID)) + if err != nil { + log.Printf("[net] Failed to serialize subnet vlan policy due to error: %v", err) + return nil, err + } + + vlanid = (int)(vlanID) + } + */ + + // Set network mode. + switch nwInfo.Mode { + case opModeBridge: + hcnNetwork.Type = hcn.L2Bridge + case opModeTunnel: + hcnNetwork.Type = hcn.L2Tunnel + default: + return nil, errNetworkModeInvalid + } + + // Populate subnets. + hnsSubnet := hcn.Subnet{ + //IpAddressPrefix: "169.254.0.0/16", + IpAddressPrefix: "172.21.11.0/24", + Routes: []hcn.Route{ + hcn.Route{ + //NextHop: "169.254.0.1", + NextHop: "172.21.11.1", + DestinationPrefix: "0.0.0.0/0", + }, + }, + } + + // Set the subnet policy + if vlanid > 0 { + hnsSubnet.Policies = append(hnsSubnet.Policies, subnetPolicy) + } + + hcnNetwork.Ipams[0].Subnets = append(hcnNetwork.Ipams[0].Subnets, hnsSubnet) + + return hcnNetwork, nil +} + +// createTempNw creates a new container network for HNSv2. +func (nm *networkManager) createTempNw(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { + log.Printf("[net] ashvind -- creating temp network...") + var hcnNetwork *hcn.HostComputeNetwork + var err error + if hcnNetwork, err = hcn.GetNetworkByName("secondary-nw"); err == nil { + log.Printf("[net] Found temp nw") + return nil, nil + } + + hcnNetwork, err = nm.configureTempHcnNetwork(nwInfo, extIf) + if err != nil { + log.Printf("[net] Failed to configure hcn network due to error: %v", err) + return nil, err + } + + // Create the HNS network. + log.Printf("[net] Creating temp hcn network: %+v", hcnNetwork) + hnsResponse, err := hcnNetwork.Create() + + if err != nil { + log.Printf("[net] Failed to create temp hcn network due to error: %v", err) + return nil, fmt.Errorf("Failed to create hcn network: %s due to error: %v", hcnNetwork.Name, err) + } + + log.Printf("[net] Successfully created temp hcn network with response: %+v", hnsResponse) + + // Create the network object. + nw := &network{ + Id: nwInfo.Id, + HnsId: hnsResponse.Id, + Mode: nwInfo.Mode, + Endpoints: make(map[string]*endpoint), + extIf: extIf, + VlanId: 0, + EnableSnatOnHost: nwInfo.EnableSnatOnHost, + NetNs: nwInfo.NetNs, + } + + return nw, nil +} + // NewNetworkImpl creates a new container network. func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { if useHnsV2, err := UseHnsV2(nwInfo.NetNs); useHnsV2 { From b1147b316fce6e840e7bf7f8a32370f2bd75d26a Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Tue, 21 May 2019 06:08:13 -0700 Subject: [PATCH 09/37] WIP-vm to container 2 --- network/endpoint_windows.go | 54 +++++++++++++++++++++++--------- network/network_windows.go | 8 ++--- network/policy/policy_windows.go | 18 +++++++++-- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 8ca92b31e5..bf47176398 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -173,7 +173,7 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE MacAddress: epInfo.MacAddress.String(), } - if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { + if endpointPolicies, err := policy.GetHcnEndpointPolicies(false, policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { for _, epPolicy := range endpointPolicies { hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) } @@ -205,7 +205,7 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE } // configureTempHcnEndpoint configures hcn endpoint for creation -func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeEndpoint, error) { +func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo, gwEp bool) (*hcn.HostComputeEndpoint, error) { //infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) var hcnNetwork *hcn.HostComputeNetwork var err error @@ -214,8 +214,14 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp return nil, fmt.Errorf("Failed to get hcn network with id: %s due to err: %v", nw.HnsId, err) } + var name string + if gwEp { + name = "secondaryepgw" + } else { + name = "secondaryepwin" + } hcnEndpoint := &hcn.HostComputeEndpoint{ - Name: "secondary-ep", + Name: name, HostComputeNetwork: hcnNetwork.Id, /* Dns: hcn.Dns{ Domain: epInfo.DNS.Suffix, @@ -228,8 +234,8 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp //MacAddress: epInfo.MacAddress.String(), } - /* - if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { + if !gwEp { + if endpointPolicies, err := policy.GetHcnEndpointPolicies(true, policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { for _, epPolicy := range endpointPolicies { hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) } @@ -237,23 +243,35 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp log.Printf("[net] Failed to get endpoint policies due to error: %v", err) return nil, err } - */ + } + var nexthop string + if gwEp { + nexthop = "0.0.0.0" + } else { + nexthop = "169.254.0.2" + } for _, route := range epInfo.Routes { hcnRoute := hcn.Route{ - //NextHop: "169.254.0.1", - NextHop: "172.21.11.1", + NextHop: nexthop, //"169.254.0.1", + //NextHop: "172.21.9.1", DestinationPrefix: route.Dst.String(), } hcnEndpoint.Routes = append(hcnEndpoint.Routes, hcnRoute) } + var ipaddress string + if gwEp { + ipaddress = "169.254.0.2" + } else { + ipaddress = "169.254.0.8" + } for _, ipAddress := range epInfo.IPAddresses { prefixLength, _ := ipAddress.Mask.Size() ipConfiguration := hcn.IpConfig{ - //IpAddress: "169.254.0.6", - IpAddress: "172.21.11.5", + IpAddress: ipaddress, + //IpAddress: "172.21.9.5", PrefixLength: uint8(prefixLength), } @@ -264,8 +282,8 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp } // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 -func (nw *network) createTempEp(epInfo *EndpointInfo) (*endpoint, error) { - hcnEndpoint, err := nw.configureTempHcnEndpoint(epInfo) +func (nw *network) createTempEp(epInfo *EndpointInfo, gwEp bool) (*endpoint, error) { + hcnEndpoint, err := nw.configureTempHcnEndpoint(epInfo, gwEp) if err != nil { log.Printf("[net] Failed to configure hcn endpoint due to error: %v", err) return nil, err @@ -275,6 +293,7 @@ func (nw *network) createTempEp(epInfo *EndpointInfo) (*endpoint, error) { log.Printf("[net] Creating temp hcn endpoint: %+v", hcnEndpoint) hnsResponse, err := hcnEndpoint.Create() if err != nil { + log.Printf("[net] failed to created temp hcn endpoint err %v", err) return nil, fmt.Errorf("Failed to create endpoint: %s due to error: %v", hcnEndpoint.Name, err) } @@ -293,7 +312,13 @@ func (nw *network) createTempEp(epInfo *EndpointInfo) (*endpoint, error) { return nil, fmt.Errorf("Failed to get hcn namespace: %s due to error: %v", epInfo.NetNsPath, err) } - if err = hcn.AddNamespaceEndpoint(namespace.Id, hnsResponse.Id); err != nil { + var nsId string + if gwEp { + nsId = "b1062982-2b18-4b4f-b3d5-a78ddb9cdd49" + } else { + nsId = namespace.Id + } + if err = hcn.AddNamespaceEndpoint(nsId, hnsResponse.Id); err != nil { return nil, fmt.Errorf("[net] Failed to add endpoint: %s to hcn namespace: %s due to error: %v", hnsResponse.Id, namespace.Id, err) } @@ -348,7 +373,8 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) gateway = net.ParseIP(hnsResponse.Routes[0].NextHop) } - nw.createTempEp(epInfo) + nw.createTempEp(epInfo, false) + //nw.createTempEp(epInfo, true) // Create the endpoint object. ep := &endpoint{ diff --git a/network/network_windows.go b/network/network_windows.go index 66aeabf4b9..602a0971fa 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -318,12 +318,12 @@ func (nm *networkManager) configureTempHcnNetwork(nwInfo *NetworkInfo, extIf *ex // Populate subnets. hnsSubnet := hcn.Subnet{ - //IpAddressPrefix: "169.254.0.0/16", - IpAddressPrefix: "172.21.11.0/24", + IpAddressPrefix: "169.254.0.0/16", + //IpAddressPrefix: "172.21.9.0/24", Routes: []hcn.Route{ hcn.Route{ - //NextHop: "169.254.0.1", - NextHop: "172.21.11.1", + NextHop: "169.254.0.1", + //NextHop: "172.21.9.1", DestinationPrefix: "0.0.0.0/0", }, }, diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index 4ad4d3008b..eb8dd04704 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -327,7 +327,7 @@ func GetHcnPortMappingPolicy(policy Policy) (hcn.EndpointPolicy, error) { } // GetHcnEndpointPolicies returns array of all endpoint policies. -func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}) ([]hcn.EndpointPolicy, error) { +func GetHcnEndpointPolicies(onlyPortMapping bool, policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}) ([]hcn.EndpointPolicy, error) { var hcnEndPointPolicies []hcn.EndpointPolicy for _, policy := range policies { if policy.Type == policyType { @@ -336,11 +336,23 @@ func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoD switch GetPolicyType(policy) { case OutBoundNatPolicy: + //if !onlyPortMapping { endpointPolicy, err = GetHcnOutBoundNATPolicy(policy, epInfoData) + hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) + log.Printf("Successfully set the policy: %+v", endpointPolicy) + //} case RoutePolicy: + //if !onlyPortMapping { endpointPolicy, err = GetHcnRoutePolicy(policy) + hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) + log.Printf("Successfully set the policy: %+v", endpointPolicy) + //} case PortMappingPolicy: + //if onlyPortMapping { endpointPolicy, err = GetHcnPortMappingPolicy(policy) + hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) + log.Printf("Successfully set the policy: %+v", endpointPolicy) + //} default: // return error as we should be able to parse all the policies specified return hcnEndPointPolicies, fmt.Errorf("Failed to set Policy: Type: %s, Data: %s", policy.Type, policy.Data) @@ -351,8 +363,8 @@ func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoD return hcnEndPointPolicies, err } - hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) - log.Printf("Successfully set the policy: %+v", endpointPolicy) + //hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) + //log.Printf("Successfully set the policy: %+v", endpointPolicy) } } From 76dfe9c104570d87ea7f6feda874de2f4189c288 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Wed, 22 May 2019 01:51:18 -0700 Subject: [PATCH 10/37] vm container connect 3 --- cni/network/multitenancy.go | 6 +- network/endpoint_windows.go | 127 ++++++++++++++++++++++++++++-------- network/network_windows.go | 2 + 3 files changed, 104 insertions(+), 31 deletions(-) diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index a7b6ebc707..758135cfaf 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -28,12 +28,12 @@ func SetupRoutingForMultitenancy( if nwCfg.MultiTenancy { // if snat enabled, add 169.254.0.1 as default gateway if nwCfg.EnableSnatOnHost { - log.Printf("add default route for multitenancy.snat on host enabled") - addDefaultRoute(cnsNetworkConfig.LocalIPConfiguration.GatewayIPAddress, epInfo, result) + log.Printf("add default route for multitenancy.snat on host enabled with .2") + addDefaultRoute("169.254.0.2" /*cnsNetworkConfig.LocalIPConfiguration.GatewayIPAddress*/, epInfo, result) } else { _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} - gwIP := net.ParseIP(cnsNetworkConfig.IPConfiguration.GatewayIPAddress) + gwIP := net.ParseIP("169.254.0.2") //cnsNetworkConfig.IPConfiguration.GatewayIPAddress) epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP}) result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index bf47176398..b155309812 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -218,7 +218,7 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo, gwEp bool) (*h if gwEp { name = "secondaryepgw" } else { - name = "secondaryepwin" + name = "secondaryepwin2" } hcnEndpoint := &hcn.HostComputeEndpoint{ Name: name, @@ -234,16 +234,17 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo, gwEp bool) (*h //MacAddress: epInfo.MacAddress.String(), } - if !gwEp { - if endpointPolicies, err := policy.GetHcnEndpointPolicies(true, policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { - for _, epPolicy := range endpointPolicies { - hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) + /* + if !gwEp { + if endpointPolicies, err := policy.GetHcnEndpointPolicies(true, policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { + for _, epPolicy := range endpointPolicies { + hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) + } + } else { + log.Printf("[net] Failed to get endpoint policies due to error: %v", err) + return nil, err } - } else { - log.Printf("[net] Failed to get endpoint policies due to error: %v", err) - return nil, err - } - } + }*/ var nexthop string if gwEp { @@ -251,33 +252,33 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo, gwEp bool) (*h } else { nexthop = "169.254.0.2" } - for _, route := range epInfo.Routes { - hcnRoute := hcn.Route{ - NextHop: nexthop, //"169.254.0.1", - //NextHop: "172.21.9.1", - DestinationPrefix: route.Dst.String(), - } - - hcnEndpoint.Routes = append(hcnEndpoint.Routes, hcnRoute) + //for _, route := range epInfo.Routes { + hcnRoute := hcn.Route{ + NextHop: nexthop, //"169.254.0.1", + //NextHop: "172.21.9.1", + DestinationPrefix: "0.0.0.0/0", //route.Dst.String(), } + hcnEndpoint.Routes = append(hcnEndpoint.Routes, hcnRoute) + //} + var ipaddress string if gwEp { ipaddress = "169.254.0.2" } else { - ipaddress = "169.254.0.8" + ipaddress = "169.254.0.13" } - for _, ipAddress := range epInfo.IPAddresses { - prefixLength, _ := ipAddress.Mask.Size() - ipConfiguration := hcn.IpConfig{ - IpAddress: ipaddress, - //IpAddress: "172.21.9.5", - PrefixLength: uint8(prefixLength), - } - - hcnEndpoint.IpConfigurations = append(hcnEndpoint.IpConfigurations, ipConfiguration) + //for _, ipAddress := range epInfo.IPAddresses { + //prefixLength, _ := ipAddress.Mask.Size() + ipConfiguration := hcn.IpConfig{ + IpAddress: ipaddress, + //IpAddress: "172.21.9.5", + PrefixLength: 16, //uint8(prefixLength), } + hcnEndpoint.IpConfigurations = append(hcnEndpoint.IpConfigurations, ipConfiguration) + //} + return hcnEndpoint, nil } @@ -326,6 +327,75 @@ func (nw *network) createTempEp(epInfo *EndpointInfo, gwEp bool) (*endpoint, err return nil, nil } +// createGwEpv1 creates a new endpoint in the network using HnsV1 +func (nw *network) createGwEpv1(epInfo *EndpointInfo) (*endpoint, error) { + + var hnsNetwork *hcsshim.HNSNetwork + var err error + var nwid string + if hnsNetwork, err = hcsshim.GetHNSNetworkByName("secondary-nw"); err != nil { + log.Printf("[net] Failed to get temp nw due to error: %v", err) + return nil, fmt.Errorf("Failed to get hcn network with id: %s due to err: %v", nw.HnsId, err) + } + + nwid = hnsNetwork.Id + + // Get Infrastructure containerID. Handle ADD calls for workload container. + //var err error + hnsEndpoint := &hcsshim.HNSEndpoint{ + Name: "secondaryepgw", + VirtualNetwork: nwid, + GatewayAddress: "0.0.0.0", + //DNSSuffix: epInfo.DNS.Suffix, + //DNSServerList: strings.Join(epInfo.DNS.Servers, ","), + //Policies: policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data), + } + + // HNS currently supports only one IP address per endpoint. + if epInfo.IPAddresses != nil { + hnsEndpoint.IPAddress = net.ParseIP("169.254.0.2") + //pl, _ := epInfo.IPAddresses[0].Mask.Size() + hnsEndpoint.PrefixLength = 16 //uint8(pl) + } + + log.Printf("[net] HNSEndpointRequest :%+v", hnsEndpoint) + + // Marshal the request. + buffer, err := json.Marshal(hnsEndpoint) + if err != nil { + return nil, err + } + hnsRequest := string(buffer) + + // Create the HNS endpoint. + log.Printf("[net] HNSEndpointRequest POST request:%+v", hnsRequest) + hnsResponse, err := hcsshim.HNSEndpointRequest("POST", "", hnsRequest) + log.Printf("[net] HNSEndpointRequest POST response:%+v err:%v.", hnsResponse, err) + if err != nil { + return nil, err + } + + defer func() { + if err != nil { + log.Printf("[net] HNSEndpointRequest DELETE id:%v", hnsResponse.Id) + hnsResponse, err := hcsshim.HNSEndpointRequest("DELETE", hnsResponse.Id, "") + log.Printf("[net] HNSEndpointRequest DELETE response:%+v err:%v.", hnsResponse, err) + } + }() + + // Attach the endpoint. + log.Printf("[net] Attaching endpoint %v host.", hnsResponse.Id) + err = hnsResponse.HostAttach(1) + if err != nil { + log.Printf("[net] Failed to attach endpoint to default compartment due to err %v.", err) + return nil, err + } else { + log.Printf("[net] Succeeded attaching %v to host.", hnsResponse.Id) + } + + return nil, nil +} + // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { hcnEndpoint, err := nw.configureHcnEndpoint(epInfo) @@ -375,6 +445,7 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) nw.createTempEp(epInfo, false) //nw.createTempEp(epInfo, true) + //nw.createGwEpv1(epInfo) // Create the endpoint object. ep := &endpoint{ diff --git a/network/network_windows.go b/network/network_windows.go index 602a0971fa..0b5ca796ce 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -316,6 +316,8 @@ func (nm *networkManager) configureTempHcnNetwork(nwInfo *NetworkInfo, extIf *ex return nil, errNetworkModeInvalid } + //hcnNetwork.Type = hcn.L2Bridge + // Populate subnets. hnsSubnet := hcn.Subnet{ IpAddressPrefix: "169.254.0.0/16", From adf07f5c9db62f9fcc1627141498bb4c119aa2a0 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Thu, 13 Jun 2019 17:16:19 -0700 Subject: [PATCH 11/37] Fix dns_config searches and options --- cni/network/network_windows.go | 7 ++++--- network/endpoint_windows.go | 3 ++- network/network.go | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index fba8f6cbb3..ecc3caf717 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -185,11 +185,13 @@ func getEndpointDNSSettings(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Resul epDNS = network.DNSInfo{ Servers: nwCfg.DNS.Nameservers, Suffix: namespace + "." + strings.Join(nwCfg.DNS.Search, ","), + Options: nwCfg.DNS.Options, } } else { epDNS = network.DNSInfo{ - Suffix: result.DNS.Domain, Servers: result.DNS.Nameservers, + Suffix: result.DNS.Domain, + Options: nwCfg.DNS.Options, } } @@ -221,8 +223,6 @@ func getPoliciesFromRuntimeCfg(nwCfg *cni.NetworkConfig) []policy.Policy { } func getCustomDNS(nwCfg *cni.NetworkConfig) network.DNSInfo { - log.Printf("[net] RuntimeConfigs: %+v", nwCfg.RuntimeConfig) - var search string if len(nwCfg.RuntimeConfig.DNS.Searches) > 0 { search = strings.Join(nwCfg.RuntimeConfig.DNS.Searches, ",") @@ -231,5 +231,6 @@ func getCustomDNS(nwCfg *cni.NetworkConfig) network.DNSInfo { return network.DNSInfo{ Servers: nwCfg.RuntimeConfig.DNS.Servers, Suffix: search, + Options: nwCfg.RuntimeConfig.DNS.Options, } } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 700d7a2ad9..86cf768264 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -163,8 +163,9 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE Name: infraEpName, HostComputeNetwork: nw.HnsId, Dns: hcn.Dns{ - Domain: epInfo.DNS.Suffix, + Search: strings.Split(epInfo.DNS.Suffix, ","), ServerList: epInfo.DNS.Servers, + Options: epInfo.DNS.Options, }, SchemaVersion: hcn.SchemaVersion{ Major: hcnSchemaVersionMajor, diff --git a/network/network.go b/network/network.go index d5ec10dc7c..15b6a2d65f 100644 --- a/network/network.go +++ b/network/network.go @@ -73,6 +73,7 @@ type SubnetInfo struct { type DNSInfo struct { Suffix string Servers []string + Options []string } // NewExternalInterface adds a host interface to the list of available external interfaces. From 381c3c6363be7cac76aabbb6a0dc022f0ca5e97d Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Mon, 22 Jul 2019 09:19:54 -0700 Subject: [PATCH 12/37] enable the gw endpoint creation --- network/endpoint_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index b155309812..585bf10872 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -445,7 +445,7 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) nw.createTempEp(epInfo, false) //nw.createTempEp(epInfo, true) - //nw.createGwEpv1(epInfo) + nw.createGwEpv1(epInfo) // Create the endpoint object. ep := &endpoint{ From 1649ad8823a6b9d6e0105cb1768b2229ce5f5d86 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Tue, 13 Aug 2019 17:06:04 -0700 Subject: [PATCH 13/37] Use the localIP and do not create gw ep - use the loopback with gw IP --- cni/network/network_windows.go | 1 + network/endpoint_windows.go | 52 +++++++++++++--------------------- network/manager.go | 1 + network/network_windows.go | 22 +++++++------- 4 files changed, 33 insertions(+), 43 deletions(-) diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index fba8f6cbb3..2fc2448f85 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -93,6 +93,7 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne cnetAddressMap = append(cnetAddressMap, ipSubnet.IPAddress+"/"+strconv.Itoa(int(ipSubnet.PrefixLength))) } epInfo.Data[network.CnetAddressSpace] = cnetAddressMap + epInfo.Data[network.LocalIPKey] = cnsNwConfig.LocalIPConfiguration.IPSubnet.IPAddress } } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 585bf10872..0b934c4451 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -205,7 +205,7 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE } // configureTempHcnEndpoint configures hcn endpoint for creation -func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo, gwEp bool) (*hcn.HostComputeEndpoint, error) { +func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeEndpoint, error) { //infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) var hcnNetwork *hcn.HostComputeNetwork var err error @@ -214,12 +214,8 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo, gwEp bool) (*h return nil, fmt.Errorf("Failed to get hcn network with id: %s due to err: %v", nw.HnsId, err) } - var name string - if gwEp { - name = "secondaryepgw" - } else { - name = "secondaryepwin2" - } + name := "secondaryepwin" + hcnEndpoint := &hcn.HostComputeEndpoint{ Name: name, HostComputeNetwork: hcnNetwork.Id, /* @@ -246,32 +242,29 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo, gwEp bool) (*h } }*/ - var nexthop string - if gwEp { - nexthop = "0.0.0.0" - } else { - nexthop = "169.254.0.2" - } + nexthop := "169.254.0.2" //for _, route := range epInfo.Routes { hcnRoute := hcn.Route{ - NextHop: nexthop, //"169.254.0.1", - //NextHop: "172.21.9.1", + NextHop: nexthop, DestinationPrefix: "0.0.0.0/0", //route.Dst.String(), } hcnEndpoint.Routes = append(hcnEndpoint.Routes, hcnRoute) //} - var ipaddress string - if gwEp { - ipaddress = "169.254.0.2" - } else { - ipaddress = "169.254.0.13" + var localIP string + if epInfo.Data != nil { + if localIPData, ok := epInfo.Data[LocalIPKey]; ok { + localIP = localIPData.(string) + } } + + //ipaddress := "169.254.0.13" + //for _, ipAddress := range epInfo.IPAddresses { //prefixLength, _ := ipAddress.Mask.Size() ipConfiguration := hcn.IpConfig{ - IpAddress: ipaddress, + IpAddress: localIP, //IpAddress: "172.21.9.5", PrefixLength: 16, //uint8(prefixLength), } @@ -283,8 +276,8 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo, gwEp bool) (*h } // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 -func (nw *network) createTempEp(epInfo *EndpointInfo, gwEp bool) (*endpoint, error) { - hcnEndpoint, err := nw.configureTempHcnEndpoint(epInfo, gwEp) +func (nw *network) createTempEp(epInfo *EndpointInfo) (*endpoint, error) { + hcnEndpoint, err := nw.configureTempHcnEndpoint(epInfo) if err != nil { log.Printf("[net] Failed to configure hcn endpoint due to error: %v", err) return nil, err @@ -313,12 +306,8 @@ func (nw *network) createTempEp(epInfo *EndpointInfo, gwEp bool) (*endpoint, err return nil, fmt.Errorf("Failed to get hcn namespace: %s due to error: %v", epInfo.NetNsPath, err) } - var nsId string - if gwEp { - nsId = "b1062982-2b18-4b4f-b3d5-a78ddb9cdd49" - } else { - nsId = namespace.Id - } + nsId := namespace.Id + if err = hcn.AddNamespaceEndpoint(nsId, hnsResponse.Id); err != nil { return nil, fmt.Errorf("[net] Failed to add endpoint: %s to hcn namespace: %s due to error: %v", hnsResponse.Id, namespace.Id, err) @@ -443,9 +432,8 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) gateway = net.ParseIP(hnsResponse.Routes[0].NextHop) } - nw.createTempEp(epInfo, false) - //nw.createTempEp(epInfo, true) - nw.createGwEpv1(epInfo) + nw.createTempEp(epInfo) + //nw.createGwEpv1(epInfo) // Create the endpoint object. ep := &endpoint{ diff --git a/network/manager.go b/network/manager.go index 7ac8aaab43..b762d2448b 100644 --- a/network/manager.go +++ b/network/manager.go @@ -17,6 +17,7 @@ const ( // Network store key. storeKey = "Network" VlanIDKey = "VlanID" + LocalIPKey = "localIP" genericData = "com.docker.network.generic" ) diff --git a/network/network_windows.go b/network/network_windows.go index 0b5ca796ce..ccf3af0644 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -276,20 +276,20 @@ func (nm *networkManager) configureTempHcnNetwork(nwInfo *NetworkInfo, extIf *ex // Set hcn network adaptor name policy // FixMe: Find a better way to check if a nic that is selected is not part of a vSwitch - /* - if !strings.HasPrefix(extIf.Name, "vEthernet") { - netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy(extIf.Name) - if err != nil { - log.Printf("[net] Failed to serialize network adapter policy due to error: %v", err) - return nil, err - } + //if !strings.HasPrefix(extIf.Name, "vEthernet") { + netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy("Ethernet 6") + if err != nil { + log.Printf("[net] Failed to serialize network adapter policy due to error: %v", err) + return nil, err + } - hcnNetwork.Policies = append(hcnNetwork.Policies, netAdapterNamePolicy) - }*/ + hcnNetwork.Policies = append(hcnNetwork.Policies, netAdapterNamePolicy) + //} // Set hcn subnet policy var vlanid int vlanid = 0 + var subnetPolicy []byte /* opt, _ := nwInfo.Options[genericData].(map[string]interface{}) @@ -316,7 +316,7 @@ func (nm *networkManager) configureTempHcnNetwork(nwInfo *NetworkInfo, extIf *ex return nil, errNetworkModeInvalid } - //hcnNetwork.Type = hcn.L2Bridge + hcnNetwork.Type = hcn.L2Bridge // Populate subnets. hnsSubnet := hcn.Subnet{ @@ -324,7 +324,7 @@ func (nm *networkManager) configureTempHcnNetwork(nwInfo *NetworkInfo, extIf *ex //IpAddressPrefix: "172.21.9.0/24", Routes: []hcn.Route{ hcn.Route{ - NextHop: "169.254.0.1", + NextHop: "169.254.0.2", //NextHop: "172.21.9.1", DestinationPrefix: "0.0.0.0/0", }, From 3348ce3083500632853c5f128964d4debcc5c42c Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Tue, 20 Aug 2019 13:24:33 -0700 Subject: [PATCH 14/37] add endpoint acls --- network/endpoint_windows.go | 92 ++++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 7 deletions(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 0b934c4451..c49a5e719c 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -242,6 +242,91 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp } }*/ + var localIP string + if epInfo.Data != nil { + if localIPData, ok := epInfo.Data[LocalIPKey]; ok { + localIP = localIPData.(string) + } + } + + /* + var remoteIP string + if localIP == "169.254.0.5" { + remoteIP = "169.254.0.4" + } else { + remoteIP = "169.254.0.5" + }*/ + + /* + // Add endpoint ACL for preventing the comm to other apipa + endpointAclBlock := hcn.AclPolicySetting{ + Protocols: "1", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err := json.Marshal(endpointAclBlock) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for preventing the comm to other apipa + endpointAclAllow := hcn.AclPolicySetting{ + Protocols: "1", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RemoteAddresses: "169.254.0.2", + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(endpointAclAllow) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + + + // Add endpoint ACL for preventing the comm to other apipa + endpointAclInbound := hcn.AclPolicySetting{ + Protocols: "1", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(endpointAclInbound) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + */ + nexthop := "169.254.0.2" //for _, route := range epInfo.Routes { hcnRoute := hcn.Route{ @@ -252,13 +337,6 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp hcnEndpoint.Routes = append(hcnEndpoint.Routes, hcnRoute) //} - var localIP string - if epInfo.Data != nil { - if localIPData, ok := epInfo.Data[LocalIPKey]; ok { - localIP = localIPData.(string) - } - } - //ipaddress := "169.254.0.13" //for _, ipAddress := range epInfo.IPAddresses { From 04e4e5d97a66169d95e31c08ee59637e21522a48 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Tue, 17 Sep 2019 18:04:57 -0700 Subject: [PATCH 15/37] WIP --- network/endpoint_windows.go | 301 ++++++++++++++++++++++++++++++------ 1 file changed, 256 insertions(+), 45 deletions(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index c49a5e719c..56b1c78617 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -257,64 +257,136 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp remoteIP = "169.254.0.5" }*/ - /* - // Add endpoint ACL for preventing the comm to other apipa - endpointAclBlock := hcn.AclPolicySetting{ - Protocols: "1", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } + /********************************************************************************************************/ + // Add ICMP ACLs + { + // Add endpoint ACL for preventing the comm to other apipa + aclOutBlockAll := hcn.AclPolicySetting{ + Protocols: "1", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } - rawJSON, err := json.Marshal(endpointAclBlock) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } + rawJSON, err := json.Marshal(aclOutBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } - endpointPolicy := hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - - // Add endpoint ACL for preventing the comm to other apipa - endpointAclAllow := hcn.AclPolicySetting{ - Protocols: "1", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RemoteAddresses: "169.254.0.2", - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - rawJSON, err = json.Marshal(endpointAclAllow) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } + // Add endpoint ACL for allowing out to host apipa + aclOutAllowToHostOnly := hcn.AclPolicySetting{ + Protocols: "1", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RemoteAddresses: "169.254.0.2", + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } + rawJSON, err = json.Marshal(aclOutAllowToHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - // Add endpoint ACL for preventing the comm to other apipa - endpointAclInbound := hcn.AclPolicySetting{ + // Add endpoint ACL for allowing in from host apipa + aclInBlockAll := hcn.AclPolicySetting{ Protocols: "1", - Action: hcn.ActionTypeAllow, + Action: hcn.ActionTypeBlock, Direction: hcn.DirectionTypeIn, LocalAddresses: localIP, RuleType: hcn.RuleTypeSwitch, - Priority: 200, + Priority: 2000, + } + + rawJSON, err = json.Marshal(aclInBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing in from host apipa + aclInAllowFromHostOnly := hcn.AclPolicySetting{ + Protocols: "1", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RemoteAddresses: "169.254.0.2", + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(aclInAllowFromHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + } + + // Add TCP ACLs + { + // Add endpoint ACL for preventing the comm to other apipa + aclOutBlockAll := hcn.AclPolicySetting{ + Protocols: "6", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err := json.Marshal(aclOutBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing out to host apipa + aclOutAllowToHostOnly := hcn.AclPolicySetting{ + Protocols: "6", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RemoteAddresses: "169.254.0.2", + RuleType: hcn.RuleTypeSwitch, + Priority: 200, } - rawJSON, err = json.Marshal(endpointAclInbound) + rawJSON, err = json.Marshal(aclOutAllowToHostOnly) if err != nil { return nil, fmt.Errorf("Failed to marshal the endpoint ACL") } @@ -325,7 +397,146 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp } hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - */ + + // Add endpoint ACL for allowing in from host apipa + aclInBlockAll := hcn.AclPolicySetting{ + Protocols: "6", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err = json.Marshal(aclInBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing in from host apipa + aclInAllowFromHostOnly := hcn.AclPolicySetting{ + Protocols: "6", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RemoteAddresses: "169.254.0.2", + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(aclInAllowFromHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + } + + // Add UDP ACLs + { + // Add endpoint ACL for preventing the comm to other apipa + aclOutBlockAll := hcn.AclPolicySetting{ + Protocols: "17", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err := json.Marshal(aclOutBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing out to host apipa + aclOutAllowToHostOnly := hcn.AclPolicySetting{ + Protocols: "17", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RemoteAddresses: "169.254.0.2", + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(aclOutAllowToHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing in from host apipa + aclInBlockAll := hcn.AclPolicySetting{ + Protocols: "17", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err = json.Marshal(aclInBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing in from host apipa + aclInAllowFromHostOnly := hcn.AclPolicySetting{ + Protocols: "17", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RemoteAddresses: "169.254.0.2", + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(aclInAllowFromHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) + } + /********************************************************************************************************/ nexthop := "169.254.0.2" //for _, route := range epInfo.Routes { From 69c6d6b4fb57ba14001e4e7f8796d336f5ccdac8 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Wed, 18 Sep 2019 15:16:12 -0700 Subject: [PATCH 16/37] WIP2 --- network/endpoint_windows.go | 133 +++++------------------------------- network/network_windows.go | 63 +++++------------ 2 files changed, 34 insertions(+), 162 deletions(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 56b1c78617..1443d8876e 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -204,8 +204,8 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE return hcnEndpoint, nil } -// configureTempHcnEndpoint configures hcn endpoint for creation -func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeEndpoint, error) { +// configureApipaEndpoint configures hcn endpoint for creation +func (nw *network) configureApipaEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeEndpoint, error) { //infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) var hcnNetwork *hcn.HostComputeNetwork var err error @@ -218,30 +218,14 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp hcnEndpoint := &hcn.HostComputeEndpoint{ Name: name, - HostComputeNetwork: hcnNetwork.Id, /* - Dns: hcn.Dns{ - Domain: epInfo.DNS.Suffix, - ServerList: epInfo.DNS.Servers, - },*/ + HostComputeNetwork: hcnNetwork.Id, SchemaVersion: hcn.SchemaVersion{ Major: hcnSchemaVersionMajor, Minor: hcnSchemaVersionMinor, }, - //MacAddress: epInfo.MacAddress.String(), } - /* - if !gwEp { - if endpointPolicies, err := policy.GetHcnEndpointPolicies(true, policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { - for _, epPolicy := range endpointPolicies { - hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) - } - } else { - log.Printf("[net] Failed to get endpoint policies due to error: %v", err) - return nil, err - } - }*/ - + // TODO: below code can be handled by passing the context to the CNS and CNS looking up localIP from the CNS config var localIP string if epInfo.Data != nil { if localIPData, ok := epInfo.Data[LocalIPKey]; ok { @@ -249,14 +233,6 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp } } - /* - var remoteIP string - if localIP == "169.254.0.5" { - remoteIP = "169.254.0.4" - } else { - remoteIP = "169.254.0.5" - }*/ - /********************************************************************************************************/ // Add ICMP ACLs { @@ -539,48 +515,40 @@ func (nw *network) configureTempHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComp /********************************************************************************************************/ nexthop := "169.254.0.2" - //for _, route := range epInfo.Routes { hcnRoute := hcn.Route{ NextHop: nexthop, - DestinationPrefix: "0.0.0.0/0", //route.Dst.String(), + DestinationPrefix: "0.0.0.0/0", } hcnEndpoint.Routes = append(hcnEndpoint.Routes, hcnRoute) - //} - //ipaddress := "169.254.0.13" - - //for _, ipAddress := range epInfo.IPAddresses { - //prefixLength, _ := ipAddress.Mask.Size() ipConfiguration := hcn.IpConfig{ - IpAddress: localIP, - //IpAddress: "172.21.9.5", - PrefixLength: 16, //uint8(prefixLength), + IpAddress: localIP, + PrefixLength: 16, // TODO: this should come from the cns config } hcnEndpoint.IpConfigurations = append(hcnEndpoint.IpConfigurations, ipConfiguration) - //} return hcnEndpoint, nil } // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 -func (nw *network) createTempEp(epInfo *EndpointInfo) (*endpoint, error) { - hcnEndpoint, err := nw.configureTempHcnEndpoint(epInfo) +func (nw *network) createApipaEndpoint(epInfo *EndpointInfo) (*endpoint, error) { + hcnEndpoint, err := nw.configureApipaEndpoint(epInfo) if err != nil { log.Printf("[net] Failed to configure hcn endpoint due to error: %v", err) return nil, err } // Create the HCN endpoint. - log.Printf("[net] Creating temp hcn endpoint: %+v", hcnEndpoint) + log.Printf("[net] Creating APIPA endpoint for host-container connectivity: %+v", hcnEndpoint) hnsResponse, err := hcnEndpoint.Create() if err != nil { - log.Printf("[net] failed to created temp hcn endpoint err %v", err) - return nil, fmt.Errorf("Failed to create endpoint: %s due to error: %v", hcnEndpoint.Name, err) + log.Printf("[net] failed to create APIPA endpoint due to error: %v", err) + return nil, fmt.Errorf("Failed to create APIPA endpoint: %s due to error: %v", hcnEndpoint.Name, err) } - log.Printf("[net] Successfully created temp hcn endpoint with response: %+v", hnsResponse) + log.Printf("[net] Successfully created APIPA endpoint for host-container connectivity: %+v", hnsResponse) defer func() { if err != nil { @@ -590,6 +558,9 @@ func (nw *network) createTempEp(epInfo *EndpointInfo) (*endpoint, error) { } }() + //TODO: until this point, ep creation should be handled by CNS + // after getting that ep, CNI will attach it to the container. + var namespace *hcn.HostComputeNamespace if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { return nil, fmt.Errorf("Failed to get hcn namespace: %s due to error: %v", epInfo.NetNsPath, err) @@ -605,75 +576,6 @@ func (nw *network) createTempEp(epInfo *EndpointInfo) (*endpoint, error) { return nil, nil } -// createGwEpv1 creates a new endpoint in the network using HnsV1 -func (nw *network) createGwEpv1(epInfo *EndpointInfo) (*endpoint, error) { - - var hnsNetwork *hcsshim.HNSNetwork - var err error - var nwid string - if hnsNetwork, err = hcsshim.GetHNSNetworkByName("secondary-nw"); err != nil { - log.Printf("[net] Failed to get temp nw due to error: %v", err) - return nil, fmt.Errorf("Failed to get hcn network with id: %s due to err: %v", nw.HnsId, err) - } - - nwid = hnsNetwork.Id - - // Get Infrastructure containerID. Handle ADD calls for workload container. - //var err error - hnsEndpoint := &hcsshim.HNSEndpoint{ - Name: "secondaryepgw", - VirtualNetwork: nwid, - GatewayAddress: "0.0.0.0", - //DNSSuffix: epInfo.DNS.Suffix, - //DNSServerList: strings.Join(epInfo.DNS.Servers, ","), - //Policies: policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data), - } - - // HNS currently supports only one IP address per endpoint. - if epInfo.IPAddresses != nil { - hnsEndpoint.IPAddress = net.ParseIP("169.254.0.2") - //pl, _ := epInfo.IPAddresses[0].Mask.Size() - hnsEndpoint.PrefixLength = 16 //uint8(pl) - } - - log.Printf("[net] HNSEndpointRequest :%+v", hnsEndpoint) - - // Marshal the request. - buffer, err := json.Marshal(hnsEndpoint) - if err != nil { - return nil, err - } - hnsRequest := string(buffer) - - // Create the HNS endpoint. - log.Printf("[net] HNSEndpointRequest POST request:%+v", hnsRequest) - hnsResponse, err := hcsshim.HNSEndpointRequest("POST", "", hnsRequest) - log.Printf("[net] HNSEndpointRequest POST response:%+v err:%v.", hnsResponse, err) - if err != nil { - return nil, err - } - - defer func() { - if err != nil { - log.Printf("[net] HNSEndpointRequest DELETE id:%v", hnsResponse.Id) - hnsResponse, err := hcsshim.HNSEndpointRequest("DELETE", hnsResponse.Id, "") - log.Printf("[net] HNSEndpointRequest DELETE response:%+v err:%v.", hnsResponse, err) - } - }() - - // Attach the endpoint. - log.Printf("[net] Attaching endpoint %v host.", hnsResponse.Id) - err = hnsResponse.HostAttach(1) - if err != nil { - log.Printf("[net] Failed to attach endpoint to default compartment due to err %v.", err) - return nil, err - } else { - log.Printf("[net] Succeeded attaching %v to host.", hnsResponse.Id) - } - - return nil, nil -} - // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { hcnEndpoint, err := nw.configureHcnEndpoint(epInfo) @@ -721,8 +623,7 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) gateway = net.ParseIP(hnsResponse.Routes[0].NextHop) } - nw.createTempEp(epInfo) - //nw.createGwEpv1(epInfo) + nw.createApipaEndpoint(epInfo) // Create the endpoint object. ep := &endpoint{ diff --git a/network/network_windows.go b/network/network_windows.go index ccf3af0644..8d0a636dc9 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -214,6 +214,19 @@ func (nm *networkManager) configureHcnNetwork(nwInfo *NetworkInfo, extIf *extern // newNetworkImplHnsV2 creates a new container network for HNSv2. func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { + // Do this only if there hostToCont / ContToHost is set + // if hostToCont / ContToHost + { + apipaNw, err := nm.createTempNw() + if err != nil { + err := fmt.Errorf("Failed to create APIPA bridge network for host to container connectivity due to error: %v", err) + log.Errorf("[net] %s", err.Error()) + return nil, err + } + + log.Printf("[net] Successfully setup APIPA bridge network for host to container connectivity: %+v", apipaNw) + } + hcnNetwork, err := nm.configureHcnNetwork(nwInfo, extIf) if err != nil { log.Printf("[net] Failed to configure hcn network due to error: %v", err) @@ -237,8 +250,6 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern vlanid = (int)(vlanID) } - nm.createTempNw(nwInfo, extIf) - // Create the network object. nw := &network{ Id: nwInfo.Id, @@ -255,14 +266,10 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern } // configureHcnEndpoint configures hcn endpoint for creation -func (nm *networkManager) configureTempHcnNetwork(nwInfo *NetworkInfo, extIf *externalInterface) (*hcn.HostComputeNetwork, error) { +func (nm *networkManager) configureTempHcnNetwork() (*hcn.HostComputeNetwork, error) { // Initialize HNS network. hcnNetwork := &hcn.HostComputeNetwork{ Name: "secondary-nw", - Dns: hcn.Dns{ - Domain: nwInfo.DNS.Suffix, - ServerList: nwInfo.DNS.Servers, - }, Ipams: []hcn.Ipam{ hcn.Ipam{ Type: hcnIpamTypeStatic, @@ -291,30 +298,6 @@ func (nm *networkManager) configureTempHcnNetwork(nwInfo *NetworkInfo, extIf *ex vlanid = 0 var subnetPolicy []byte - /* - opt, _ := nwInfo.Options[genericData].(map[string]interface{}) - if opt != nil && opt[VlanIDKey] != nil { - var err error - vlanID, _ := strconv.ParseUint(opt[VlanIDKey].(string), 10, 32) - subnetPolicy, err = policy.SerializeHcnSubnetVlanPolicy((uint32)(vlanID)) - if err != nil { - log.Printf("[net] Failed to serialize subnet vlan policy due to error: %v", err) - return nil, err - } - - vlanid = (int)(vlanID) - } - */ - - // Set network mode. - switch nwInfo.Mode { - case opModeBridge: - hcnNetwork.Type = hcn.L2Bridge - case opModeTunnel: - hcnNetwork.Type = hcn.L2Tunnel - default: - return nil, errNetworkModeInvalid - } hcnNetwork.Type = hcn.L2Bridge @@ -342,7 +325,7 @@ func (nm *networkManager) configureTempHcnNetwork(nwInfo *NetworkInfo, extIf *ex } // createTempNw creates a new container network for HNSv2. -func (nm *networkManager) createTempNw(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { +func (nm *networkManager) createTempNw() (*hcn.HostComputeNetwork, error) { log.Printf("[net] ashvind -- creating temp network...") var hcnNetwork *hcn.HostComputeNetwork var err error @@ -351,7 +334,7 @@ func (nm *networkManager) createTempNw(nwInfo *NetworkInfo, extIf *externalInter return nil, nil } - hcnNetwork, err = nm.configureTempHcnNetwork(nwInfo, extIf) + hcnNetwork, err = nm.configureTempHcnNetwork() if err != nil { log.Printf("[net] Failed to configure hcn network due to error: %v", err) return nil, err @@ -368,19 +351,7 @@ func (nm *networkManager) createTempNw(nwInfo *NetworkInfo, extIf *externalInter log.Printf("[net] Successfully created temp hcn network with response: %+v", hnsResponse) - // Create the network object. - nw := &network{ - Id: nwInfo.Id, - HnsId: hnsResponse.Id, - Mode: nwInfo.Mode, - Endpoints: make(map[string]*endpoint), - extIf: extIf, - VlanId: 0, - EnableSnatOnHost: nwInfo.EnableSnatOnHost, - NetNs: nwInfo.NetNs, - } - - return nw, nil + return hnsResponse, nil } // NewNetworkImpl creates a new container network. From 7c4236b381b5dbff3aee363e38140efc5e4d1921 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Wed, 18 Sep 2019 15:17:04 -0700 Subject: [PATCH 17/37] WIP3 --- network/network_windows.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/network/network_windows.go b/network/network_windows.go index 8d0a636dc9..6a79fdefdc 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -217,7 +217,7 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern // Do this only if there hostToCont / ContToHost is set // if hostToCont / ContToHost { - apipaNw, err := nm.createTempNw() + apipaNw, err := nm.createApipaNw() if err != nil { err := fmt.Errorf("Failed to create APIPA bridge network for host to container connectivity due to error: %v", err) log.Errorf("[net] %s", err.Error()) @@ -266,7 +266,7 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern } // configureHcnEndpoint configures hcn endpoint for creation -func (nm *networkManager) configureTempHcnNetwork() (*hcn.HostComputeNetwork, error) { +func (nm *networkManager) configureApipaNetwork() (*hcn.HostComputeNetwork, error) { // Initialize HNS network. hcnNetwork := &hcn.HostComputeNetwork{ Name: "secondary-nw", @@ -281,9 +281,7 @@ func (nm *networkManager) configureTempHcnNetwork() (*hcn.HostComputeNetwork, er }, } - // Set hcn network adaptor name policy - // FixMe: Find a better way to check if a nic that is selected is not part of a vSwitch - //if !strings.HasPrefix(extIf.Name, "vEthernet") { + // TODO: How to get this string from the created loopback adapter? netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy("Ethernet 6") if err != nil { log.Printf("[net] Failed to serialize network adapter policy due to error: %v", err) @@ -291,7 +289,6 @@ func (nm *networkManager) configureTempHcnNetwork() (*hcn.HostComputeNetwork, er } hcnNetwork.Policies = append(hcnNetwork.Policies, netAdapterNamePolicy) - //} // Set hcn subnet policy var vlanid int @@ -304,11 +301,9 @@ func (nm *networkManager) configureTempHcnNetwork() (*hcn.HostComputeNetwork, er // Populate subnets. hnsSubnet := hcn.Subnet{ IpAddressPrefix: "169.254.0.0/16", - //IpAddressPrefix: "172.21.9.0/24", Routes: []hcn.Route{ hcn.Route{ - NextHop: "169.254.0.2", - //NextHop: "172.21.9.1", + NextHop: "169.254.0.2", DestinationPrefix: "0.0.0.0/0", }, }, @@ -324,17 +319,16 @@ func (nm *networkManager) configureTempHcnNetwork() (*hcn.HostComputeNetwork, er return hcnNetwork, nil } -// createTempNw creates a new container network for HNSv2. -func (nm *networkManager) createTempNw() (*hcn.HostComputeNetwork, error) { - log.Printf("[net] ashvind -- creating temp network...") +// createApipaNw creates a new container network for HNSv2. +func (nm *networkManager) createApipaNw() (*hcn.HostComputeNetwork, error) { var hcnNetwork *hcn.HostComputeNetwork var err error if hcnNetwork, err = hcn.GetNetworkByName("secondary-nw"); err == nil { - log.Printf("[net] Found temp nw") + log.Printf("[net] Found existing APIPA network: %+v", hcnNetwork) return nil, nil } - hcnNetwork, err = nm.configureTempHcnNetwork() + hcnNetwork, err = nm.configureApipaNetwork() if err != nil { log.Printf("[net] Failed to configure hcn network due to error: %v", err) return nil, err From c28b37770a4ab141e92b4217350923d23ce7f238 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Thu, 19 Sep 2019 18:37:34 -0700 Subject: [PATCH 18/37] WIP 4 --- cni/network/multitenancy.go | 6 + cns/api.go | 44 ++- cns/cnsclient/cnsclient.go | 58 ++++ cns/hnsclient/hnsclient_windows.go | 458 ++++++++++++++++++++++++++++- cns/restserver/restserver.go | 59 ++++ network/endpoint_windows.go | 131 +++++++-- network/network_windows.go | 24 +- 7 files changed, 741 insertions(+), 39 deletions(-) diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 758135cfaf..2af4164da9 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -59,6 +59,7 @@ func getContainerNetworkConfiguration( return getContainerNetworkConfigurationInternal(nwCfg.CNSUrl, podNamespace, podNameWithoutSuffix, ifName) } +//TODO: there is no need for this internal function - maybe called from other place func getContainerNetworkConfigurationInternal( address string, namespace string, @@ -252,3 +253,8 @@ func CleanupMultitenancyResources(enableInfraVnet bool, nwCfg *cni.NetworkConfig cleanupInfraVnetIP(enableInfraVnet, &azIpamResult.IPs[0].Address, nwCfg, plugin) } } + +/* +func CreateApipaEndpoint2(podName, podNamespace string) error { + return nil +}*/ diff --git a/cns/api.go b/cns/api.go index 42ed300fa3..7bb3af547c 100644 --- a/cns/api.go +++ b/cns/api.go @@ -3,7 +3,9 @@ package cns -import "encoding/json" +import ( + "encoding/json" +) // Container Network Service remote API Contract const ( @@ -19,8 +21,13 @@ const ( GetUnhealthyIPAddressesPath = "/network/ipaddresses/unhealthy" GetHealthReportPath = "/network/health" NumberOfCPUCoresPath = "/hostcpucores" + CreateApipaEndpointPath = "/network/createapipaendpoint" + DeleteApipaEndpointPath = "/network/deleteapipaendpoint" V1Prefix = "/v0.1" V2Prefix = "/v0.2" + + OptOrchContext = "OrchestratorContext" + OptNCID = "NCID" ) // SetEnvironmentRequest describes the Request to set the environment in CNS. @@ -153,3 +160,38 @@ type OptionMap map[string]interface{} type errorResponse struct { Err string } + +// CreateApipaEndpointRequest describes request for create apipa endpoint. +type CreateApipaEndpointRequest struct { + //OptionsNCIdentifier map[string]interface{} + NetworkContainerid string + OrchestratorContext json.RawMessage +} + +// CreateApipaEndpointResponse describes response for create apipa endpoint request. +type CreateApipaEndpointResponse struct { + Response Response + ID string /* + ContainerID string + NetNsPath string + IfName string + SandboxKey string + IfIndex int + MacAddress net.HardwareAddr + DNS DNSInfo + IPAddresses []net.IPNet + InfraVnetIP net.IPNet + Routes []RouteInfo + Policies []policy.Policy + Gateways []net.IP + EnableSnatOnHost bool + EnableInfraVnet bool + EnableMultiTenancy bool + AllowInboundFromHostToNC bool + AllowInboundFromNCToHost bool + PODName string + PODNameSpace string + Data map[string]interface{} + InfraVnetAddressSpace string + SkipHotAttachEp bool*/ +} diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index aef2749c6d..fd3cdf5642 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -77,3 +77,61 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(orchestratorContext []byte) return &resp, nil } + +// CreateApipaEndpoint creates an endpoint in APIPA network for host container connectivity. +func (cnsClient *CNSClient) CreateApipaEndpoint(podName, podNamespace string /*orchestratorContext []byte*/) (*cns.CreateApipaEndpointResponse, error) { + var body bytes.Buffer + + httpc := &http.Client{} + url := cnsClient.connectionURL + cns.CreateApipaEndpointPath + log.Printf("CreateApipaEndpoint url: %v", url) + + podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace} + orchestratorContext, err := json.Marshal(podInfo) + if err != nil { + log.Printf("Failed to marshall podInfo for orchestrator context due to error: %v", err) + return nil, err + } + + // What can be used here? + payload := &cns.CreateApipaEndpointRequest{ + OrchestratorContext: orchestratorContext, + } + + err = json.NewEncoder(&body).Encode(payload) + if err != nil { + log.Errorf("encoding json failed with %v", err) + return nil, err + } + + res, err := httpc.Post(url, "application/json", &body) + if err != nil { + log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return nil, err + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("[Azure CNSClient] CreateEndpointForHostContainerConnectivity: Invalid http status code: %v", + res.StatusCode) + log.Errorf(errMsg) + return nil, fmt.Errorf(errMsg) + } + + var resp cns.CreateApipaEndpointResponse + + err = json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Errorf("[Azure CNSClient] Error parsing CreateEndpointForHostContainerConnectivity response resp: %v err: %v", + res.Body, err.Error()) + return nil, err + } + + if resp.Response.ReturnCode != 0 { + log.Errorf("[Azure CNSClient] CreateEndpointForHostContainerConnectivity received error response :%v", resp.Response.Message) + return nil, fmt.Errorf(resp.Response.Message) + } + + return &resp, nil +} diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 249807efda..f2e0f9aa94 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -3,11 +3,15 @@ package hnsclient import ( "encoding/json" "fmt" - "log" + "net" + "strconv" "strings" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" + "github.com/Microsoft/hcsshim/hcn" ) const ( @@ -23,6 +27,21 @@ const ( // HNS network types hnsL2Bridge = "l2bridge" hnsL2Tunnel = "l2tunnel" + + // hcnSchemaVersionMajor indicates major version number for hcn schema + hcnSchemaVersionMajor = 2 + + // hcnSchemaVersionMinor indicates minor version number for hcn schema + hcnSchemaVersionMinor = 0 + + // hcnIpamTypeStatic indicates the static type of ipam + hcnIpamTypeStatic = "Static" + + // apipaNetworkName indicates the name of the apipa network used for host container connectivity + apipaNetworkName = "secondary-nw" + + // apipaEndpointName indicates the name of the apipa endpoint used for host container connectivity + apipaEndpointName = "apipaEndpointHostNCConnectivity" ) // CreateHnsNetwork creates the HNS network with the provided configuration @@ -153,3 +172,440 @@ func deleteHnsNetwork(networkName string) error { return err } + +func configureApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeNetwork, error) { + apipaNetwork := &hcn.HostComputeNetwork{ + Name: apipaNetworkName, + Ipams: []hcn.Ipam{ + hcn.Ipam{ + Type: hcnIpamTypeStatic, + }, + }, + SchemaVersion: hcn.SchemaVersion{ + Major: hcnSchemaVersionMajor, + Minor: hcnSchemaVersionMinor, + }, + Type: hcn.L2Bridge, + } + + // TODO: How to get this string from the created loopback adapter? + // TODO: Create the loopback adapter using the LocalIPConfiguration passed in. + if netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy("Ethernet 6"); err == nil { + apipaNetwork.Policies = append(apipaNetwork.Policies, netAdapterNamePolicy) + } else { + log.Errorf("[Azure CNS] Failed to serialize network adapter policy due to error: %v", err) + return nil, err + } + + // Calculate subnet prefix + var subnetPrefix net.IPNet + var subnetPrefixStr string + ipAddr := net.ParseIP(localIPConfiguration.IPSubnet.IPAddress) + if ipAddr.To4() != nil { + subnetPrefix = net.IPNet{Mask: net.CIDRMask(int(localIPConfiguration.IPSubnet.PrefixLength), 32)} + } else if ipAddr.To16() != nil { + subnetPrefix = net.IPNet{Mask: net.CIDRMask(int(localIPConfiguration.IPSubnet.PrefixLength), 128)} + } else { + return nil, fmt.Errorf("[Azure CNS] Failed get subnet prefix for localIPConfiguration: %+v", localIPConfiguration) + } + + subnetPrefix.IP = ipAddr.Mask(subnetPrefix.Mask) + subnetPrefixStr = subnetPrefix.IP.String() + "/" + strconv.Itoa(int(localIPConfiguration.IPSubnet.PrefixLength)) + log.Printf("[tempdebug] configureApipaNetwork: subnetPrefixStr: %s, GW: %s", subnetPrefixStr, localIPConfiguration.GatewayIPAddress) + + subnet := hcn.Subnet{ + //IpAddressPrefix: "169.254.0.0/16", // TODO: this needs be calculated from LocalIPConfiguration passed in + IpAddressPrefix: subnetPrefixStr, + Routes: []hcn.Route{ + hcn.Route{ + NextHop: localIPConfiguration.GatewayIPAddress, + DestinationPrefix: "0.0.0.0/0", + }, + }, + } + + apipaNetwork.Ipams[0].Subnets = append(apipaNetwork.Ipams[0].Subnets, subnet) + + return apipaNetwork, nil +} + +func createApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeNetwork, error) { + var ( + apipaNetwork *hcn.HostComputeNetwork + err error + ) + + // Check if the APIPA network exists + if apipaNetwork, err := hcn.GetNetworkByName(apipaNetworkName); err != nil { + // If error is anything other than networkNotFound, mark this as error + if _, networkNotFound := err.(hcsshim.NetworkNotFoundError); !networkNotFound { + return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed due to error with GetNetworkByName: %v", err) + } + + // APIPA network doesn't exist. Create one. + if apipaNetwork, err = configureApipaNetwork(localIPConfiguration); err != nil { + log.Printf("[Azure CNS] Failed to configure apipa network due to error: %v", err) + return nil, err + } + + // Create the HNS network. + log.Printf("[net] Creating apipa network: %+v", apipaNetwork) + apipaNetwork, err = apipaNetwork.Create() + + if err != nil { + log.Printf("[net] Failed to create apipa network due to error: %v", err) + return nil, fmt.Errorf("Failed to create apipa network: %s due to error: %v", apipaNetwork.Name, err) + } + + log.Printf("[net] Successfully created apipa network for host container connectivity: %+v", apipaNetwork) + } else { + log.Printf("[Azure CNS] Found existing APIPA network: %+v", apipaNetwork) + } + + return apipaNetwork, err +} + +func configureApipaEndpoint( + apipaNetwork *hcn.HostComputeNetwork, + localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeEndpoint, error) { + apipaEndpoint := &hcn.HostComputeEndpoint{ + Name: apipaEndpointName, + HostComputeNetwork: apipaNetwork.Id, + SchemaVersion: hcn.SchemaVersion{ + Major: hcnSchemaVersionMajor, + Minor: hcnSchemaVersionMinor, + }, + } + + localIP := localIPConfiguration.IPSubnet.IPAddress + remoteIP := localIPConfiguration.GatewayIPAddress + log.Printf("[tempdebug] configureApipaEndpoint localIP: %s, remoteIP: %s", localIP, remoteIP) + /********************************************************************************************************/ + // Add ICMP ACLs + { + // Add endpoint ACL for preventing the comm to other apipa + aclOutBlockAll := hcn.AclPolicySetting{ + Protocols: "1", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err := json.Marshal(aclOutBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing out to host apipa + aclOutAllowToHostOnly := hcn.AclPolicySetting{ + Protocols: "1", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RemoteAddresses: remoteIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(aclOutAllowToHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing in from host apipa + aclInBlockAll := hcn.AclPolicySetting{ + Protocols: "1", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err = json.Marshal(aclInBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing in from host apipa + aclInAllowFromHostOnly := hcn.AclPolicySetting{ + Protocols: "1", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RemoteAddresses: remoteIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(aclInAllowFromHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + } + + // Add TCP ACLs + { + // Add endpoint ACL for preventing the comm to other apipa + aclOutBlockAll := hcn.AclPolicySetting{ + Protocols: "6", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err := json.Marshal(aclOutBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing out to host apipa + aclOutAllowToHostOnly := hcn.AclPolicySetting{ + Protocols: "6", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RemoteAddresses: remoteIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(aclOutAllowToHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing in from host apipa + aclInBlockAll := hcn.AclPolicySetting{ + Protocols: "6", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err = json.Marshal(aclInBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing in from host apipa + aclInAllowFromHostOnly := hcn.AclPolicySetting{ + Protocols: "6", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RemoteAddresses: remoteIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(aclInAllowFromHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + } + + // Add UDP ACLs + { + // Add endpoint ACL for preventing the comm to other apipa + aclOutBlockAll := hcn.AclPolicySetting{ + Protocols: "17", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err := json.Marshal(aclOutBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing out to host apipa + aclOutAllowToHostOnly := hcn.AclPolicySetting{ + Protocols: "17", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: localIP, + RemoteAddresses: remoteIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(aclOutAllowToHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing in from host apipa + aclInBlockAll := hcn.AclPolicySetting{ + Protocols: "17", + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + rawJSON, err = json.Marshal(aclInBlockAll) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + + // Add endpoint ACL for allowing in from host apipa + aclInAllowFromHostOnly := hcn.AclPolicySetting{ + Protocols: "17", + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: localIP, + RemoteAddresses: remoteIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + rawJSON, err = json.Marshal(aclInAllowFromHostOnly) + if err != nil { + return nil, fmt.Errorf("Failed to marshal the endpoint ACL") + } + + endpointPolicy = hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + } + /********************************************************************************************************/ + + nexthop := remoteIP + hcnRoute := hcn.Route{ + NextHop: nexthop, + DestinationPrefix: "0.0.0.0/0", + } + + apipaEndpoint.Routes = append(apipaEndpoint.Routes, hcnRoute) + + ipConfiguration := hcn.IpConfig{ + IpAddress: localIP, + PrefixLength: localIPConfiguration.IPSubnet.PrefixLength, // TODO: this should come from the cns config + } + + apipaEndpoint.IpConfigurations = append(apipaEndpoint.IpConfigurations, ipConfiguration) + + return apipaEndpoint, nil +} + +// CreateApipaEndpoint creates the endpoint in the apipa network for host container connectivity +func CreateApipaEndpoint(localIPConfiguration cns.IPConfiguration) (string, error) { + var ( + apipaNetwork *hcn.HostComputeNetwork + apipaEndpoint *hcn.HostComputeEndpoint + err error + ) + + if apipaNetwork, err = createApipaNetwork(localIPConfiguration); err != nil { + log.Errorf("[Azure CNS] Failed to create apipa network for host container connectivity due to error: %v", err) + return "", err + } + + if apipaEndpoint, err = configureApipaEndpoint(apipaNetwork, localIPConfiguration); err != nil { + log.Errorf("[Azure CNS] Failed to configure apipa endpoint for host container connectivity due to error: %v", err) + return "", err + } + + // Create the apipa endpoint + log.Printf("[Azure CNS] Creating apipa endpoint for host-container connectivity: %+v", apipaEndpoint) + if apipaEndpoint, err = apipaEndpoint.Create(); err != nil { + err = fmt.Errorf("Failed to create apipa endpoint: %s due to error: %v", apipaEndpoint.Name, err) + log.Errorf("[Azure CNS] %s", err.Error()) + return "", err + } + + log.Printf("[Azure CNS] Successfully created apipa endpoint for host-container connectivity: %+v", apipaEndpoint) + + return apipaEndpoint.Id, nil +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 9ee6894a09..d4de65a91f 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -180,6 +180,8 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.V2Prefix+cns.CreateHnsNetworkPath, service.createHnsNetwork) listener.AddHandler(cns.V2Prefix+cns.DeleteHnsNetworkPath, service.deleteHnsNetwork) listener.AddHandler(cns.V2Prefix+cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores) + listener.AddHandler(cns.V2Prefix+cns.CreateApipaEndpointPath, service.createApipaEndpoint) + //listener.AddHandler(cns.V2Prefix+cns.deleteapipaendpointpath, service.deleteApipaEndpoint) log.Printf("[Azure CNS] Listening.") return nil @@ -1615,3 +1617,60 @@ func (service *HTTPRestService) getNumberOfCPUCores(w http.ResponseWriter, r *ht log.Response(service.Name, numOfCPUCoresResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } + +func (service *HTTPRestService) createApipaEndpoint(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure-CNS] createApipaEndpoint") + + var ( + returnCode int + err error + returnMessage string + req cns.CreateApipaEndpointRequest + endpointID string + ) + + err = service.Listener.Decode(w, r, &req) + log.Request(service.Name, &req, err) + if err != nil { + return + } + + switch r.Method { + case "POST": + // Get the NC goal state from the NC identifier passed in request + /* + if req.OptionsNCIdentifier != nil { + if _, ok := req.OptionsNCIdentifier[OptOrchContext]; ok { + enableSnat = false + } + } + */ + + var req2 cns.GetNetworkContainerRequest + req2.NetworkContainerid = req.NetworkContainerid + req2.OrchestratorContext = req.OrchestratorContext + networkContainerGoalState := service.getNetworkContainerResponse(req2) + log.Printf("[tempdebug] restServer: networkContainerGoalState: %+v", networkContainerGoalState) + if endpointID, err = hnsclient.CreateApipaEndpoint(networkContainerGoalState.LocalIPConfiguration); err != nil { + returnMessage = fmt.Sprintf("createApipaEndpoint failed with error: %v", err) + returnCode = UnexpectedError + } + default: + returnMessage = "createApipaEndpoint API expects a POST" + returnCode = UnsupportedVerb + } + + resp := cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + } + + createApipaEndpointResp := cns.CreateApipaEndpointResponse{ + Response: resp, + ID: endpointID, + } + log.Printf("[tempdebug] createApipaEndpointResp: %+v", createApipaEndpointResp) + + err = service.Listener.Encode(w, &createApipaEndpointResp) + log.Response(service.Name, createApipaEndpointResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) +} diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 2d25e09a9b..72cd83b02a 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -9,6 +9,7 @@ import ( "net" "strings" + "github.com/Azure/azure-container-networking/cns/cnsclient" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" @@ -534,51 +535,127 @@ func (nw *network) configureApipaEndpoint(epInfo *EndpointInfo) (*hcn.HostComput } // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 -func (nw *network) createApipaEndpoint(epInfo *EndpointInfo) (*endpoint, error) { - hcnEndpoint, err := nw.configureApipaEndpoint(epInfo) +func (nw *network) createApipaEndpoint(epInfo *EndpointInfo) error { + + // TODO: cnsclient shouldn't be here. Need to move and encap this somewhere else. + cnsClient, err := cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg if err != nil { - log.Printf("[net] Failed to configure hcn endpoint due to error: %v", err) - return nil, err + log.Errorf("Initializing CNS client error %v", err) + return err + } + + // TODO: need to safeguard against repeatitive calls to ADD + // TODO: need to pass orch context to createApipaEnpoint - need to get it thru epInfo + //createApipaEndpoint2(epInfo.PODName, epInfo.PODNameSpace) + // TODO: need to add the following condition: + /* + if !nwCfg.EnableExactMatchForPodName { + podNameWithoutSuffix = network.GetPodNameWithoutSuffix(podName) + } else { + podNameWithoutSuffix = podName + } + */ + resp, err := cnsClient.CreateApipaEndpoint(epInfo.PODName, epInfo.PODNameSpace) + if err != nil { + return fmt.Errorf("Failed to create apipa endpoint due to error: %v", err) } - // Create the HCN endpoint. - log.Printf("[net] Creating APIPA endpoint for host-container connectivity: %+v", hcnEndpoint) - hnsResponse, err := hcnEndpoint.Create() - if err != nil { - log.Printf("[net] failed to create APIPA endpoint due to error: %v", err) - return nil, fmt.Errorf("Failed to create APIPA endpoint: %s due to error: %v", hcnEndpoint.Name, err) + if resp.Response.ReturnCode != 0 { + return fmt.Errorf("Failed to create apipa endpoint due to error: %s", resp.Response.Message) } - log.Printf("[net] Successfully created APIPA endpoint for host-container connectivity: %+v", hnsResponse) + var namespace *hcn.HostComputeNamespace + if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { + return fmt.Errorf("Failed to create apipa endpoint due to GetNamespaceByID NetNsPath: %s error: %v", epInfo.NetNsPath, err) + } - defer func() { + //nsId := namespace.Id + endpointID := resp.ID + if err = hcn.AddNamespaceEndpoint(namespace.Id, endpointID); err != nil { + return fmt.Errorf("[net] Failed to add apipa endpoint: %s to namespace: %s due to error: %v", + endpointID, namespace.Id, err) + } + + // TODO: If any of the following fails, delete the created apipa endpoint + + return nil + + // Create APIPA network if not present + /* + { + apipaNw, err := nm.createApipaNw() + if err != nil { + err := fmt.Errorf("Failed to create APIPA bridge network for host to container connectivity due to error: %v", err) + log.Errorf("[net] %s", err.Error()) + return nil, err + } + + log.Printf("[net] Successfully setup APIPA bridge network for host to container connectivity: %+v", apipaNw) + } + */ + + /* + hcnEndpoint, err := nw.configureApipaEndpoint(epInfo) if err != nil { - log.Printf("[net] Deleting hcn endpoint with id: %s", hnsResponse.Id) - err = hnsResponse.Delete() - log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", hnsResponse.Id, err) + log.Printf("[net] Failed to configure hcn endpoint due to error: %v", err) + return nil, err } - }() + + // Create the HCN endpoint. + log.Printf("[net] Creating APIPA endpoint for host-container connectivity: %+v", hcnEndpoint) + hnsResponse, err := hcnEndpoint.Create() + if err != nil { + log.Printf("[net] failed to create APIPA endpoint due to error: %v", err) + return nil, fmt.Errorf("Failed to create APIPA endpoint: %s due to error: %v", hcnEndpoint.Name, err) + } + + log.Printf("[net] Successfully created APIPA endpoint for host-container connectivity: %+v", hnsResponse) + + defer func() { + if err != nil { + log.Printf("[net] Deleting hcn endpoint with id: %s", hnsResponse.Id) + err = hnsResponse.Delete() + log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", hnsResponse.Id, err) + } + }() + */ //TODO: until this point, ep creation should be handled by CNS // after getting that ep, CNI will attach it to the container. - var namespace *hcn.HostComputeNamespace - if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { - return nil, fmt.Errorf("Failed to get hcn namespace: %s due to error: %v", epInfo.NetNsPath, err) - } + /* + var namespace *hcn.HostComputeNamespace + if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { + return nil, fmt.Errorf("Failed to get hcn namespace: %s due to error: %v", epInfo.NetNsPath, err) + } - nsId := namespace.Id + nsId := namespace.Id - if err = hcn.AddNamespaceEndpoint(nsId, hnsResponse.Id); err != nil { - return nil, fmt.Errorf("[net] Failed to add endpoint: %s to hcn namespace: %s due to error: %v", - hnsResponse.Id, namespace.Id, err) - } + if err = hcn.AddNamespaceEndpoint(nsId, hnsResponse.Id); err != nil { + return nil, fmt.Errorf("[net] Failed to add endpoint: %s to hcn namespace: %s due to error: %v", + hnsResponse.Id, namespace.Id, err) + } + */ - return nil, nil + /******************************************************************************************/ + + // return nil } // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { + // If the host to container connectivity is requested, create endpoint in APIPA + // bridge network to facilitate that + + //TODO: uncomment this in the new setup with the new DNC + //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { + { + if err := nw.createApipaEndpoint(epInfo); err != nil { + log.Errorf("[net] Failed to create APIPA endpoint due to error: %v", err) + return nil, err + } + } + hcnEndpoint, err := nw.configureHcnEndpoint(epInfo) if err != nil { log.Printf("[net] Failed to configure hcn endpoint due to error: %v", err) @@ -624,8 +701,6 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) gateway = net.ParseIP(hnsResponse.Routes[0].NextHop) } - nw.createApipaEndpoint(epInfo) - // Create the endpoint object. ep := &endpoint{ Id: hcnEndpoint.Name, diff --git a/network/network_windows.go b/network/network_windows.go index 6a79fdefdc..dbc1573884 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -216,16 +216,18 @@ func (nm *networkManager) configureHcnNetwork(nwInfo *NetworkInfo, extIf *extern func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { // Do this only if there hostToCont / ContToHost is set // if hostToCont / ContToHost - { - apipaNw, err := nm.createApipaNw() - if err != nil { - err := fmt.Errorf("Failed to create APIPA bridge network for host to container connectivity due to error: %v", err) - log.Errorf("[net] %s", err.Error()) - return nil, err + /* + { + apipaNw, err := nm.createApipaNw() + if err != nil { + err := fmt.Errorf("Failed to create APIPA bridge network for host to container connectivity due to error: %v", err) + log.Errorf("[net] %s", err.Error()) + return nil, err + } + + log.Printf("[net] Successfully setup APIPA bridge network for host to container connectivity: %+v", apipaNw) } - - log.Printf("[net] Successfully setup APIPA bridge network for host to container connectivity: %+v", apipaNw) - } + */ hcnNetwork, err := nm.configureHcnNetwork(nwInfo, extIf) if err != nil { @@ -265,6 +267,7 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern return nw, nil } +/* // configureHcnEndpoint configures hcn endpoint for creation func (nm *networkManager) configureApipaNetwork() (*hcn.HostComputeNetwork, error) { // Initialize HNS network. @@ -318,7 +321,9 @@ func (nm *networkManager) configureApipaNetwork() (*hcn.HostComputeNetwork, erro return hcnNetwork, nil } +*/ +/* // createApipaNw creates a new container network for HNSv2. func (nm *networkManager) createApipaNw() (*hcn.HostComputeNetwork, error) { var hcnNetwork *hcn.HostComputeNetwork @@ -347,6 +352,7 @@ func (nm *networkManager) createApipaNw() (*hcn.HostComputeNetwork, error) { return hnsResponse, nil } +*/ // NewNetworkImpl creates a new container network. func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { From 2ec3ee9982c958db29a558d2d03aa7c9ad2de19b Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Thu, 19 Sep 2019 23:16:47 -0700 Subject: [PATCH 19/37] WIP6 --- Makefile | 1 + cns/cnsclient/cnsclient.go | 1 + cns/hnsclient/hnsclient_windows.go | 16 ++++++++++------ cns/restserver/restserver.go | 1 + 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index af3b7b4a84..1117f097ac 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,7 @@ CNSFILES = \ $(wildcard cns/dockerclient/*.go) \ $(wildcard cns/imdsclient/*.go) \ $(wildcard cns/ipamclient/*.go) \ + $(wildcard cns/hnsclient/*.go) \ $(wildcard cns/restserver/*.go) \ $(wildcard cns/routes/*.go) \ $(wildcard cns/service/*.go) \ diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index fd3cdf5642..90f75bf231 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -104,6 +104,7 @@ func (cnsClient *CNSClient) CreateApipaEndpoint(podName, podNamespace string /*o return nil, err } + log.Printf("CreateApipaEndpoint posting body: %v", body) res, err := httpc.Post(url, "application/json", &body) if err != nil { log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index f2e0f9aa94..936355baa0 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -38,10 +38,10 @@ const ( hcnIpamTypeStatic = "Static" // apipaNetworkName indicates the name of the apipa network used for host container connectivity - apipaNetworkName = "secondary-nw" + apipaNetworkName = "apipa-network" // apipaEndpointName indicates the name of the apipa endpoint used for host container connectivity - apipaEndpointName = "apipaEndpointHostNCConnectivity" + apipaEndpointName = "apipa-endpoint" ) // CreateHnsNetwork creates the HNS network with the provided configuration @@ -236,11 +236,14 @@ func createApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostComp ) // Check if the APIPA network exists - if apipaNetwork, err := hcn.GetNetworkByName(apipaNetworkName); err != nil { + if apipaNetwork, err = hcn.GetNetworkByName(apipaNetworkName); err != nil { // If error is anything other than networkNotFound, mark this as error - if _, networkNotFound := err.(hcsshim.NetworkNotFoundError); !networkNotFound { - return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed due to error with GetNetworkByName: %v", err) - } + // TODO: why is following part not working? + /* + if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { + return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed due to error with GetNetworkByName: %v", err) + } + */ // APIPA network doesn't exist. Create one. if apipaNetwork, err = configureApipaNetwork(localIPConfiguration); err != nil { @@ -268,6 +271,7 @@ func createApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostComp func configureApipaEndpoint( apipaNetwork *hcn.HostComputeNetwork, localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeEndpoint, error) { + log.Printf("[tempdebug] configureApipaEndpoint ID: %+v", apipaNetwork) apipaEndpoint := &hcn.HostComputeEndpoint{ Name: apipaEndpointName, HostComputeNetwork: apipaNetwork.Id, diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index d4de65a91f..9c91f2dbe5 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -159,6 +159,7 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.CreateHnsNetworkPath, service.createHnsNetwork) listener.AddHandler(cns.DeleteHnsNetworkPath, service.deleteHnsNetwork) listener.AddHandler(cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores) + listener.AddHandler(cns.CreateApipaEndpointPath, service.createApipaEndpoint) // handlers for v0.2 listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment) From 6b1c3c07f7e7be5f5f4ce1afc653a7bea330a6c8 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Tue, 1 Oct 2019 18:16:17 -0700 Subject: [PATCH 20/37] WIP-10-01 --- cns/api.go | 34 +++------ cns/cnsclient/cnsclient.go | 52 ++++++++++++++ cns/hnsclient/hnsclient_windows.go | 35 +++++++++ cns/restserver/restserver.go | 64 ++++++++++++++++- network/endpoint_windows.go | 109 +++++++++-------------------- 5 files changed, 191 insertions(+), 103 deletions(-) diff --git a/cns/api.go b/cns/api.go index 7bb3af547c..f2f7f3682e 100644 --- a/cns/api.go +++ b/cns/api.go @@ -170,28 +170,16 @@ type CreateApipaEndpointRequest struct { // CreateApipaEndpointResponse describes response for create apipa endpoint request. type CreateApipaEndpointResponse struct { + Response Response + EndpointID string +} + +// DeleteApipaEndpointRequest describes request for deleting apipa endpoint. +type DeleteApipaEndpointRequest struct { + EndpointID string +} + +// DeleteApipaEndpointResponse describes response for delete apipa endpoint request. +type DeleteApipaEndpointResponse struct { Response Response - ID string /* - ContainerID string - NetNsPath string - IfName string - SandboxKey string - IfIndex int - MacAddress net.HardwareAddr - DNS DNSInfo - IPAddresses []net.IPNet - InfraVnetIP net.IPNet - Routes []RouteInfo - Policies []policy.Policy - Gateways []net.IP - EnableSnatOnHost bool - EnableInfraVnet bool - EnableMultiTenancy bool - AllowInboundFromHostToNC bool - AllowInboundFromNCToHost bool - PODName string - PODNameSpace string - Data map[string]interface{} - InfraVnetAddressSpace string - SkipHotAttachEp bool*/ } diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index 90f75bf231..365c7991bd 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -136,3 +136,55 @@ func (cnsClient *CNSClient) CreateApipaEndpoint(podName, podNamespace string /*o return &resp, nil } + +// DeleteApipaEndpoint deletes the endpoint in APIPA network created for host container connectivity. +func (cnsClient *CNSClient) DeleteApipaEndpoint(endpointID string) error { + var body bytes.Buffer + + // TODO: Move this to create a reusable http client. + httpc := &http.Client{} + url := cnsClient.connectionURL + cns.DeleteApipaEndpointPath + log.Printf("DeleteApipaEndpoint url: %v", url) + + payload := &cns.DeleteApipaEndpointRequest{ + EndpointID: endpointID, + } + + err := json.NewEncoder(&body).Encode(payload) + if err != nil { + log.Errorf("encoding json failed with %v", err) + return err + } + + log.Printf("DeleteApipaEndpoint posting body: %v", body) + res, err := httpc.Post(url, "application/json", &body) + if err != nil { + log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return err + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("[Azure CNSClient] DeleteApipaEndpoint: Invalid http status code: %v", + res.StatusCode) + log.Errorf(errMsg) + return fmt.Errorf(errMsg) + } + + var resp cns.DeleteApipaEndpointResponse + + err = json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Errorf("[Azure CNSClient] Error parsing DeleteApipaEndpoint response resp: %v err: %v", + res.Body, err.Error()) + return err + } + + if resp.Response.ReturnCode != 0 { + log.Errorf("[Azure CNSClient] DeleteApipaEndpoint received error response :%v", resp.Response.Message) + return fmt.Errorf(resp.Response.Message) + } + + return nil +} diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 936355baa0..39f042474e 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -583,6 +583,7 @@ func configureApipaEndpoint( return apipaEndpoint, nil } +//TODO: lock // CreateApipaEndpoint creates the endpoint in the apipa network for host container connectivity func CreateApipaEndpoint(localIPConfiguration cns.IPConfiguration) (string, error) { var ( @@ -591,6 +592,7 @@ func CreateApipaEndpoint(localIPConfiguration cns.IPConfiguration) (string, erro err error ) + //TODO: check if the endpoint exists if apipaNetwork, err = createApipaNetwork(localIPConfiguration); err != nil { log.Errorf("[Azure CNS] Failed to create apipa network for host container connectivity due to error: %v", err) return "", err @@ -613,3 +615,36 @@ func CreateApipaEndpoint(localIPConfiguration cns.IPConfiguration) (string, erro return apipaEndpoint.Id, nil } + +//TODO: lock +// DeleteApipaEndpoint deletes the endpoint in the apipa network created for host <-> container connectivity +func DeleteApipaEndpoint(endpointID string) error { + var ( + //apipaNetwork *hcn.HostComputeNetwork + apipaEndpoint *hcn.HostComputeEndpoint + err error + ) + + // Check if the endpoint with the provided ID exists + if apipaEndpoint, err = hcn.GetEndpointByID(apipaNetworkName); err != nil { + // If error is anything other than EndpointNotFoundError, return error. + // else log the error but don't return error because endpoint is already deleted. + if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { + return fmt.Errorf("[Azure CNS] ERROR: DeleteApipaEndpoint failed due to "+ + "error with GetEndpointByName: %v", err) + } + + log.Errorf("[Azure CNS] Failed to find endpoint: %s for deletion", endpointID) + return nil + } + + if err = apipaEndpoint.Delete(); err != nil { + err = fmt.Errorf("Failed to delete endpoint: %+v due to error: %v", apipaEndpoint, err) + log.Errorf("[Azure CNS] %v", err) + return err + } + + // TODO: Delete APIPA network if it doesn't have any endpoints + + return nil +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 9c91f2dbe5..4b24ba9ecc 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -160,6 +160,7 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.DeleteHnsNetworkPath, service.deleteHnsNetwork) listener.AddHandler(cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores) listener.AddHandler(cns.CreateApipaEndpointPath, service.createApipaEndpoint) + listener.AddHandler(cns.DeleteApipaEndpointPath, service.deleteApipaEndpoint) // handlers for v0.2 listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment) @@ -182,7 +183,7 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.V2Prefix+cns.DeleteHnsNetworkPath, service.deleteHnsNetwork) listener.AddHandler(cns.V2Prefix+cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores) listener.AddHandler(cns.V2Prefix+cns.CreateApipaEndpointPath, service.createApipaEndpoint) - //listener.AddHandler(cns.V2Prefix+cns.deleteapipaendpointpath, service.deleteApipaEndpoint) + listener.AddHandler(cns.V2Prefix+cns.DeleteApipaEndpointPath, service.deleteApipaEndpoint) log.Printf("[Azure CNS] Listening.") return nil @@ -1667,8 +1668,65 @@ func (service *HTTPRestService) createApipaEndpoint(w http.ResponseWriter, r *ht } createApipaEndpointResp := cns.CreateApipaEndpointResponse{ - Response: resp, - ID: endpointID, + Response: resp, + EndpointID: endpointID, + } + log.Printf("[tempdebug] createApipaEndpointResp: %+v", createApipaEndpointResp) + + err = service.Listener.Encode(w, &createApipaEndpointResp) + log.Response(service.Name, createApipaEndpointResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) +} + +func (service *HTTPRestService) deleteApipaEndpoint(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure-CNS] DeleteApipaEndpoint") + + var ( + returnCode int + err error + returnMessage string + req cns.DeleteApipaEndpointRequest + endpointID string + ) + + err = service.Listener.Decode(w, r, &req) + log.Request(service.Name, &req, err) + if err != nil { + return + } + + switch r.Method { + case "DELETE": + // Get the NC goal state from the NC identifier passed in request + /* + if req.OptionsNCIdentifier != nil { + if _, ok := req.OptionsNCIdentifier[OptOrchContext]; ok { + enableSnat = false + } + } + */ + + var req2 cns.GetNetworkContainerRequest + req2.NetworkContainerid = req.NetworkContainerid + req2.OrchestratorContext = req.OrchestratorContext + networkContainerGoalState := service.getNetworkContainerResponse(req2) + log.Printf("[tempdebug] restServer: networkContainerGoalState: %+v", networkContainerGoalState) + if endpointID, err = hnsclient.CreateApipaEndpoint(networkContainerGoalState.LocalIPConfiguration); err != nil { + returnMessage = fmt.Sprintf("createApipaEndpoint failed with error: %v", err) + returnCode = UnexpectedError + } + default: + returnMessage = "createApipaEndpoint API expects a POST" + returnCode = UnsupportedVerb + } + + resp := cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + } + + createApipaEndpointResp := cns.CreateApipaEndpointResponse{ + Response: resp, + EndpointID: endpointID, } log.Printf("[tempdebug] createApipaEndpointResp: %+v", createApipaEndpointResp) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 72cd83b02a..38db8cc3e8 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -555,6 +555,7 @@ func (nw *network) createApipaEndpoint(epInfo *EndpointInfo) error { podNameWithoutSuffix = podName } */ + resp, err := cnsClient.CreateApipaEndpoint(epInfo.PODName, epInfo.PODNameSpace) if err != nil { return fmt.Errorf("Failed to create apipa endpoint due to error: %v", err) @@ -564,98 +565,33 @@ func (nw *network) createApipaEndpoint(epInfo *EndpointInfo) error { return fmt.Errorf("Failed to create apipa endpoint due to error: %s", resp.Response.Message) } + endpointID := resp.ID + + // TODO: Add defer func to delete apipa endpoint + defer func() { + if err != nil { + cnsClient.CreateApipaEndpoint(endpointID) + //log.Printf("[net] Deleting hcn endpoint with id: %s", endpointID) + //err = hnsResponse.Delete() + //log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", hnsResponse.Id, err) + } + }() + var namespace *hcn.HostComputeNamespace if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { return fmt.Errorf("Failed to create apipa endpoint due to GetNamespaceByID NetNsPath: %s error: %v", epInfo.NetNsPath, err) } - //nsId := namespace.Id - endpointID := resp.ID if err = hcn.AddNamespaceEndpoint(namespace.Id, endpointID); err != nil { return fmt.Errorf("[net] Failed to add apipa endpoint: %s to namespace: %s due to error: %v", endpointID, namespace.Id, err) } - // TODO: If any of the following fails, delete the created apipa endpoint - return nil - - // Create APIPA network if not present - /* - { - apipaNw, err := nm.createApipaNw() - if err != nil { - err := fmt.Errorf("Failed to create APIPA bridge network for host to container connectivity due to error: %v", err) - log.Errorf("[net] %s", err.Error()) - return nil, err - } - - log.Printf("[net] Successfully setup APIPA bridge network for host to container connectivity: %+v", apipaNw) - } - */ - - /* - hcnEndpoint, err := nw.configureApipaEndpoint(epInfo) - if err != nil { - log.Printf("[net] Failed to configure hcn endpoint due to error: %v", err) - return nil, err - } - - // Create the HCN endpoint. - log.Printf("[net] Creating APIPA endpoint for host-container connectivity: %+v", hcnEndpoint) - hnsResponse, err := hcnEndpoint.Create() - if err != nil { - log.Printf("[net] failed to create APIPA endpoint due to error: %v", err) - return nil, fmt.Errorf("Failed to create APIPA endpoint: %s due to error: %v", hcnEndpoint.Name, err) - } - - log.Printf("[net] Successfully created APIPA endpoint for host-container connectivity: %+v", hnsResponse) - - defer func() { - if err != nil { - log.Printf("[net] Deleting hcn endpoint with id: %s", hnsResponse.Id) - err = hnsResponse.Delete() - log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", hnsResponse.Id, err) - } - }() - */ - - //TODO: until this point, ep creation should be handled by CNS - // after getting that ep, CNI will attach it to the container. - - /* - var namespace *hcn.HostComputeNamespace - if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { - return nil, fmt.Errorf("Failed to get hcn namespace: %s due to error: %v", epInfo.NetNsPath, err) - } - - nsId := namespace.Id - - if err = hcn.AddNamespaceEndpoint(nsId, hnsResponse.Id); err != nil { - return nil, fmt.Errorf("[net] Failed to add endpoint: %s to hcn namespace: %s due to error: %v", - hnsResponse.Id, namespace.Id, err) - } - */ - - /******************************************************************************************/ - - // return nil } // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { - // If the host to container connectivity is requested, create endpoint in APIPA - // bridge network to facilitate that - - //TODO: uncomment this in the new setup with the new DNC - //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { - { - if err := nw.createApipaEndpoint(epInfo); err != nil { - log.Errorf("[net] Failed to create APIPA endpoint due to error: %v", err) - return nil, err - } - } - hcnEndpoint, err := nw.configureHcnEndpoint(epInfo) if err != nil { log.Printf("[net] Failed to configure hcn endpoint due to error: %v", err) @@ -689,6 +625,25 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) hnsResponse.Id, namespace.Id, err) } + defer func() { + if err != nil { + if errRemoveNsEp := hcn.RemoveNamespaceEndpoint(namespace.Id, hnsResponse.Id); errRemoveNsEp != nil { + log.Printf("[net] Failed to remove endpoint: %s from namespace: %s due to error: %v", + hnsResponse.Id, hnsResponse.Id, errRemoveNsEp) + } + } + }() + + // If the host <-> container connectivity is requested, create endpoint in APIPA + // bridge network to facilitate that + //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { + { + if err = nw.createApipaEndpoint(epInfo); err != nil { + log.Errorf("[net] Failed to create APIPA endpoint due to error: %v", err) + return nil, err + } + } + var vlanid int if epInfo.Data != nil { if vlanData, ok := epInfo.Data[VlanIDKey]; ok { From afcc71cb315694858dfa1af79f51c43e45cf2c08 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Sat, 5 Oct 2019 15:12:57 -0700 Subject: [PATCH 21/37] apipa ep delete --- cns/hnsclient/hnsclient_windows.go | 52 +++++++++++++++++++++++-- cns/restserver/restserver.go | 34 +++++------------ network/endpoint.go | 15 +++++--- network/endpoint_windows.go | 61 ++++++++++++++++++++++-------- 4 files changed, 112 insertions(+), 50 deletions(-) diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 39f042474e..e65dfc9f96 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -618,15 +618,16 @@ func CreateApipaEndpoint(localIPConfiguration cns.IPConfiguration) (string, erro //TODO: lock // DeleteApipaEndpoint deletes the endpoint in the apipa network created for host <-> container connectivity +// Can this be generalized to createEndpoint / DeleteEndpoint - which can used by general CNI calls +// If you don't delete this APIPA network / if VM gets rebooted, how will you clean this upon restart? func DeleteApipaEndpoint(endpointID string) error { var ( - //apipaNetwork *hcn.HostComputeNetwork apipaEndpoint *hcn.HostComputeEndpoint err error ) // Check if the endpoint with the provided ID exists - if apipaEndpoint, err = hcn.GetEndpointByID(apipaNetworkName); err != nil { + if apipaEndpoint, err = hcn.GetEndpointByID(endpointID); err != nil { // If error is anything other than EndpointNotFoundError, return error. // else log the error but don't return error because endpoint is already deleted. if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { @@ -634,17 +635,60 @@ func DeleteApipaEndpoint(endpointID string) error { "error with GetEndpointByName: %v", err) } - log.Errorf("[Azure CNS] Failed to find endpoint: %s for deletion", endpointID) + log.Errorf("[Azure CNS] Failed to find endpoint: %s for deletion due to error: %v", endpointID, err) return nil } + //networkID := apipaEndpoint.HostComputeNetwork + if err = apipaEndpoint.Delete(); err != nil { err = fmt.Errorf("Failed to delete endpoint: %+v due to error: %v", apipaEndpoint, err) log.Errorf("[Azure CNS] %v", err) return err } - // TODO: Delete APIPA network if it doesn't have any endpoints + log.Debugf("[Azure CNS] Successfully deleted endpoint: %v", apipaNetworkName) + + var endpoints []hcn.HostComputeEndpoint + // Check if the network has any endpoints left + if endpoints, err = hcn.ListEndpointsOfNetwork(apipaEndpoint.HostComputeNetwork); err != nil { + log.Errorf("[Azure CNS] Failed to list endpoints in the network: %s due to error", apipaNetworkName, err) + return nil + } + + // Delete network if it doesn't have any endpoints + if len(endpoints) == 0 { + DeleteApipaNetwork(apipaEndpoint.HostComputeNetwork) + } + + return nil +} + +func DeleteApipaNetwork(networkID string) error { + var ( + network *hcn.HostComputeNetwork + err error + ) + + if network, err = hcn.GetNetworkByID(networkID); err != nil { + // If error is anything other than NetworkNotFoundError, return error. + // else log the error but don't return error because network is already deleted. + if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { + return fmt.Errorf("[Azure CNS] ERROR: DeleteApipaNetwork failed due to "+ + "error with GetNetworkByID: %v", err) + } + + log.Errorf("[Azure CNS] Failed to find network with ID: %s for deletion", networkID) + return nil + } + + if err = network.Delete(); err != nil { + err = fmt.Errorf("Failed to delete network: %+v due to error: %v", network, err) + log.Errorf("[Azure CNS] %v", err) + return err + } + + log.Errorf("[Azure CNS] Successfully deleted network: %+v", network) return nil } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 4b24ba9ecc..029875eba8 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -1685,7 +1685,6 @@ func (service *HTTPRestService) deleteApipaEndpoint(w http.ResponseWriter, r *ht err error returnMessage string req cns.DeleteApipaEndpointRequest - endpointID string ) err = service.Listener.Decode(w, r, &req) @@ -1695,27 +1694,13 @@ func (service *HTTPRestService) deleteApipaEndpoint(w http.ResponseWriter, r *ht } switch r.Method { - case "DELETE": - // Get the NC goal state from the NC identifier passed in request - /* - if req.OptionsNCIdentifier != nil { - if _, ok := req.OptionsNCIdentifier[OptOrchContext]; ok { - enableSnat = false - } - } - */ - - var req2 cns.GetNetworkContainerRequest - req2.NetworkContainerid = req.NetworkContainerid - req2.OrchestratorContext = req.OrchestratorContext - networkContainerGoalState := service.getNetworkContainerResponse(req2) - log.Printf("[tempdebug] restServer: networkContainerGoalState: %+v", networkContainerGoalState) - if endpointID, err = hnsclient.CreateApipaEndpoint(networkContainerGoalState.LocalIPConfiguration); err != nil { - returnMessage = fmt.Sprintf("createApipaEndpoint failed with error: %v", err) + case "POST": + if err = hnsclient.DeleteApipaEndpoint(req.EndpointID); err != nil { + returnMessage = fmt.Sprintf("Failed to delete endpoint: %s due to error: %v", req.EndpointID, err) returnCode = UnexpectedError } default: - returnMessage = "createApipaEndpoint API expects a POST" + returnMessage = "deleteApipaEndpoint API expects a DELETE" returnCode = UnsupportedVerb } @@ -1724,12 +1709,11 @@ func (service *HTTPRestService) deleteApipaEndpoint(w http.ResponseWriter, r *ht Message: returnMessage, } - createApipaEndpointResp := cns.CreateApipaEndpointResponse{ - Response: resp, - EndpointID: endpointID, + deleteApipaEndpointResp := cns.DeleteApipaEndpointResponse{ + Response: resp, } - log.Printf("[tempdebug] createApipaEndpointResp: %+v", createApipaEndpointResp) + log.Printf("[tempdebug] deleteApipaEndpointResp: %+v", deleteApipaEndpointResp) - err = service.Listener.Encode(w, &createApipaEndpointResp) - log.Response(service.Name, createApipaEndpointResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) + err = service.Listener.Encode(w, &deleteApipaEndpointResp) + log.Response(service.Name, deleteApipaEndpointResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } diff --git a/network/endpoint.go b/network/endpoint.go index fa1755d061..9e4b240b9a 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -35,6 +35,7 @@ type endpoint struct { EnableMultitenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool + TempApipaEpID string NetworkNameSpace string `json:",omitempty"` ContainerID string PODName string `json:",omitempty"` @@ -63,6 +64,7 @@ type EndpointInfo struct { EnableMultiTenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool + TempApipaEpID string PODName string PODNameSpace string Data map[string]interface{} @@ -100,6 +102,7 @@ func (nw *network) newEndpoint(epInfo *EndpointInfo) (*endpoint, error) { nw.Endpoints[epInfo.Id] = ep log.Printf("[net] Created endpoint %+v.", ep) + log.Printf("[net] Created endpoint2 %+v.", nw.Endpoints[epInfo.Id]) return ep, nil } @@ -117,6 +120,7 @@ func (nw *network) deleteEndpoint(endpointId string) error { // Look up the endpoint. ep, err := nw.getEndpoint(endpointId) + log.Printf("[net] tempdebug:EP %+v.", ep) if err != nil { log.Printf("[net] Endpoint %v not found. Not Returning error", endpointId) return nil @@ -202,11 +206,12 @@ func (ep *endpoint) getInfo() *EndpointInfo { EnableMultiTenancy: ep.EnableMultitenancy, AllowInboundFromHostToNC: ep.AllowInboundFromHostToNC, AllowInboundFromNCToHost: ep.AllowInboundFromNCToHost, - IfName: ep.IfName, - ContainerID: ep.ContainerID, - NetNsPath: ep.NetworkNameSpace, - PODName: ep.PODName, - PODNameSpace: ep.PODNameSpace, + IfName: ep.IfName, + ContainerID: ep.ContainerID, + NetNsPath: ep.NetworkNameSpace, + PODName: ep.PODName, + PODNameSpace: ep.PODNameSpace, + TempApipaEpID: ep.TempApipaEpID, } for _, route := range ep.Routes { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 38db8cc3e8..92c1fdb2c4 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -534,14 +534,30 @@ func (nw *network) configureApipaEndpoint(epInfo *EndpointInfo) (*hcn.HostComput return hcnEndpoint, nil } +func (nw *network) deleteApipaEndpoint(endpointID string) error { + // TODO: cnsclient shouldn't be here. Need to move and encap this somewhere else. + cnsClient, err := cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg + if err != nil { + log.Errorf("Initializing CNS client error %v", err) + return err + } + + log.Printf("[net] Deleting apipa hcn endpoint with id: %s", endpointID) + err = cnsClient.DeleteApipaEndpoint(endpointID) + log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", endpointID, err) + + return nil +} + // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 -func (nw *network) createApipaEndpoint(epInfo *EndpointInfo) error { +func (nw *network) createApipaEndpoint(epInfo *EndpointInfo) (string, error) { + var endpointID string // TODO: cnsclient shouldn't be here. Need to move and encap this somewhere else. cnsClient, err := cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg if err != nil { log.Errorf("Initializing CNS client error %v", err) - return err + return endpointID, err } // TODO: need to safeguard against repeatitive calls to ADD @@ -558,36 +574,35 @@ func (nw *network) createApipaEndpoint(epInfo *EndpointInfo) error { resp, err := cnsClient.CreateApipaEndpoint(epInfo.PODName, epInfo.PODNameSpace) if err != nil { - return fmt.Errorf("Failed to create apipa endpoint due to error: %v", err) + return endpointID, fmt.Errorf("Failed to create apipa endpoint due to error: %v", err) } if resp.Response.ReturnCode != 0 { - return fmt.Errorf("Failed to create apipa endpoint due to error: %s", resp.Response.Message) + return endpointID, fmt.Errorf("Failed to create apipa endpoint due to error: %s", resp.Response.Message) } - endpointID := resp.ID + endpointID = resp.EndpointID - // TODO: Add defer func to delete apipa endpoint defer func() { if err != nil { - cnsClient.CreateApipaEndpoint(endpointID) - //log.Printf("[net] Deleting hcn endpoint with id: %s", endpointID) - //err = hnsResponse.Delete() - //log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", hnsResponse.Id, err) + log.Printf("[net] Deleting hcn endpoint with id: %s", endpointID) + // TODO: when this becomes generic, localIP can be passed to delete the endpoint + err = cnsClient.DeleteApipaEndpoint(endpointID) + log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", endpointID, err) } }() var namespace *hcn.HostComputeNamespace if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { - return fmt.Errorf("Failed to create apipa endpoint due to GetNamespaceByID NetNsPath: %s error: %v", epInfo.NetNsPath, err) + return endpointID, fmt.Errorf("Failed to create apipa endpoint due to GetNamespaceByID NetNsPath: %s error: %v", epInfo.NetNsPath, err) } if err = hcn.AddNamespaceEndpoint(namespace.Id, endpointID); err != nil { - return fmt.Errorf("[net] Failed to add apipa endpoint: %s to namespace: %s due to error: %v", + return endpointID, fmt.Errorf("[net] Failed to add apipa endpoint: %s to namespace: %s due to error: %v", endpointID, namespace.Id, err) } - return nil + return endpointID, nil } // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 @@ -638,7 +653,7 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) // bridge network to facilitate that //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { { - if err = nw.createApipaEndpoint(epInfo); err != nil { + if epInfo.TempApipaEpID, err = nw.createApipaEndpoint(epInfo); err != nil { log.Errorf("[net] Failed to create APIPA endpoint due to error: %v", err) return nil, err } @@ -656,6 +671,8 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) gateway = net.ParseIP(hnsResponse.Routes[0].NextHop) } + log.Errorf("[net] tempdebug: tempApipaEP: %s", epInfo.TempApipaEpID) + // Create the endpoint object. ep := &endpoint{ Id: hcnEndpoint.Name, @@ -668,6 +685,7 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) VlanID: vlanid, EnableSnatOnHost: epInfo.EnableSnatOnHost, NetNs: epInfo.NetNsPath, + TempApipaEpID: epInfo.TempApipaEpID, } for _, route := range epInfo.Routes { @@ -703,8 +721,19 @@ func (nw *network) deleteEndpointImplHnsV1(ep *endpoint) error { // deleteEndpointImplHnsV2 deletes an existing endpoint from the network using HNS v2. func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { - var hcnEndpoint *hcn.HostComputeEndpoint - var err error + var ( + hcnEndpoint *hcn.HostComputeEndpoint + err error + ) + + log.Printf("[net] deleteEndpointImplHnsV2 DELETE id:%+v", ep) + //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { + { + if err = nw.deleteApipaEndpoint(ep.TempApipaEpID); err != nil { + log.Errorf("[net] Failed to delete APIPA endpoint due to error: %v", err) + return err + } + } log.Printf("[net] Deleting hcn endpoint with id: %s", ep.HnsId) From d62727002d5a29ffea67e4d350b2b027c83fe37f Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Sun, 6 Oct 2019 16:02:27 -0700 Subject: [PATCH 22/37] generic change --- cns/api.go | 83 +++++++++++++ cns/cnsclient/cnsclient.go | 182 +++++++++++++++++++++++++++++ cns/hnsclient/hnsclient_windows.go | 147 ++++++++++++++++++++++- cns/restserver/restserver.go | 108 +++++++++++++++++ network/endpoint.go | 2 + network/endpoint_windows.go | 134 ++++++++++++++++++++- network/models/models.go | 140 ++++++++++++++++++++++ 7 files changed, 790 insertions(+), 6 deletions(-) create mode 100644 network/models/models.go diff --git a/cns/api.go b/cns/api.go index f2f7f3682e..61196bd3e0 100644 --- a/cns/api.go +++ b/cns/api.go @@ -5,6 +5,8 @@ package cns import ( "encoding/json" + //"github.com/Azure/azure-container-networking/network" + models "github.com/Azure/azure-container-networking/network/models" ) // Container Network Service remote API Contract @@ -23,6 +25,10 @@ const ( NumberOfCPUCoresPath = "/hostcpucores" CreateApipaEndpointPath = "/network/createapipaendpoint" DeleteApipaEndpointPath = "/network/deleteapipaendpoint" + CreateNewNetworkPath = "/network/createnewnetwork" + DeleteNewNetworkPath = "/network/deletenewnetwork" + CreateNewEndpointPath = "/network/createnewendpoint" + DeleteNewEndpointPath = "/network/deletenewendpoint" V1Prefix = "/v0.1" V2Prefix = "/v0.2" @@ -30,6 +36,72 @@ const ( OptNCID = "NCID" ) +/* +// DNSInfo contains DNS information for a container network or endpoint. +type DNSInfo struct { + Suffix string + Servers []string + Options []string +} + +// NetworkInfo contains read-only information about a container network. +type NetworkInfo struct { + MasterIfName string + Id string + Mode string + Subnets []SubnetInfo + DNS DNSInfo + Policies []policy.Policy + BridgeName string + EnableSnatOnHost bool + NetNs string + Options map[string]interface{} +} + +// ExternalInterface is a host network interface that bridges containers to external networks. +type externalInterface struct { + Name string + Networks map[string]*network + Subnets []string + BridgeName string + DNSInfo DNSInfo + MacAddress net.HardwareAddr + IPAddresses []*net.IPNet + Routes []*route + IPv4Gateway net.IP + IPv6Gateway net.IP +} + +// EndpointInfo contains read-only information about an endpoint. +type EndpointInfo struct { + Id string + ContainerID string + NetNsPath string + IfName string + SandboxKey string + IfIndex int + MacAddress net.HardwareAddr + DNS DNSInfo + IPAddresses []net.IPNet + InfraVnetIP net.IPNet + Routes []RouteInfo + Policies []policy.Policy + Gateways []net.IP + EnableSnatOnHost bool + EnableInfraVnet bool + EnableMultiTenancy bool + AllowInboundFromHostToNC bool + AllowInboundFromNCToHost bool + TempApipaEpID string + PODName string + PODNameSpace string + Data map[string]interface{} + InfraVnetAddressSpace string + SkipHotAttachEp bool + NetworkID string +} +*/ + // SetEnvironmentRequest describes the Request to set the environment in CNS. type SetEnvironmentRequest struct { Location string @@ -161,6 +233,17 @@ type errorResponse struct { Err string } +// CreateNewNetworkRequest describes request to create new network. +type CreateNewNetworkRequest struct { + NetworkInfo models.NetworkInfo + ExternalInterface models.ExternalInterface +} + +// CreateNewEndpointRequest describes request to create new endpoint. +type CreateNewEndpointRequest struct { + EndpointInfo models.EndpointInfo +} + // CreateApipaEndpointRequest describes request for create apipa endpoint. type CreateApipaEndpointRequest struct { //OptionsNCIdentifier map[string]interface{} diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index 365c7991bd..c166d619bd 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -8,8 +8,76 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/log" + //"github.com/Azure/azure-container-networking/network" + models "github.com/Azure/azure-container-networking/network/models" ) +/* +// DNSInfo contains DNS information for a container network or endpoint. +type DNSInfo struct { + Suffix string + Servers []string + Options []string +} + +// NetworkInfo contains read-only information about a container network. +type NetworkInfo struct { + MasterIfName string + Id string + Mode string + Subnets []SubnetInfo + DNS DNSInfo + Policies []policy.Policy + BridgeName string + EnableSnatOnHost bool + NetNs string + Options map[string]interface{} +} + +// ExternalInterface is a host network interface that bridges containers to external networks. +type externalInterface struct { + Name string + Networks map[string]*network + Subnets []string + BridgeName string + DNSInfo DNSInfo + MacAddress net.HardwareAddr + IPAddresses []*net.IPNet + Routes []*route + IPv4Gateway net.IP + IPv6Gateway net.IP +} + +// EndpointInfo contains read-only information about an endpoint. +type EndpointInfo struct { + Id string + ContainerID string + NetNsPath string + IfName string + SandboxKey string + IfIndex int + MacAddress net.HardwareAddr + DNS DNSInfo + IPAddresses []net.IPNet + InfraVnetIP net.IPNet + Routes []RouteInfo + Policies []policy.Policy + Gateways []net.IP + EnableSnatOnHost bool + EnableInfraVnet bool + EnableMultiTenancy bool + AllowInboundFromHostToNC bool + AllowInboundFromNCToHost bool + TempApipaEpID string + PODName string + PODNameSpace string + Data map[string]interface{} + InfraVnetAddressSpace string + SkipHotAttachEp bool + NetworkID string +} +*/ + // CNSClient specifies a client to connect to Ipam Plugin. type CNSClient struct { connectionURL string @@ -188,3 +256,117 @@ func (cnsClient *CNSClient) DeleteApipaEndpoint(endpointID string) error { return nil } + +// CreateNetwork creates an network. +func (cnsClient *CNSClient) CreateNetwork( + networkInfo models.NetworkInfo, + extIf models.ExternalInterface) /*network.network, - this might need to be Network to be xported*/ error { + var ( + body bytes.Buffer + err error + ) + + httpc := &http.Client{} + url := cnsClient.connectionURL + cns.CreateNewNetworkPath + log.Printf("CreateNewNetworkPath url: %v", url) + + // What can be used here? + payload := &cns.CreateNewNetworkRequest{ + NetworkInfo: networkInfo, + ExternalInterface: extIf, + } + + err = json.NewEncoder(&body).Encode(payload) + if err != nil { + log.Errorf("encoding json failed with %v", err) + return err + } + + log.Printf("CreateNetwork posting body: %v", body) + res, err := httpc.Post(url, "application/json", &body) + if err != nil { + log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return err + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("[Azure CNSClient] CreateNetwork: Invalid http status code: %v", + res.StatusCode) + log.Errorf(errMsg) + return fmt.Errorf(errMsg) + } + + var resp cns.Response + + err = json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Errorf("[Azure CNSClient] Error parsing CreateNetwork response resp: %v err: %v", + res.Body, err.Error()) + return err + } + + if resp.ReturnCode != 0 { + log.Errorf("[Azure CNSClient] CreateNetwork received error response :%v", resp.Message) + return fmt.Errorf(resp.Message) + } + + return nil +} + +// CreateEndpoint creates an endpoint. +func (cnsClient *CNSClient) CreateEndpoint( + endpointInfo models.EndpointInfo) /*network.endpoint,*/ (*cns.CreateApipaEndpointResponse, error) { + var ( + body bytes.Buffer + err error + ) + + httpc := &http.Client{} + url := cnsClient.connectionURL + cns.CreateNewEndpointPath + log.Printf("CreateEndpoint url: %v", url) + + // What can be used here? + payload := &cns.CreateNewEndpointRequest{ + EndpointInfo: endpointInfo, + } + + err = json.NewEncoder(&body).Encode(payload) + if err != nil { + log.Errorf("encoding json failed with %v", err) + return nil, err + } + + log.Printf("CreateEndpoint posting body: %v", body) + res, err := httpc.Post(url, "application/json", &body) + if err != nil { + log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return nil, err + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("[Azure CNSClient] CreateEndpoint: Invalid http status code: %v", + res.StatusCode) + log.Errorf(errMsg) + return nil, fmt.Errorf(errMsg) + } + + var resp cns.CreateApipaEndpointResponse + + err = json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Errorf("[Azure CNSClient] Error parsing CreateEndpoint response resp: %v err: %v", + res.Body, err.Error()) + return nil, err + } + + if resp.Response.ReturnCode != 0 { + log.Errorf("[Azure CNSClient] CreateEndpoint received error response :%v", resp.Response.Message) + return nil, fmt.Errorf(resp.Response.Message) + } + + return &resp, nil +} diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index e65dfc9f96..ad004834e1 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/models" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" @@ -269,12 +270,12 @@ func createApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostComp } func configureApipaEndpoint( - apipaNetwork *hcn.HostComputeNetwork, + apipaNetworkID string, localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeEndpoint, error) { - log.Printf("[tempdebug] configureApipaEndpoint ID: %+v", apipaNetwork) + //log.Printf("[tempdebug] configureApipaEndpoint ID: %+v", apipaNetwork) apipaEndpoint := &hcn.HostComputeEndpoint{ Name: apipaEndpointName, - HostComputeNetwork: apipaNetwork.Id, + HostComputeNetwork: apipaNetworkID, SchemaVersion: hcn.SchemaVersion{ Major: hcnSchemaVersionMajor, Minor: hcnSchemaVersionMinor, @@ -598,7 +599,7 @@ func CreateApipaEndpoint(localIPConfiguration cns.IPConfiguration) (string, erro return "", err } - if apipaEndpoint, err = configureApipaEndpoint(apipaNetwork, localIPConfiguration); err != nil { + if apipaEndpoint, err = configureApipaEndpoint(apipaNetwork.Id, localIPConfiguration); err != nil { log.Errorf("[Azure CNS] Failed to configure apipa endpoint for host container connectivity due to error: %v", err) return "", err } @@ -692,3 +693,141 @@ func DeleteApipaNetwork(networkID string) error { return nil } + +func CreateNewNetwork( + networkInfo models.NetworkInfo, + extInterface models.ExternalInterface) /**hcn.HostComputeNetwork, replace this by network.network*/ error { + var ( + apipaNetwork *hcn.HostComputeNetwork + err error + ) + + // Check if the APIPA network exists + if apipaNetwork, err = hcn.GetNetworkByName(networkInfo.Id); err != nil { + // If error is anything other than networkNotFound, mark this as error + // TODO: why is following part not working? + /* + if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { + return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed due to error with GetNetworkByName: %v", err) + } + */ + + // APIPA network doesn't exist. Create one. + if apipaNetwork, err = configureApipaNetwork2(networkInfo); err != nil { + log.Printf("[Azure CNS] Failed to configure apipa network due to error: %v", err) + return err + } + + // Create the HNS network. + log.Printf("[net] Creating apipa network: %+v", apipaNetwork) + apipaNetwork, err = apipaNetwork.Create() + + if err != nil { + log.Printf("[net] Failed to create apipa network due to error: %v", err) + return fmt.Errorf("Failed to create apipa network: %s due to error: %v", apipaNetwork.Name, err) + } + + log.Printf("[net] Successfully created apipa network for host container connectivity: %+v", apipaNetwork) + } else { + log.Printf("[Azure CNS] Found existing APIPA network: %+v", apipaNetwork) + } + + return nil +} + +func configureApipaNetwork2(networkInfo models.NetworkInfo) (*hcn.HostComputeNetwork, error) { + // TODO: this needs to be the generic hnsv2 path implementation + apipaNetwork := &hcn.HostComputeNetwork{ + Name: networkInfo.Id, + Ipams: []hcn.Ipam{ + hcn.Ipam{ + Type: hcnIpamTypeStatic, + }, + }, + SchemaVersion: hcn.SchemaVersion{ + Major: hcnSchemaVersionMajor, + Minor: hcnSchemaVersionMinor, + }, + Type: hcn.L2Bridge, + } + + // TODO: How to get this string from the created loopback adapter? + // TODO: Create the loopback adapter using the LocalIPConfiguration passed in. + if netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy("Ethernet 6"); err == nil { + apipaNetwork.Policies = append(apipaNetwork.Policies, netAdapterNamePolicy) + } else { + log.Errorf("[Azure CNS] Failed to serialize network adapter policy due to error: %v", err) + return nil, err + } + + /* + // Calculate subnet prefix + var subnetPrefix net.IPNet + var subnetPrefixStr string + ipAddr := net.ParseIP(localIPConfiguration.IPSubnet.IPAddress) + if ipAddr.To4() != nil { + subnetPrefix = net.IPNet{Mask: net.CIDRMask(int(localIPConfiguration.IPSubnet.PrefixLength), 32)} + } else if ipAddr.To16() != nil { + subnetPrefix = net.IPNet{Mask: net.CIDRMask(int(localIPConfiguration.IPSubnet.PrefixLength), 128)} + } else { + return nil, fmt.Errorf("[Azure CNS] Failed get subnet prefix for localIPConfiguration: %+v", localIPConfiguration) + } + + subnetPrefix.IP = ipAddr.Mask(subnetPrefix.Mask) + subnetPrefixStr = subnetPrefix.IP.String() + "/" + strconv.Itoa(int(localIPConfiguration.IPSubnet.PrefixLength)) + log.Printf("[tempdebug] configureApipaNetwork: subnetPrefixStr: %s, GW: %s", subnetPrefixStr, localIPConfiguration.GatewayIPAddress) + */ + subnet := hcn.Subnet{ + IpAddressPrefix: "169.254.0.0/16", // TODO: this needs be calculated from LocalIPConfiguration passed in + //IpAddressPrefix: subnetPrefixStr, + Routes: []hcn.Route{ + hcn.Route{ + //NextHop: localIPConfiguration.GatewayIPAddress, + NextHop: "169.254.0.2", + DestinationPrefix: "0.0.0.0/0", + }, + }, + } + + apipaNetwork.Ipams[0].Subnets = append(apipaNetwork.Ipams[0].Subnets, subnet) + + return apipaNetwork, nil +} + +func CreateNewEndpoint( + endpointInfo models.EndpointInfo, + localIPConfiguration cns.IPConfiguration) (string, error) { + var ( + //apipaNetwork *hcn.HostComputeNetwork + apipaEndpoint *hcn.HostComputeEndpoint + err error + ) + + //TODO: this needs to be generic implementation of create endpoint with v2 + + //TODO: check if the endpoint exists + + /* + if apipaNetwork, err = createApipaNetwork(localIPConfiguration); err != nil { + log.Errorf("[Azure CNS] Failed to create apipa network for host container connectivity due to error: %v", err) + return "", err + } + */ + + if apipaEndpoint, err = configureApipaEndpoint(endpointInfo.NetworkID, localIPConfiguration); err != nil { + log.Errorf("[Azure CNS] Failed to configure apipa endpoint for host container connectivity due to error: %v", err) + return "", err + } + + // Create the apipa endpoint + log.Printf("[Azure CNS] Creating apipa endpoint for host-container connectivity: %+v", apipaEndpoint) + if apipaEndpoint, err = apipaEndpoint.Create(); err != nil { + err = fmt.Errorf("Failed to create apipa endpoint: %s due to error: %v", apipaEndpoint.Name, err) + log.Errorf("[Azure CNS] %s", err.Error()) + return "", err + } + + log.Printf("[Azure CNS] Successfully created apipa endpoint for host-container connectivity: %+v", apipaEndpoint) + + return apipaEndpoint.Id, nil +} diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 029875eba8..aa5f75885b 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -161,6 +161,8 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores) listener.AddHandler(cns.CreateApipaEndpointPath, service.createApipaEndpoint) listener.AddHandler(cns.DeleteApipaEndpointPath, service.deleteApipaEndpoint) + listener.AddHandler(cns.CreateNewEndpointPath, service.createNewEndpoint) + listener.AddHandler(cns.CreateNewNetworkPath, service.createNewNetwork) // handlers for v0.2 listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment) @@ -1717,3 +1719,109 @@ func (service *HTTPRestService) deleteApipaEndpoint(w http.ResponseWriter, r *ht err = service.Listener.Encode(w, &deleteApipaEndpointResp) log.Response(service.Name, deleteApipaEndpointResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } + +func (service *HTTPRestService) createNewNetwork(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure-CNS] createNewNetwork") + + var ( + returnCode int + err error + returnMessage string + req cns.CreateNewNetworkRequest + ) + + err = service.Listener.Decode(w, r, &req) + log.Request(service.Name, &req, err) + if err != nil { + return + } + + switch r.Method { + case "POST": + // Get the NC goal state from the NC identifier passed in request + /* + if req.OptionsNCIdentifier != nil { + if _, ok := req.OptionsNCIdentifier[OptOrchContext]; ok { + enableSnat = false + } + } + */ + if err = hnsclient.CreateNewNetwork(req.NetworkInfo, req.ExternalInterface); err != nil { + returnMessage = fmt.Sprintf("CreateNewNetwork failed with error: %v", err) + returnCode = UnexpectedError + } + default: + returnMessage = "CreateNewNetwork API expects a POST" + returnCode = UnsupportedVerb + } + + resp := cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + } + + log.Printf("[tempdebug] CreateNewNetwork: %+v", resp) + + err = service.Listener.Encode(w, &resp) + log.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) +} + +func (service *HTTPRestService) createNewEndpoint(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure-CNS] createNewEndpoint") + + var ( + returnCode int + err error + returnMessage string + req cns.CreateNewEndpointRequest + endpointID string + ) + + err = service.Listener.Decode(w, r, &req) + log.Request(service.Name, &req, err) + if err != nil { + return + } + + switch r.Method { + case "POST": + // Get the NC goal state from the NC identifier passed in request + /* + if req.OptionsNCIdentifier != nil { + if _, ok := req.OptionsNCIdentifier[OptOrchContext]; ok { + enableSnat = false + } + } + */ + + podInfo := cns.KubernetesPodInfo{PodName: req.EndpointInfo.PODName, PodNamespace: req.EndpointInfo.PODNameSpace} + orchestratorContext, _ := json.Marshal(podInfo) + + var req2 cns.GetNetworkContainerRequest + //req2.NetworkContainerid = req.NetworkContainerid + req2.OrchestratorContext = orchestratorContext + networkContainerGoalState := service.getNetworkContainerResponse(req2) + log.Printf("[tempdebug] restServer: networkContainerGoalState2: %+v", networkContainerGoalState) + if endpointID, err = hnsclient.CreateNewEndpoint(req.EndpointInfo, networkContainerGoalState.LocalIPConfiguration); err != nil { + returnMessage = fmt.Sprintf("CreateNewEndpoint failed with error: %v", err) + returnCode = UnexpectedError + } + default: + returnMessage = "CreateNewEndpoint API expects a POST" + returnCode = UnsupportedVerb + } + + resp := cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + } + + createApipaEndpointResp := cns.CreateApipaEndpointResponse{ + Response: resp, + EndpointID: endpointID, + } + log.Printf("[tempdebug] CreateNewEndpoint: %+v", createApipaEndpointResp) + + err = service.Listener.Encode(w, &createApipaEndpointResp) + log.Response(service.Name, createApipaEndpointResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) +} diff --git a/network/endpoint.go b/network/endpoint.go index 9e4b240b9a..b91c902523 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -42,6 +42,7 @@ type endpoint struct { PODNameSpace string `json:",omitempty"` InfraVnetAddressSpace string `json:",omitempty"` NetNs string `json:",omitempty"` + NetworkID string } // EndpointInfo contains read-only information about an endpoint. @@ -70,6 +71,7 @@ type EndpointInfo struct { Data map[string]interface{} InfraVnetAddressSpace string SkipHotAttachEp bool + NetworkID string } // RouteInfo contains information about an IP route. diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 92c1fdb2c4..5ac29fc1f8 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/cns/cnsclient" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/models" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" @@ -25,6 +26,9 @@ const ( // hcnIpamTypeStatic indicates the static type of ipam hcnIpamTypeStatic = "Static" + + // hostNCApipaNetworkName indicates the name of the apipa network used for host container connectivity + hostNCApipaNetworkName = "host-nc-apipa-network" ) // HotAttachEndpoint is a wrapper of hcsshim's HotAttachEndpoint. @@ -652,13 +656,138 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) // If the host <-> container connectivity is requested, create endpoint in APIPA // bridge network to facilitate that //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { + /* + { + if epInfo.TempApipaEpID, err = nw.createApipaEndpoint(epInfo); err != nil { + log.Errorf("[net] Failed to create APIPA endpoint due to error: %v", err) + return nil, err + } + } + */ + + //////////////////////////////////////////////////////////////////// + //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { + // Check if hostNCApipaNetwork already exists. + // If it doesn't exist, create one using cns api + // create endpoint in this network using cns api { - if epInfo.TempApipaEpID, err = nw.createApipaEndpoint(epInfo); err != nil { - log.Errorf("[net] Failed to create APIPA endpoint due to error: %v", err) + // TODO: if the below endpoint creation fails - delete the endpoint created above + var ( + hostNCApipaNetwork *hcn.HostComputeNetwork + err error + cnsClient *cnsclient.CNSClient + ) + + cnsClient, err = cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg + if err != nil { + log.Errorf("Initializing CNS client error %v", err) + return nil, err // upfate this to meaningful error + } + + // Check if the hostNCApipaNetwork exists + if hostNCApipaNetwork, err = hcn.GetNetworkByName(hostNCApipaNetworkName); err != nil { + // If error is anything other than networkNotFound, mark this as error + // TODO: why is following part not working? + /* + if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { + return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed due to error with GetNetworkByName: %v", err) + } + */ + + // hostNCApipaNetwork network doesn't exist. Create the network + //log.Printf("[net] Creating apipa network for host container connectivity: %+v", apipaNetwork) + // Create the network. + /* + nwInfo := network.NetworkInfo{ + Id: networkId, + Mode: nwCfg.Mode, + MasterIfName: masterIfName, + Subnets: []network.SubnetInfo{ + network.SubnetInfo{ + Family: platform.AfINET, + Prefix: subnetPrefix, + Gateway: gateway, + }, + }, + BridgeName: nwCfg.Bridge, + EnableSnatOnHost: nwCfg.EnableSnatOnHost, + DNS: nwDNSInfo, + Policies: policies, + NetNs: args.Netns, + } + */ + networkInfo := models.NetworkInfo{ + Id: hostNCApipaNetworkName, + Mode: opModeBridge, + } + + err = cnsClient.CreateNetwork(networkInfo, models.ExternalInterface{}) + if err != nil { + return nil, fmt.Errorf("Failed to create hostNCApipaNetworkName due to error: %v", err) + } + + log.Printf("[net] Successfully created hostNCApipaNetworkName for host container connectivity: %+v", hostNCApipaNetwork) + } else { + log.Printf("[Azure CNS] Found existing hostNCApipaNetwork: %+v", hostNCApipaNetwork) + } + + // Create endpoint in the apipa network for host container connectivity + // TODO: use the returned info by createNetwork to get the network id instead of following call. + if hostNCApipaNetwork, err = hcn.GetNetworkByName(hostNCApipaNetworkName); err != nil { + log.Printf("[Azure CNS] tempdebug: something is wrong. Error: %v", err) return nil, err } + ///////////////////////////////////////////////////////////////////////////////// + + epInfo.NetworkID = hostNCApipaNetwork.Id + + epInfo2 := models.EndpointInfo{ + NetworkID: epInfo.NetworkID, + PODName: epInfo.PODName, + PODNameSpace: epInfo.PODNameSpace, + } + // Dup models.EndpointInfo + + log.Printf("[Azure CNS] tempdebug: creating endpoint with Info: %+v", epInfo) + //epInfo2 := models.EndpointInfo{} + + resp, err := cnsClient.CreateEndpoint(epInfo2) + if err != nil { + return nil, fmt.Errorf("Failed to create apipa endpoint due to error: %v", err) + } + + if resp.Response.ReturnCode != 0 { + return nil, fmt.Errorf("Failed to create apipa endpoint due to error: %s", resp.Response.Message) + } + + endpointID := resp.EndpointID + + defer func() { + if err != nil { + log.Printf("[net] Deleting hcn endpoint with id: %s", endpointID) + // TODO: when this becomes generic, localIP can be passed to delete the endpoint + err = cnsClient.DeleteApipaEndpoint(endpointID) + log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", endpointID, err) + } + }() + + var namespace *hcn.HostComputeNamespace + if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { + return nil, fmt.Errorf("Failed to create apipa endpoint due to GetNamespaceByID NetNsPath: %s error: %v", epInfo.NetNsPath, err) + } + + if err = hcn.AddNamespaceEndpoint(namespace.Id, endpointID); err != nil { + return nil, fmt.Errorf("[net] Failed to add apipa endpoint: %s to namespace: %s due to error: %v", + endpointID, namespace.Id, err) + } + + epInfo.TempApipaEpID = endpointID + + /////////////////////////////////////////////////////////////////////////////// } + //////////////////////////////////////////////////////////////////// + var vlanid int if epInfo.Data != nil { if vlanData, ok := epInfo.Data[VlanIDKey]; ok { @@ -686,6 +815,7 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) EnableSnatOnHost: epInfo.EnableSnatOnHost, NetNs: epInfo.NetNsPath, TempApipaEpID: epInfo.TempApipaEpID, + NetworkID: epInfo.NetworkID, } for _, route := range epInfo.Routes { diff --git a/network/models/models.go b/network/models/models.go new file mode 100644 index 0000000000..087d11058d --- /dev/null +++ b/network/models/models.go @@ -0,0 +1,140 @@ +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package models + +import ( + "net" + + "github.com/Azure/azure-container-networking/network/policy" + "github.com/Azure/azure-container-networking/platform" +) + +//todo: linux has different def for this. +type route interface{} + +// ExternalInterface is a host network interface that bridges containers to external networks. +type ExternalInterface struct { + Name string + Networks map[string]*network + Subnets []string + BridgeName string + DNSInfo DNSInfo + MacAddress net.HardwareAddr + IPAddresses []*net.IPNet + Routes []*route + IPv4Gateway net.IP + IPv6Gateway net.IP +} + +// A container network is a set of endpoints allowed to communicate with each other. +type network struct { + Id string + HnsId string `json:",omitempty"` + Mode string + VlanId int + Subnets []SubnetInfo + Endpoints map[string]*endpoint + extIf *ExternalInterface + DNS DNSInfo + EnableSnatOnHost bool + NetNs string + SnatBridgeIP string +} + +// NetworkInfo contains read-only information about a container network. +type NetworkInfo struct { + MasterIfName string + Id string + Mode string + Subnets []SubnetInfo + DNS DNSInfo + Policies []policy.Policy + BridgeName string + EnableSnatOnHost bool + NetNs string + Options map[string]interface{} +} + +// SubnetInfo contains subnet information for a container network. +type SubnetInfo struct { + Family platform.AddressFamily + Prefix net.IPNet + Gateway net.IP +} + +// DNSInfo contains DNS information for a container network or endpoint. +type DNSInfo struct { + Suffix string + Servers []string + Options []string +} + +// Endpoint represents a container network interface. +type endpoint struct { + Id string + HnsId string `json:",omitempty"` + SandboxKey string + IfName string + HostIfName string + MacAddress net.HardwareAddr + InfraVnetIP net.IPNet + LocalIP string + IPAddresses []net.IPNet + Gateways []net.IP + DNS DNSInfo + Routes []RouteInfo + VlanID int + EnableSnatOnHost bool + EnableInfraVnet bool + EnableMultitenancy bool + AllowInboundFromHostToNC bool + AllowInboundFromNCToHost bool + TempApipaEpID string + NetworkNameSpace string `json:",omitempty"` + ContainerID string + PODName string `json:",omitempty"` + PODNameSpace string `json:",omitempty"` + InfraVnetAddressSpace string `json:",omitempty"` + NetNs string `json:",omitempty"` + NetworkID string +} + +// EndpointInfo contains read-only information about an endpoint. +type EndpointInfo struct { + Id string + ContainerID string + NetNsPath string + IfName string + SandboxKey string + IfIndex int + MacAddress net.HardwareAddr + DNS DNSInfo + IPAddresses []net.IPNet + InfraVnetIP net.IPNet + Routes []RouteInfo + Policies []policy.Policy + Gateways []net.IP + EnableSnatOnHost bool + EnableInfraVnet bool + EnableMultiTenancy bool + AllowInboundFromHostToNC bool + AllowInboundFromNCToHost bool + TempApipaEpID string + PODName string + PODNameSpace string + Data map[string]interface{} + InfraVnetAddressSpace string + SkipHotAttachEp bool + NetworkID string +} + +// RouteInfo contains information about an IP route. +type RouteInfo struct { + Dst net.IPNet + Src net.IP + Gw net.IP + Protocol int + DevName string + Scope int +} From a8897a9bc0b4bc3a9d2d53545a3e54e6f6963a76 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Mon, 7 Oct 2019 12:48:17 -0700 Subject: [PATCH 23/37] WIP7 --- cns/hnsclient/hnsclient_windows.go | 143 +++++++++++++++++++++++++++-- 1 file changed, 136 insertions(+), 7 deletions(-) diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index ad004834e1..49375ddb80 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -2,8 +2,11 @@ package hnsclient import ( "encoding/json" + "errors" "fmt" "net" + "os" + "os/exec" "strconv" "strings" @@ -653,13 +656,16 @@ func DeleteApipaEndpoint(endpointID string) error { var endpoints []hcn.HostComputeEndpoint // Check if the network has any endpoints left if endpoints, err = hcn.ListEndpointsOfNetwork(apipaEndpoint.HostComputeNetwork); err != nil { - log.Errorf("[Azure CNS] Failed to list endpoints in the network: %s due to error", apipaNetworkName, err) + log.Errorf("[Azure CNS] Failed to list endpoints in the network: %s due to error: %v", apipaNetworkName, err) return nil } // Delete network if it doesn't have any endpoints if len(endpoints) == 0 { - DeleteApipaNetwork(apipaEndpoint.HostComputeNetwork) + if err = DeleteApipaNetwork(apipaEndpoint.HostComputeNetwork); err == nil { + // Delete the loopback adapter created for this network + deleteLoopbackAdapter("LoopbackAdapterHostNCConnectivity") + } } return nil @@ -713,7 +719,7 @@ func CreateNewNetwork( */ // APIPA network doesn't exist. Create one. - if apipaNetwork, err = configureApipaNetwork2(networkInfo); err != nil { + if apipaNetwork, err = configureApipaNetwork2(networkInfo, extInterface); err != nil { log.Printf("[Azure CNS] Failed to configure apipa network due to error: %v", err) return err } @@ -735,7 +741,9 @@ func CreateNewNetwork( return nil } -func configureApipaNetwork2(networkInfo models.NetworkInfo) (*hcn.HostComputeNetwork, error) { +func configureApipaNetwork2( + networkInfo models.NetworkInfo, + extInterface models.ExternalInterface) (*hcn.HostComputeNetwork, error) { // TODO: this needs to be the generic hnsv2 path implementation apipaNetwork := &hcn.HostComputeNetwork{ Name: networkInfo.Id, @@ -751,9 +759,25 @@ func configureApipaNetwork2(networkInfo models.NetworkInfo) (*hcn.HostComputeNet Type: hcn.L2Bridge, } - // TODO: How to get this string from the created loopback adapter? - // TODO: Create the loopback adapter using the LocalIPConfiguration passed in. - if netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy("Ethernet 6"); err == nil { + // Create loopback adapter if needed + // TODO: check the settings from the options in networkInfo and create the loopback adapter if needed. + ipconfig := cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: "169.254.0.2", + PrefixLength: 16, + }, + GatewayIPAddress: "169.254.0.2", + } + + if exists, _ := interfaceExists("LoopbackAdapterHostNCConnectivity"); !exists { + if err := createLoopbackAdapter("LoopbackAdapterHostNCConnectivity", ipconfig); err != nil { + err = fmt.Errorf("Failed to create loopback adapter for host container connectivity due to error: %v", err) + log.Errorf("[Azure CNS] %v", err) + return nil, err + } + } + + if netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy( /*"Ethernet 6"*/ "LoopbackAdapterHostNCConnectivity"); err == nil { apipaNetwork.Policies = append(apipaNetwork.Policies, netAdapterNamePolicy) } else { log.Errorf("[Azure CNS] Failed to serialize network adapter policy due to error: %v", err) @@ -831,3 +855,108 @@ func CreateNewEndpoint( return apipaEndpoint.Id, nil } + +func interfaceExists(iFaceName string) (bool, error) { + _, err := net.InterfaceByName(iFaceName) + if err != nil { + errMsg := fmt.Sprintf("[Azure CNS] Unable to get interface by name %s. Error: %v", iFaceName, err) + log.Printf(errMsg) + return false, fmt.Errorf(errMsg) + } + + log.Printf("[Azure CNS] Found interface by name %s", iFaceName) + + return true, nil +} + +func createLoopbackAdapter( + adapterName string, + ipConfig cns.IPConfiguration) error { + if _, err := os.Stat("./AzureNetworkContainer.exe"); err != nil { + return fmt.Errorf("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") + } + + if ipConfig.IPSubnet.IPAddress == "" { + return fmt.Errorf("[Azure CNS] IPAddress in IPConfiguration is nil") + } + + ipv4AddrCidr := fmt.Sprintf("%v/%d", ipConfig.IPSubnet.IPAddress, ipConfig.IPSubnet.PrefixLength) + log.Printf("[Azure CNS] Created ipv4Cidr as %v", ipv4AddrCidr) + ipv4Addr, _, err := net.ParseCIDR(ipv4AddrCidr) + ipv4NetInt := net.CIDRMask((int)(ipConfig.IPSubnet.PrefixLength), 32) + log.Printf("[Azure CNS] Created netmask as %v", ipv4NetInt) + ipv4NetStr := fmt.Sprintf("%d.%d.%d.%d", ipv4NetInt[0], ipv4NetInt[1], ipv4NetInt[2], ipv4NetInt[3]) + log.Printf("[Azure CNS] Created netmask in string format %v", ipv4NetStr) + + args := []string{"/C", "AzureNetworkContainer.exe", "/logpath", log.GetLogDirectory(), + "/name", + adapterName, + "/operation", + "CREATE", + "/ip", + ipv4Addr.String(), + "/netmask", + ipv4NetStr, + "/gateway", + ipConfig.GatewayIPAddress, + "/weakhostsend", + "true", + "/weakhostreceive", + "true"} + + c := exec.Command("cmd", args...) + + //loopbackOperationLock.Lock() + log.Printf("[Azure CNS] Going to create/update network loopback adapter: %v", args) + bytes, err := c.Output() + /* + if err == nil { + err = setWeakHostOnInterface(createNetworkContainerRequest.PrimaryInterfaceIdentifier, + createNetworkContainerRequest.NetworkContainerid) + } + */ + //loopbackOperationLock.Unlock() + + if err == nil { + log.Printf("[Azure CNS] Successfully created network loopback adapter for ipConfig: %+v. Output:%v.", + ipConfig, string(bytes)) + } else { + log.Printf("Failed to create loopback adapter for IP config: %+v. Error: %v. Output: %v", + ipConfig, err, string(bytes)) + } + + return err +} + +func deleteLoopbackAdapter(adapterName string) error { + if _, err := os.Stat("./AzureNetworkContainer.exe"); err != nil { + return fmt.Errorf("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") + } + + if adapterName == "" { + return errors.New("[Azure CNS] Adapter name is not specified") + } + + args := []string{"/C", "AzureNetworkContainer.exe", "/logpath", log.GetLogDirectory(), + "/name", + adapterName, + "/operation", + "DELETE"} + + c := exec.Command("cmd", args...) + + //loopbackOperationLock.Lock() + log.Printf("[Azure CNS] Going to delete network loopback adapter: %v", args) + bytes, err := c.Output() + // loopbackOperationLock.Unlock() + + if err == nil { + log.Printf("[Azure CNS] Successfully deleted loopback adapter: %s. Output: %v.", + adapterName, string(bytes)) + } else { + log.Printf("Failed to delete loopback adapter: %s. Error: %v. Output: %v", + adapterName, err, string(bytes)) + //return err + } + return err +} From 12e7a7182b0928976a65ec277003b16855f17df1 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Mon, 7 Oct 2019 16:12:41 -0700 Subject: [PATCH 24/37] WIP - not tested --- cns/api.go | 2 +- cns/cnsclient/cnsclient.go | 20 +-- network/endpoint.go | 16 +-- network/endpoint_windows.go | 265 +++++++++++++++++------------------- network/models/models.go | 4 +- 5 files changed, 144 insertions(+), 163 deletions(-) diff --git a/cns/api.go b/cns/api.go index 61196bd3e0..c92fafa90c 100644 --- a/cns/api.go +++ b/cns/api.go @@ -92,7 +92,7 @@ type EndpointInfo struct { EnableMultiTenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool - TempApipaEpID string + HostNCApipaEndpointID string PODName string PODNameSpace string Data map[string]interface{} diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index c166d619bd..8c375eb53f 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -68,7 +68,7 @@ type EndpointInfo struct { EnableMultiTenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool - TempApipaEpID string + HostNCApipaEndpointID string PODName string PODNameSpace string Data map[string]interface{} @@ -257,7 +257,7 @@ func (cnsClient *CNSClient) DeleteApipaEndpoint(endpointID string) error { return nil } -// CreateNetwork creates an network. +// CreateNetwork creates the network. func (cnsClient *CNSClient) CreateNetwork( networkInfo models.NetworkInfo, extIf models.ExternalInterface) /*network.network, - this might need to be Network to be xported*/ error { @@ -270,19 +270,16 @@ func (cnsClient *CNSClient) CreateNetwork( url := cnsClient.connectionURL + cns.CreateNewNetworkPath log.Printf("CreateNewNetworkPath url: %v", url) - // What can be used here? payload := &cns.CreateNewNetworkRequest{ NetworkInfo: networkInfo, ExternalInterface: extIf, } - err = json.NewEncoder(&body).Encode(payload) - if err != nil { + if err = json.NewEncoder(&body).Encode(payload); err != nil { log.Errorf("encoding json failed with %v", err) return err } - log.Printf("CreateNetwork posting body: %v", body) res, err := httpc.Post(url, "application/json", &body) if err != nil { log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) @@ -292,16 +289,14 @@ func (cnsClient *CNSClient) CreateNetwork( defer res.Body.Close() if res.StatusCode != http.StatusOK { - errMsg := fmt.Sprintf("[Azure CNSClient] CreateNetwork: Invalid http status code: %v", - res.StatusCode) + errMsg := fmt.Sprintf("[Azure CNSClient] CreateNetwork: Invalid http status code: %v", res.StatusCode) log.Errorf(errMsg) return fmt.Errorf(errMsg) } var resp cns.Response - err = json.NewDecoder(res.Body).Decode(&resp) - if err != nil { + if err = json.NewDecoder(res.Body).Decode(&resp); err != nil { log.Errorf("[Azure CNSClient] Error parsing CreateNetwork response resp: %v err: %v", res.Body, err.Error()) return err @@ -327,7 +322,6 @@ func (cnsClient *CNSClient) CreateEndpoint( url := cnsClient.connectionURL + cns.CreateNewEndpointPath log.Printf("CreateEndpoint url: %v", url) - // What can be used here? payload := &cns.CreateNewEndpointRequest{ EndpointInfo: endpointInfo, } @@ -338,7 +332,6 @@ func (cnsClient *CNSClient) CreateEndpoint( return nil, err } - log.Printf("CreateEndpoint posting body: %v", body) res, err := httpc.Post(url, "application/json", &body) if err != nil { log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) @@ -356,8 +349,7 @@ func (cnsClient *CNSClient) CreateEndpoint( var resp cns.CreateApipaEndpointResponse - err = json.NewDecoder(res.Body).Decode(&resp) - if err != nil { + if err = json.NewDecoder(res.Body).Decode(&resp); err != nil { log.Errorf("[Azure CNSClient] Error parsing CreateEndpoint response resp: %v err: %v", res.Body, err.Error()) return nil, err diff --git a/network/endpoint.go b/network/endpoint.go index b91c902523..84cdf41c5e 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -35,7 +35,7 @@ type endpoint struct { EnableMultitenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool - TempApipaEpID string + HostNCApipaEndpointID string NetworkNameSpace string `json:",omitempty"` ContainerID string PODName string `json:",omitempty"` @@ -65,7 +65,7 @@ type EndpointInfo struct { EnableMultiTenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool - TempApipaEpID string + HostNCApipaEndpointID string PODName string PODNameSpace string Data map[string]interface{} @@ -208,12 +208,12 @@ func (ep *endpoint) getInfo() *EndpointInfo { EnableMultiTenancy: ep.EnableMultitenancy, AllowInboundFromHostToNC: ep.AllowInboundFromHostToNC, AllowInboundFromNCToHost: ep.AllowInboundFromNCToHost, - IfName: ep.IfName, - ContainerID: ep.ContainerID, - NetNsPath: ep.NetworkNameSpace, - PODName: ep.PODName, - PODNameSpace: ep.PODNameSpace, - TempApipaEpID: ep.TempApipaEpID, + IfName: ep.IfName, + ContainerID: ep.ContainerID, + NetNsPath: ep.NetworkNameSpace, + PODName: ep.PODName, + PODNameSpace: ep.PODNameSpace, + HostNCApipaEndpointID: ep.HostNCApipaEndpointID, } for _, route := range ep.Routes { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 5ac29fc1f8..272aafe325 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -27,8 +27,8 @@ const ( // hcnIpamTypeStatic indicates the static type of ipam hcnIpamTypeStatic = "Static" - // hostNCApipaNetworkName indicates the name of the apipa network used for host container connectivity - hostNCApipaNetworkName = "host-nc-apipa-network" + // HostNCApipaNetworkName indicates the name of the apipa network used for host container connectivity + HostNCApipaNetworkName = "host-nc-apipa-network" ) // HotAttachEndpoint is a wrapper of hcsshim's HotAttachEndpoint. @@ -609,6 +609,123 @@ func (nw *network) createApipaEndpoint(epInfo *EndpointInfo) (string, error) { return endpointID, nil } +func (nw *network) createHostNCApipaEndpoint(epInfo *EndpointInfo) error { + var ( + hostNCApipaNetwork *hcn.HostComputeNetwork + err error + cnsClient *cnsclient.CNSClient + ) + + // TODO: This should be changed to GetCnsClient() + cnsClient, err = cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg + if err != nil { + log.Errorf("Initializing CNS client error %v", err) + return err // upfate this to meaningful error + } + + // Check if the HostNCApipaNetwork exists + if hostNCApipaNetwork, err = hcn.GetNetworkByName(HostNCApipaNetworkName); err != nil { + // If error is anything other than networkNotFound, mark this as error + // TODO: why is following part not working? + /* + if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { + return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed due to error with GetNetworkByName: %v", err) + } + */ + + // HostNCApipaNetwork network doesn't exist. Create the network. + // Create the network. + /* + nwInfo := network.NetworkInfo{ + Id: networkId, + Mode: nwCfg.Mode, + MasterIfName: masterIfName, + Subnets: []network.SubnetInfo{ + network.SubnetInfo{ + Family: platform.AfINET, + Prefix: subnetPrefix, + Gateway: gateway, + }, + }, + BridgeName: nwCfg.Bridge, + EnableSnatOnHost: nwCfg.EnableSnatOnHost, + DNS: nwDNSInfo, + Policies: policies, + NetNs: args.Netns, + } + */ + networkInfo := models.NetworkInfo{ + Id: HostNCApipaNetworkName, + Mode: opModeBridge, + } + + // Set options to indicate there is no external interface and needs loopback adapter creation + + log.Printf("[net] Creating HostNCApipaNetwork for host container connectivity: %+v", networkInfo) + + if err = cnsClient.CreateNetwork(networkInfo, models.ExternalInterface{}); err != nil { + return fmt.Errorf("Failed to create HostNCApipaNetworkName due to error: %v", err) + } + + log.Printf("[net] Successfully created HostNCApipaNetworkName for host container connectivity") + } else { + log.Printf("[net] Found existing hostNCApipaNetwork: %+v", hostNCApipaNetwork) + } + + // Create endpoint in the network for host container connectivity + // TODO: use the returned info by createNetwork to get the network id instead of following call. + if hostNCApipaNetwork, err = hcn.GetNetworkByName(HostNCApipaNetworkName); err != nil { + log.Printf("[Azure CNS] tempdebug: something is wrong. Error: %v", err) + return err + } + ///////////////////////////////////////////////////////////////////////////////// + + epInfo.NetworkID = hostNCApipaNetwork.Id + + epInfo2 := models.EndpointInfo{ + NetworkID: epInfo.NetworkID, + PODName: epInfo.PODName, + PODNameSpace: epInfo.PODNameSpace, + } + // Dup models.EndpointInfo + + log.Printf("[Azure CNS] tempdebug: creating endpoint with Info: %+v", epInfo) + + resp, err := cnsClient.CreateEndpoint(epInfo2) + if err != nil { + return fmt.Errorf("Failed to create apipa endpoint due to error: %v", err) + } + + if resp.Response.ReturnCode != 0 { + return fmt.Errorf("Failed to create apipa endpoint due to error: %s", resp.Response.Message) + } + + endpointID := resp.EndpointID + + defer func() { + if err != nil { + log.Printf("[net] Deleting hcn endpoint with id: %s", endpointID) + // TODO: when this becomes generic, localIP can be passed to delete the endpoint + err = cnsClient.DeleteApipaEndpoint(endpointID) + log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", endpointID, err) + } + }() + + var namespace *hcn.HostComputeNamespace + if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { + return fmt.Errorf("Failed to create apipa endpoint due to GetNamespaceByID NetNsPath: %s error: %v", epInfo.NetNsPath, err) + } + + if err = hcn.AddNamespaceEndpoint(namespace.Id, endpointID); err != nil { + return fmt.Errorf("[net] Failed to add apipa endpoint: %s to namespace: %s due to error: %v", + endpointID, namespace.Id, err) + } + + epInfo.HostNCApipaEndpointID = endpointID + + return nil +} + // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { hcnEndpoint, err := nw.configureHcnEndpoint(epInfo) @@ -653,140 +770,12 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) } }() - // If the host <-> container connectivity is requested, create endpoint in APIPA - // bridge network to facilitate that + // If the host <-> container connectivity is requested, create endpoint in HostNCApipaNetwork //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { - /* - { - if epInfo.TempApipaEpID, err = nw.createApipaEndpoint(epInfo); err != nil { - log.Errorf("[net] Failed to create APIPA endpoint due to error: %v", err) - return nil, err - } - } - */ - - //////////////////////////////////////////////////////////////////// - //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { - // Check if hostNCApipaNetwork already exists. - // If it doesn't exist, create one using cns api - // create endpoint in this network using cns api - { - // TODO: if the below endpoint creation fails - delete the endpoint created above - var ( - hostNCApipaNetwork *hcn.HostComputeNetwork - err error - cnsClient *cnsclient.CNSClient - ) - - cnsClient, err = cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg - if err != nil { - log.Errorf("Initializing CNS client error %v", err) - return nil, err // upfate this to meaningful error - } - - // Check if the hostNCApipaNetwork exists - if hostNCApipaNetwork, err = hcn.GetNetworkByName(hostNCApipaNetworkName); err != nil { - // If error is anything other than networkNotFound, mark this as error - // TODO: why is following part not working? - /* - if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { - return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed due to error with GetNetworkByName: %v", err) - } - */ - - // hostNCApipaNetwork network doesn't exist. Create the network - //log.Printf("[net] Creating apipa network for host container connectivity: %+v", apipaNetwork) - // Create the network. - /* - nwInfo := network.NetworkInfo{ - Id: networkId, - Mode: nwCfg.Mode, - MasterIfName: masterIfName, - Subnets: []network.SubnetInfo{ - network.SubnetInfo{ - Family: platform.AfINET, - Prefix: subnetPrefix, - Gateway: gateway, - }, - }, - BridgeName: nwCfg.Bridge, - EnableSnatOnHost: nwCfg.EnableSnatOnHost, - DNS: nwDNSInfo, - Policies: policies, - NetNs: args.Netns, - } - */ - networkInfo := models.NetworkInfo{ - Id: hostNCApipaNetworkName, - Mode: opModeBridge, - } - - err = cnsClient.CreateNetwork(networkInfo, models.ExternalInterface{}) - if err != nil { - return nil, fmt.Errorf("Failed to create hostNCApipaNetworkName due to error: %v", err) - } - - log.Printf("[net] Successfully created hostNCApipaNetworkName for host container connectivity: %+v", hostNCApipaNetwork) - } else { - log.Printf("[Azure CNS] Found existing hostNCApipaNetwork: %+v", hostNCApipaNetwork) - } - - // Create endpoint in the apipa network for host container connectivity - // TODO: use the returned info by createNetwork to get the network id instead of following call. - if hostNCApipaNetwork, err = hcn.GetNetworkByName(hostNCApipaNetworkName); err != nil { - log.Printf("[Azure CNS] tempdebug: something is wrong. Error: %v", err) - return nil, err - } - ///////////////////////////////////////////////////////////////////////////////// - - epInfo.NetworkID = hostNCApipaNetwork.Id - - epInfo2 := models.EndpointInfo{ - NetworkID: epInfo.NetworkID, - PODName: epInfo.PODName, - PODNameSpace: epInfo.PODNameSpace, - } - // Dup models.EndpointInfo - - log.Printf("[Azure CNS] tempdebug: creating endpoint with Info: %+v", epInfo) - //epInfo2 := models.EndpointInfo{} - - resp, err := cnsClient.CreateEndpoint(epInfo2) - if err != nil { - return nil, fmt.Errorf("Failed to create apipa endpoint due to error: %v", err) - } - - if resp.Response.ReturnCode != 0 { - return nil, fmt.Errorf("Failed to create apipa endpoint due to error: %s", resp.Response.Message) - } - - endpointID := resp.EndpointID - - defer func() { - if err != nil { - log.Printf("[net] Deleting hcn endpoint with id: %s", endpointID) - // TODO: when this becomes generic, localIP can be passed to delete the endpoint - err = cnsClient.DeleteApipaEndpoint(endpointID) - log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", endpointID, err) - } - }() - - var namespace *hcn.HostComputeNamespace - if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { - return nil, fmt.Errorf("Failed to create apipa endpoint due to GetNamespaceByID NetNsPath: %s error: %v", epInfo.NetNsPath, err) - } - - if err = hcn.AddNamespaceEndpoint(namespace.Id, endpointID); err != nil { - return nil, fmt.Errorf("[net] Failed to add apipa endpoint: %s to namespace: %s due to error: %v", - endpointID, namespace.Id, err) - } - - epInfo.TempApipaEpID = endpointID - - /////////////////////////////////////////////////////////////////////////////// + if err = nw.createHostNCApipaEndpoint(epInfo); err != nil { + // TODO: delete the endpoint created above and return appropriate error. } - - //////////////////////////////////////////////////////////////////// + //} var vlanid int if epInfo.Data != nil { @@ -800,7 +789,7 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) gateway = net.ParseIP(hnsResponse.Routes[0].NextHop) } - log.Errorf("[net] tempdebug: tempApipaEP: %s", epInfo.TempApipaEpID) + log.Errorf("[net] tempdebug: HostNCApipaEndpointID: %s", epInfo.HostNCApipaEndpointID) // Create the endpoint object. ep := &endpoint{ @@ -814,8 +803,8 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) VlanID: vlanid, EnableSnatOnHost: epInfo.EnableSnatOnHost, NetNs: epInfo.NetNsPath, - TempApipaEpID: epInfo.TempApipaEpID, - NetworkID: epInfo.NetworkID, + HostNCApipaEndpointID: epInfo.HostNCApipaEndpointID, + NetworkID: epInfo.NetworkID, } for _, route := range epInfo.Routes { @@ -859,7 +848,7 @@ func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { log.Printf("[net] deleteEndpointImplHnsV2 DELETE id:%+v", ep) //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { { - if err = nw.deleteApipaEndpoint(ep.TempApipaEpID); err != nil { + if err = nw.deleteApipaEndpoint(ep.HostNCApipaEndpointID); err != nil { log.Errorf("[net] Failed to delete APIPA endpoint due to error: %v", err) return err } diff --git a/network/models/models.go b/network/models/models.go index 087d11058d..3a39cbb4ce 100644 --- a/network/models/models.go +++ b/network/models/models.go @@ -90,7 +90,7 @@ type endpoint struct { EnableMultitenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool - TempApipaEpID string + HostNCApipaEndpointID string NetworkNameSpace string `json:",omitempty"` ContainerID string PODName string `json:",omitempty"` @@ -120,7 +120,7 @@ type EndpointInfo struct { EnableMultiTenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool - TempApipaEpID string + HostNCApipaEndpointID string PODName string PODNameSpace string Data map[string]interface{} From 99663baf4fe2ebf823dd3f82573006b75cf26a3a Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Sat, 12 Oct 2019 16:37:16 -0700 Subject: [PATCH 25/37] WIP 10-12 --- cni/network/network_linux.go | 1 + cni/network/network_windows.go | 3 + cns/NetworkContainerContract.go | 1 + cns/api.go | 64 +++++----- cns/cnsclient/cnsclient.go | 192 +++++++---------------------- cns/hnsclient/hnsclient_windows.go | 15 ++- cns/restserver/restserver.go | 101 ++++++++------- network/endpoint.go | 4 +- network/endpoint_windows.go | 168 ++++--------------------- 9 files changed, 171 insertions(+), 378 deletions(-) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index cd771afe2a..85042ae4d8 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -58,6 +58,7 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne epInfo.Data[network.SnatBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) epInfo.AllowInboundFromHostToNC = cnsNwConfig.AllowHostToNCCommunication epInfo.AllowInboundFromNCToHost = cnsNwConfig.AllowNCToHostCommunication + epInfo.NetworkContainerID = cnsNwConfig.NetworkContainerID } epInfo.Data[network.OptVethName] = vethName diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 7c69714f4e..9eb67955f6 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -94,6 +94,9 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne } epInfo.Data[network.CnetAddressSpace] = cnetAddressMap epInfo.Data[network.LocalIPKey] = cnsNwConfig.LocalIPConfiguration.IPSubnet.IPAddress + epInfo.AllowInboundFromHostToNC = cnsNwConfig.AllowHostToNCCommunication + epInfo.AllowInboundFromNCToHost = cnsNwConfig.AllowNCToHostCommunication + epInfo.NetworkContainerID = cnsNwConfig.NetworkContainerID } } diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 1816e5187c..55d82e7118 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -131,6 +131,7 @@ type GetNetworkContainerRequest struct { // GetNetworkContainerResponse describes the response to retrieve a specifc network container. type GetNetworkContainerResponse struct { + NetworkContainerID string IPConfiguration IPConfiguration Routes []Route CnetAddressSpace []IPSubnet diff --git a/cns/api.go b/cns/api.go index c92fafa90c..4d65391e77 100644 --- a/cns/api.go +++ b/cns/api.go @@ -11,26 +11,26 @@ import ( // Container Network Service remote API Contract const ( - SetEnvironmentPath = "/network/environment" - CreateNetworkPath = "/network/create" - DeleteNetworkPath = "/network/delete" - CreateHnsNetworkPath = "/network/hns/create" - DeleteHnsNetworkPath = "/network/hns/delete" - ReserveIPAddressPath = "/network/ip/reserve" - ReleaseIPAddressPath = "/network/ip/release" - GetHostLocalIPPath = "/network/ip/hostlocal" - GetIPAddressUtilizationPath = "/network/ip/utilization" - GetUnhealthyIPAddressesPath = "/network/ipaddresses/unhealthy" - GetHealthReportPath = "/network/health" - NumberOfCPUCoresPath = "/hostcpucores" - CreateApipaEndpointPath = "/network/createapipaendpoint" - DeleteApipaEndpointPath = "/network/deleteapipaendpoint" - CreateNewNetworkPath = "/network/createnewnetwork" - DeleteNewNetworkPath = "/network/deletenewnetwork" - CreateNewEndpointPath = "/network/createnewendpoint" - DeleteNewEndpointPath = "/network/deletenewendpoint" - V1Prefix = "/v0.1" - V2Prefix = "/v0.2" + SetEnvironmentPath = "/network/environment" + CreateNetworkPath = "/network/create" + DeleteNetworkPath = "/network/delete" + CreateHnsNetworkPath = "/network/hns/create" + DeleteHnsNetworkPath = "/network/hns/delete" + ReserveIPAddressPath = "/network/ip/reserve" + ReleaseIPAddressPath = "/network/ip/release" + GetHostLocalIPPath = "/network/ip/hostlocal" + GetIPAddressUtilizationPath = "/network/ip/utilization" + GetUnhealthyIPAddressesPath = "/network/ipaddresses/unhealthy" + GetHealthReportPath = "/network/health" + NumberOfCPUCoresPath = "/hostcpucores" + CreateHostNCApipaEndpointPath = "/network/createhostncapipaendpoint" + DeleteHostNCApipaEndpointPath = "/network/deletehostncapipaendpoint" + CreateNewNetworkPath = "/network/createnewnetwork" + DeleteNewNetworkPath = "/network/deletenewnetwork" + CreateNewEndpointPath = "/network/createnewendpoint" + DeleteNewEndpointPath = "/network/deletenewendpoint" + V1Prefix = "/v0.1" + V2Prefix = "/v0.2" OptOrchContext = "OrchestratorContext" OptNCID = "NCID" @@ -244,25 +244,27 @@ type CreateNewEndpointRequest struct { EndpointInfo models.EndpointInfo } -// CreateApipaEndpointRequest describes request for create apipa endpoint. -type CreateApipaEndpointRequest struct { - //OptionsNCIdentifier map[string]interface{} - NetworkContainerid string - OrchestratorContext json.RawMessage +// CreateHostNCApipaEndpointRequest describes request for create apipa endpoint +// for host container connectivity for the given network container +type CreateHostNCApipaEndpointRequest struct { + NetworkContainerID string + //OrchestratorContext json.RawMessage } -// CreateApipaEndpointResponse describes response for create apipa endpoint request. -type CreateApipaEndpointResponse struct { +// CreateHostNCApipaEndpointResponse describes response for create apipa endpoint request +// for host container connectivity. +type CreateHostNCApipaEndpointResponse struct { Response Response EndpointID string } -// DeleteApipaEndpointRequest describes request for deleting apipa endpoint. -type DeleteApipaEndpointRequest struct { +// DeleteHostNCApipaEndpointRequest describes request for deleting apipa endpoint created +// for host NC connectivity. +type DeleteHostNCApipaEndpointRequest struct { EndpointID string } -// DeleteApipaEndpointResponse describes response for delete apipa endpoint request. -type DeleteApipaEndpointResponse struct { +// DeleteHostNCApipaEndpointResponse describes response for delete host NC apipa endpoint request. +type DeleteHostNCApipaEndpointResponse struct { Response Response } diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index 8c375eb53f..c8f33c9e19 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -9,7 +9,6 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/log" //"github.com/Azure/azure-container-networking/network" - models "github.com/Azure/azure-container-networking/network/models" ) /* @@ -146,75 +145,83 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(orchestratorContext []byte) return &resp, nil } -// CreateApipaEndpoint creates an endpoint in APIPA network for host container connectivity. -func (cnsClient *CNSClient) CreateApipaEndpoint(podName, podNamespace string /*orchestratorContext []byte*/) (*cns.CreateApipaEndpointResponse, error) { - var body bytes.Buffer +// CreateHostNCApipaEndpoint creates an endpoint in APIPA network for host container connectivity. +func (cnsClient *CNSClient) CreateHostNCApipaEndpoint( + networkContainerID string /*podName, podNamespace string*/ /*orchestratorContext []byte*/) (string, error) { + var ( + err error + body bytes.Buffer + ) httpc := &http.Client{} - url := cnsClient.connectionURL + cns.CreateApipaEndpointPath - log.Printf("CreateApipaEndpoint url: %v", url) + url := cnsClient.connectionURL + cns.CreateHostNCApipaEndpointPath + log.Printf("CreateHostNCApipaEndpoint url: %v", url) - podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace} - orchestratorContext, err := json.Marshal(podInfo) - if err != nil { - log.Printf("Failed to marshall podInfo for orchestrator context due to error: %v", err) - return nil, err - } + /* + podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace} + orchestratorContext, err := json.Marshal(podInfo) + if err != nil { + log.Printf("Failed to marshall podInfo for orchestrator context due to error: %v", err) + return "", err + } - // What can be used here? - payload := &cns.CreateApipaEndpointRequest{ - OrchestratorContext: orchestratorContext, + + // TODO: What can be used here? can you pass ncid? + payload := &cns.CreateHostNCApipaEndpointRequest{ + OrchestratorContext: orchestratorContext, + } + */ + + payload := &cns.CreateHostNCApipaEndpointRequest{ + NetworkContainerID: networkContainerID, } - err = json.NewEncoder(&body).Encode(payload) - if err != nil { + if err = json.NewEncoder(&body).Encode(payload); err != nil { log.Errorf("encoding json failed with %v", err) - return nil, err + return "", err } - log.Printf("CreateApipaEndpoint posting body: %v", body) res, err := httpc.Post(url, "application/json", &body) if err != nil { log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) - return nil, err + return "", err } defer res.Body.Close() if res.StatusCode != http.StatusOK { - errMsg := fmt.Sprintf("[Azure CNSClient] CreateEndpointForHostContainerConnectivity: Invalid http status code: %v", + errMsg := fmt.Sprintf("[Azure CNSClient] CreateHostNCApipaEndpoint: Invalid http status code: %v", res.StatusCode) log.Errorf(errMsg) - return nil, fmt.Errorf(errMsg) + return "", fmt.Errorf(errMsg) } - var resp cns.CreateApipaEndpointResponse + var resp cns.CreateHostNCApipaEndpointResponse - err = json.NewDecoder(res.Body).Decode(&resp) - if err != nil { - log.Errorf("[Azure CNSClient] Error parsing CreateEndpointForHostContainerConnectivity response resp: %v err: %v", + if err = json.NewDecoder(res.Body).Decode(&resp); err != nil { + log.Errorf("[Azure CNSClient] Error parsing CreateHostNCApipaEndpoint response resp: %v err: %v", res.Body, err.Error()) - return nil, err + return "", err } if resp.Response.ReturnCode != 0 { - log.Errorf("[Azure CNSClient] CreateEndpointForHostContainerConnectivity received error response :%v", resp.Response.Message) - return nil, fmt.Errorf(resp.Response.Message) + log.Errorf("[Azure CNSClient] CreateHostNCApipaEndpoint received error response :%v", resp.Response.Message) + return "", fmt.Errorf(resp.Response.Message) } - return &resp, nil + return resp.EndpointID, nil } -// DeleteApipaEndpoint deletes the endpoint in APIPA network created for host container connectivity. -func (cnsClient *CNSClient) DeleteApipaEndpoint(endpointID string) error { +// DeleteHostNCApipaEndpoint deletes the endpoint in APIPA network created for host container connectivity. +func (cnsClient *CNSClient) DeleteHostNCApipaEndpoint(endpointID string) error { var body bytes.Buffer // TODO: Move this to create a reusable http client. httpc := &http.Client{} - url := cnsClient.connectionURL + cns.DeleteApipaEndpointPath - log.Printf("DeleteApipaEndpoint url: %v", url) + url := cnsClient.connectionURL + cns.DeleteHostNCApipaEndpointPath + log.Printf("DeleteHostNCApipaEndpoint url: %v", url) - payload := &cns.DeleteApipaEndpointRequest{ + payload := &cns.DeleteHostNCApipaEndpointRequest{ EndpointID: endpointID, } @@ -224,7 +231,6 @@ func (cnsClient *CNSClient) DeleteApipaEndpoint(endpointID string) error { return err } - log.Printf("DeleteApipaEndpoint posting body: %v", body) res, err := httpc.Post(url, "application/json", &body) if err != nil { log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) @@ -234,131 +240,25 @@ func (cnsClient *CNSClient) DeleteApipaEndpoint(endpointID string) error { defer res.Body.Close() if res.StatusCode != http.StatusOK { - errMsg := fmt.Sprintf("[Azure CNSClient] DeleteApipaEndpoint: Invalid http status code: %v", + errMsg := fmt.Sprintf("[Azure CNSClient] DeleteHostNCApipaEndpoint: Invalid http status code: %v", res.StatusCode) log.Errorf(errMsg) return fmt.Errorf(errMsg) } - var resp cns.DeleteApipaEndpointResponse + var resp cns.DeleteHostNCApipaEndpointResponse err = json.NewDecoder(res.Body).Decode(&resp) if err != nil { - log.Errorf("[Azure CNSClient] Error parsing DeleteApipaEndpoint response resp: %v err: %v", + log.Errorf("[Azure CNSClient] Error parsing DeleteHostNCApipaEndpoint response resp: %v err: %v", res.Body, err.Error()) return err } if resp.Response.ReturnCode != 0 { - log.Errorf("[Azure CNSClient] DeleteApipaEndpoint received error response :%v", resp.Response.Message) + log.Errorf("[Azure CNSClient] DeleteHostNCApipaEndpoint received error response :%v", resp.Response.Message) return fmt.Errorf(resp.Response.Message) } return nil } - -// CreateNetwork creates the network. -func (cnsClient *CNSClient) CreateNetwork( - networkInfo models.NetworkInfo, - extIf models.ExternalInterface) /*network.network, - this might need to be Network to be xported*/ error { - var ( - body bytes.Buffer - err error - ) - - httpc := &http.Client{} - url := cnsClient.connectionURL + cns.CreateNewNetworkPath - log.Printf("CreateNewNetworkPath url: %v", url) - - payload := &cns.CreateNewNetworkRequest{ - NetworkInfo: networkInfo, - ExternalInterface: extIf, - } - - if err = json.NewEncoder(&body).Encode(payload); err != nil { - log.Errorf("encoding json failed with %v", err) - return err - } - - res, err := httpc.Post(url, "application/json", &body) - if err != nil { - log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) - return err - } - - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - errMsg := fmt.Sprintf("[Azure CNSClient] CreateNetwork: Invalid http status code: %v", res.StatusCode) - log.Errorf(errMsg) - return fmt.Errorf(errMsg) - } - - var resp cns.Response - - if err = json.NewDecoder(res.Body).Decode(&resp); err != nil { - log.Errorf("[Azure CNSClient] Error parsing CreateNetwork response resp: %v err: %v", - res.Body, err.Error()) - return err - } - - if resp.ReturnCode != 0 { - log.Errorf("[Azure CNSClient] CreateNetwork received error response :%v", resp.Message) - return fmt.Errorf(resp.Message) - } - - return nil -} - -// CreateEndpoint creates an endpoint. -func (cnsClient *CNSClient) CreateEndpoint( - endpointInfo models.EndpointInfo) /*network.endpoint,*/ (*cns.CreateApipaEndpointResponse, error) { - var ( - body bytes.Buffer - err error - ) - - httpc := &http.Client{} - url := cnsClient.connectionURL + cns.CreateNewEndpointPath - log.Printf("CreateEndpoint url: %v", url) - - payload := &cns.CreateNewEndpointRequest{ - EndpointInfo: endpointInfo, - } - - err = json.NewEncoder(&body).Encode(payload) - if err != nil { - log.Errorf("encoding json failed with %v", err) - return nil, err - } - - res, err := httpc.Post(url, "application/json", &body) - if err != nil { - log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) - return nil, err - } - - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - errMsg := fmt.Sprintf("[Azure CNSClient] CreateEndpoint: Invalid http status code: %v", - res.StatusCode) - log.Errorf(errMsg) - return nil, fmt.Errorf(errMsg) - } - - var resp cns.CreateApipaEndpointResponse - - if err = json.NewDecoder(res.Body).Decode(&resp); err != nil { - log.Errorf("[Azure CNSClient] Error parsing CreateEndpoint response resp: %v err: %v", - res.Body, err.Error()) - return nil, err - } - - if resp.Response.ReturnCode != 0 { - log.Errorf("[Azure CNSClient] CreateEndpoint received error response :%v", resp.Response.Message) - return nil, fmt.Errorf(resp.Response.Message) - } - - return &resp, nil -} diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 49375ddb80..9a4c862fce 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -588,8 +588,8 @@ func configureApipaEndpoint( } //TODO: lock -// CreateApipaEndpoint creates the endpoint in the apipa network for host container connectivity -func CreateApipaEndpoint(localIPConfiguration cns.IPConfiguration) (string, error) { +// CreateHostNCApipaEndpoint creates the endpoint in the apipa network for host container connectivity +func CreateHostNCApipaEndpoint(localIPConfiguration cns.IPConfiguration) (string, error) { var ( apipaNetwork *hcn.HostComputeNetwork apipaEndpoint *hcn.HostComputeEndpoint @@ -621,10 +621,9 @@ func CreateApipaEndpoint(localIPConfiguration cns.IPConfiguration) (string, erro } //TODO: lock -// DeleteApipaEndpoint deletes the endpoint in the apipa network created for host <-> container connectivity -// Can this be generalized to createEndpoint / DeleteEndpoint - which can used by general CNI calls -// If you don't delete this APIPA network / if VM gets rebooted, how will you clean this upon restart? -func DeleteApipaEndpoint(endpointID string) error { +// DeleteApipaEndpoint deletes the endpoint in the apipa network created for host container connectivity +// TODO: If you don't delete this APIPA network / if VM gets rebooted, how will you clean this upon restart? +func DeleteHostNCApipaEndpoint(endpointID string) error { var ( apipaEndpoint *hcn.HostComputeEndpoint err error @@ -635,7 +634,7 @@ func DeleteApipaEndpoint(endpointID string) error { // If error is anything other than EndpointNotFoundError, return error. // else log the error but don't return error because endpoint is already deleted. if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { - return fmt.Errorf("[Azure CNS] ERROR: DeleteApipaEndpoint failed due to "+ + return fmt.Errorf("[Azure CNS] ERROR: DeleteHostNCApipaEndpoint failed due to "+ "error with GetEndpointByName: %v", err) } @@ -651,7 +650,7 @@ func DeleteApipaEndpoint(endpointID string) error { return err } - log.Debugf("[Azure CNS] Successfully deleted endpoint: %v", apipaNetworkName) + log.Debugf("[Azure CNS] Successfully deleted apipa endpoint: %v", apipaNetworkName) var endpoints []hcn.HostComputeEndpoint // Check if the network has any endpoints left diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index aa5f75885b..0d0a2c3d17 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -159,8 +159,8 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.CreateHnsNetworkPath, service.createHnsNetwork) listener.AddHandler(cns.DeleteHnsNetworkPath, service.deleteHnsNetwork) listener.AddHandler(cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores) - listener.AddHandler(cns.CreateApipaEndpointPath, service.createApipaEndpoint) - listener.AddHandler(cns.DeleteApipaEndpointPath, service.deleteApipaEndpoint) + listener.AddHandler(cns.CreateHostNCApipaEndpointPath, service.createHostNCApipaEndpoint) + listener.AddHandler(cns.DeleteHostNCApipaEndpointPath, service.deleteHostNCApipaEndpoint) listener.AddHandler(cns.CreateNewEndpointPath, service.createNewEndpoint) listener.AddHandler(cns.CreateNewNetworkPath, service.createNewNetwork) @@ -184,8 +184,8 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.V2Prefix+cns.CreateHnsNetworkPath, service.createHnsNetwork) listener.AddHandler(cns.V2Prefix+cns.DeleteHnsNetworkPath, service.deleteHnsNetwork) listener.AddHandler(cns.V2Prefix+cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores) - listener.AddHandler(cns.V2Prefix+cns.CreateApipaEndpointPath, service.createApipaEndpoint) - listener.AddHandler(cns.V2Prefix+cns.DeleteApipaEndpointPath, service.deleteApipaEndpoint) + listener.AddHandler(cns.V2Prefix+cns.CreateHostNCApipaEndpointPath, service.createHostNCApipaEndpoint) + listener.AddHandler(cns.V2Prefix+cns.DeleteHostNCApipaEndpointPath, service.deleteHostNCApipaEndpoint) log.Printf("[Azure CNS] Listening.") return nil @@ -1218,6 +1218,7 @@ func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkCo savedReq := containerDetails.CreateNetworkContainerRequest getNetworkContainerResponse = cns.GetNetworkContainerResponse{ + NetworkContainerID: savedReq.NetworkContainerid, IPConfiguration: savedReq.IPConfiguration, Routes: savedReq.Routes, CnetAddressSpace: savedReq.CnetAddressSpace, @@ -1622,14 +1623,23 @@ func (service *HTTPRestService) getNumberOfCPUCores(w http.ResponseWriter, r *ht log.Response(service.Name, numOfCPUCoresResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } -func (service *HTTPRestService) createApipaEndpoint(w http.ResponseWriter, r *http.Request) { - log.Printf("[Azure-CNS] createApipaEndpoint") +func (service *HTTPRestService) getNetworkContainerDetails(networkContainerID string) (containerStatus, bool) { + service.lock.Lock() + defer service.lock.Unlock() + + containerDetails, containerExists := service.state.ContainerStatus[networkContainerID] + + return containerDetails, containerExists +} + +func (service *HTTPRestService) createHostNCApipaEndpoint(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure-CNS] createHostNCApipaEndpoint") var ( - returnCode int err error + req cns.CreateHostNCApipaEndpointRequest + returnCode int returnMessage string - req cns.CreateApipaEndpointRequest endpointID string ) @@ -1641,52 +1651,55 @@ func (service *HTTPRestService) createApipaEndpoint(w http.ResponseWriter, r *ht switch r.Method { case "POST": - // Get the NC goal state from the NC identifier passed in request /* - if req.OptionsNCIdentifier != nil { - if _, ok := req.OptionsNCIdentifier[OptOrchContext]; ok { - enableSnat = false - } - } - */ - var req2 cns.GetNetworkContainerRequest req2.NetworkContainerid = req.NetworkContainerid req2.OrchestratorContext = req.OrchestratorContext networkContainerGoalState := service.getNetworkContainerResponse(req2) log.Printf("[tempdebug] restServer: networkContainerGoalState: %+v", networkContainerGoalState) - if endpointID, err = hnsclient.CreateApipaEndpoint(networkContainerGoalState.LocalIPConfiguration); err != nil { - returnMessage = fmt.Sprintf("createApipaEndpoint failed with error: %v", err) + if endpointID, err = hnsclient.CreateHostNCApipaEndpoint(networkContainerGoalState.LocalIPConfiguration); err != nil { + returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: %v", err) returnCode = UnexpectedError } + */ + networkContainerDetails, found := service.getNetworkContainerDetails(req.NetworkContainerID) + if found { + if endpointID, err = hnsclient.CreateHostNCApipaEndpoint( + networkContainerDetails.CreateNetworkContainerRequest.LocalIPConfiguration); err != nil { + returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: %v", err) + returnCode = UnexpectedError + } + } else { + returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: Unable to find goal state for" + + " Network Container: %s", req.NetworkContainerID) + returnCode = UnknownContainerID + } default: - returnMessage = "createApipaEndpoint API expects a POST" + returnMessage = "createHostNCApipaEndpoint API expects a POST" returnCode = UnsupportedVerb } - resp := cns.Response{ - ReturnCode: returnCode, - Message: returnMessage, - } - - createApipaEndpointResp := cns.CreateApipaEndpointResponse{ - Response: resp, + response := cns.CreateHostNCApipaEndpointResponse{ + Response: cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + }, EndpointID: endpointID, } - log.Printf("[tempdebug] createApipaEndpointResp: %+v", createApipaEndpointResp) + log.Printf("[tempdebug] CreateHostNCApipaEndpointResponse: %+v", response) - err = service.Listener.Encode(w, &createApipaEndpointResp) - log.Response(service.Name, createApipaEndpointResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) + err = service.Listener.Encode(w, &response) + log.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err) } -func (service *HTTPRestService) deleteApipaEndpoint(w http.ResponseWriter, r *http.Request) { - log.Printf("[Azure-CNS] DeleteApipaEndpoint") +func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure-CNS] deleteHostNCApipaEndpoint") var ( - returnCode int err error + req cns.DeleteHostNCApipaEndpointRequest + returnCode int returnMessage string - req cns.DeleteApipaEndpointRequest ) err = service.Listener.Decode(w, r, &req) @@ -1697,27 +1710,25 @@ func (service *HTTPRestService) deleteApipaEndpoint(w http.ResponseWriter, r *ht switch r.Method { case "POST": - if err = hnsclient.DeleteApipaEndpoint(req.EndpointID); err != nil { + if err = hnsclient.DeleteHostNCApipaEndpoint(req.EndpointID); err != nil { returnMessage = fmt.Sprintf("Failed to delete endpoint: %s due to error: %v", req.EndpointID, err) returnCode = UnexpectedError } default: - returnMessage = "deleteApipaEndpoint API expects a DELETE" + returnMessage = "deleteHostNCApipaEndpoint API expects a DELETE" returnCode = UnsupportedVerb } - resp := cns.Response{ - ReturnCode: returnCode, - Message: returnMessage, - } - - deleteApipaEndpointResp := cns.DeleteApipaEndpointResponse{ - Response: resp, + response := cns.DeleteHostNCApipaEndpointResponse{ + Response: cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + } } - log.Printf("[tempdebug] deleteApipaEndpointResp: %+v", deleteApipaEndpointResp) + log.Printf("[tempdebug] deleteHostNCApipaEndpointResponse: %+v", response) - err = service.Listener.Encode(w, &deleteApipaEndpointResp) - log.Response(service.Name, deleteApipaEndpointResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) + err = service.Listener.Encode(w, &response) + log.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err) } func (service *HTTPRestService) createNewNetwork(w http.ResponseWriter, r *http.Request) { diff --git a/network/endpoint.go b/network/endpoint.go index 84cdf41c5e..9ccdb5fe8d 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -65,13 +65,14 @@ type EndpointInfo struct { EnableMultiTenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool + NetworkID string //TODO: check if this needed + NetworkContainerID string HostNCApipaEndpointID string PODName string PODNameSpace string Data map[string]interface{} InfraVnetAddressSpace string SkipHotAttachEp bool - NetworkID string } // RouteInfo contains information about an IP route. @@ -214,6 +215,7 @@ func (ep *endpoint) getInfo() *EndpointInfo { PODName: ep.PODName, PODNameSpace: ep.PODNameSpace, HostNCApipaEndpointID: ep.HostNCApipaEndpointID, + NetworkContainerID: ep.NetworkContainerID, } for _, route := range ep.Routes { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 272aafe325..6d52194044 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -11,7 +11,6 @@ import ( "github.com/Azure/azure-container-networking/cns/cnsclient" "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/network/models" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" @@ -553,69 +552,20 @@ func (nw *network) deleteApipaEndpoint(endpointID string) error { return nil } -// newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 -func (nw *network) createApipaEndpoint(epInfo *EndpointInfo) (string, error) { - - var endpointID string - // TODO: cnsclient shouldn't be here. Need to move and encap this somewhere else. - cnsClient, err := cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg - if err != nil { - log.Errorf("Initializing CNS client error %v", err) - return endpointID, err - } - - // TODO: need to safeguard against repeatitive calls to ADD - // TODO: need to pass orch context to createApipaEnpoint - need to get it thru epInfo - //createApipaEndpoint2(epInfo.PODName, epInfo.PODNameSpace) - // TODO: need to add the following condition: - /* - if !nwCfg.EnableExactMatchForPodName { - podNameWithoutSuffix = network.GetPodNameWithoutSuffix(podName) - } else { - podNameWithoutSuffix = podName - } - */ - - resp, err := cnsClient.CreateApipaEndpoint(epInfo.PODName, epInfo.PODNameSpace) - if err != nil { - return endpointID, fmt.Errorf("Failed to create apipa endpoint due to error: %v", err) - } - - if resp.Response.ReturnCode != 0 { - return endpointID, fmt.Errorf("Failed to create apipa endpoint due to error: %s", resp.Response.Message) - } - - endpointID = resp.EndpointID - - defer func() { - if err != nil { - log.Printf("[net] Deleting hcn endpoint with id: %s", endpointID) - // TODO: when this becomes generic, localIP can be passed to delete the endpoint - err = cnsClient.DeleteApipaEndpoint(endpointID) - log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", endpointID, err) - } - }() - - var namespace *hcn.HostComputeNamespace - if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { - return endpointID, fmt.Errorf("Failed to create apipa endpoint due to GetNamespaceByID NetNsPath: %s error: %v", epInfo.NetNsPath, err) - } - - if err = hcn.AddNamespaceEndpoint(namespace.Id, endpointID); err != nil { - return endpointID, fmt.Errorf("[net] Failed to add apipa endpoint: %s to namespace: %s due to error: %v", - endpointID, namespace.Id, err) - } - - return endpointID, nil -} - func (nw *network) createHostNCApipaEndpoint(epInfo *EndpointInfo) error { var ( - hostNCApipaNetwork *hcn.HostComputeNetwork - err error - cnsClient *cnsclient.CNSClient + err error + cnsClient *cnsclient.CNSClient + hostNCApipaEndpointID string + hostNCApipaNetwork *hcn.HostComputeNetwork + namespace *hcn.HostComputeNamespace ) + if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { + return fmt.Errorf("Failed to retrieve namespace with GetNamespaceByID for NetNsPath: %s"+ + " due to error: %v", epInfo.NetNsPath, err) + } + // TODO: This should be changed to GetCnsClient() cnsClient, err = cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg if err != nil { @@ -623,105 +573,28 @@ func (nw *network) createHostNCApipaEndpoint(epInfo *EndpointInfo) error { return err // upfate this to meaningful error } - // Check if the HostNCApipaNetwork exists - if hostNCApipaNetwork, err = hcn.GetNetworkByName(HostNCApipaNetworkName); err != nil { - // If error is anything other than networkNotFound, mark this as error - // TODO: why is following part not working? - /* - if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { - return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed due to error with GetNetworkByName: %v", err) - } - */ - - // HostNCApipaNetwork network doesn't exist. Create the network. - // Create the network. - /* - nwInfo := network.NetworkInfo{ - Id: networkId, - Mode: nwCfg.Mode, - MasterIfName: masterIfName, - Subnets: []network.SubnetInfo{ - network.SubnetInfo{ - Family: platform.AfINET, - Prefix: subnetPrefix, - Gateway: gateway, - }, - }, - BridgeName: nwCfg.Bridge, - EnableSnatOnHost: nwCfg.EnableSnatOnHost, - DNS: nwDNSInfo, - Policies: policies, - NetNs: args.Netns, - } - */ - networkInfo := models.NetworkInfo{ - Id: HostNCApipaNetworkName, - Mode: opModeBridge, - } - - // Set options to indicate there is no external interface and needs loopback adapter creation - - log.Printf("[net] Creating HostNCApipaNetwork for host container connectivity: %+v", networkInfo) - - if err = cnsClient.CreateNetwork(networkInfo, models.ExternalInterface{}); err != nil { - return fmt.Errorf("Failed to create HostNCApipaNetworkName due to error: %v", err) - } - - log.Printf("[net] Successfully created HostNCApipaNetworkName for host container connectivity") - } else { - log.Printf("[net] Found existing hostNCApipaNetwork: %+v", hostNCApipaNetwork) - } + log.Printf("[Azure CNS] Creating endpoint for host container connectivity") - // Create endpoint in the network for host container connectivity - // TODO: use the returned info by createNetwork to get the network id instead of following call. - if hostNCApipaNetwork, err = hcn.GetNetworkByName(HostNCApipaNetworkName); err != nil { - log.Printf("[Azure CNS] tempdebug: something is wrong. Error: %v", err) + if hostNCApipaEndpointID, err = + cnsClient.CreateHostNCApipaEndpoint(epInfo.NetworkContainerID /*epInfo.PODName, epInfo.PODNameSpace*/); err != nil { return err } - ///////////////////////////////////////////////////////////////////////////////// - - epInfo.NetworkID = hostNCApipaNetwork.Id - - epInfo2 := models.EndpointInfo{ - NetworkID: epInfo.NetworkID, - PODName: epInfo.PODName, - PODNameSpace: epInfo.PODNameSpace, - } - // Dup models.EndpointInfo - - log.Printf("[Azure CNS] tempdebug: creating endpoint with Info: %+v", epInfo) - - resp, err := cnsClient.CreateEndpoint(epInfo2) - if err != nil { - return fmt.Errorf("Failed to create apipa endpoint due to error: %v", err) - } - - if resp.Response.ReturnCode != 0 { - return fmt.Errorf("Failed to create apipa endpoint due to error: %s", resp.Response.Message) - } - - endpointID := resp.EndpointID defer func() { if err != nil { - log.Printf("[net] Deleting hcn endpoint with id: %s", endpointID) + log.Printf("[net] Deleting hcn endpoint with id: %s", hostNCApipaEndpointID) // TODO: when this becomes generic, localIP can be passed to delete the endpoint - err = cnsClient.DeleteApipaEndpoint(endpointID) - log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", endpointID, err) + err = cnsClient.DeleteHostNCApipaEndpoint(hostNCApipaEndpointID) + log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", hostNCApipaEndpointID, err) } }() - var namespace *hcn.HostComputeNamespace - if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { - return fmt.Errorf("Failed to create apipa endpoint due to GetNamespaceByID NetNsPath: %s error: %v", epInfo.NetNsPath, err) - } - - if err = hcn.AddNamespaceEndpoint(namespace.Id, endpointID); err != nil { - return fmt.Errorf("[net] Failed to add apipa endpoint: %s to namespace: %s due to error: %v", - endpointID, namespace.Id, err) + if err = hcn.AddNamespaceEndpoint(namespace.Id, hostNCApipaEndpointID); err != nil { + return fmt.Errorf("[net] Failed to add HostNCApipaEndpoint: %s to namespace: %s due to error: %v", + hostNCApipaEndpointID, namespace.Id, err) } - epInfo.HostNCApipaEndpointID = endpointID + epInfo.HostNCApipaEndpointID = hostNCApipaEndpointID return nil } @@ -773,6 +646,7 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) // If the host <-> container connectivity is requested, create endpoint in HostNCApipaNetwork //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { if err = nw.createHostNCApipaEndpoint(epInfo); err != nil { + return nil, fmt.Errorf("Failed to create HostNCApipaEndpoint due to error: %v", err) // TODO: delete the endpoint created above and return appropriate error. } //} From c4cbbe1237b385c7a2f3c26f6bd72e69b822108c Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Sun, 13 Oct 2019 15:48:04 -0700 Subject: [PATCH 26/37] WIP-10-13-1 --- cns/api.go | 4 - cns/hnsclient/hnsclient_windows.go | 381 ++++++++++------------------- cns/restserver/restserver.go | 138 ++--------- network/endpoint.go | 1 + network/endpoint_windows.go | 10 +- 5 files changed, 154 insertions(+), 380 deletions(-) diff --git a/cns/api.go b/cns/api.go index 4d65391e77..8fcbe4c03c 100644 --- a/cns/api.go +++ b/cns/api.go @@ -25,10 +25,6 @@ const ( NumberOfCPUCoresPath = "/hostcpucores" CreateHostNCApipaEndpointPath = "/network/createhostncapipaendpoint" DeleteHostNCApipaEndpointPath = "/network/deletehostncapipaendpoint" - CreateNewNetworkPath = "/network/createnewnetwork" - DeleteNewNetworkPath = "/network/deletenewnetwork" - CreateNewEndpointPath = "/network/createnewendpoint" - DeleteNewEndpointPath = "/network/deletenewendpoint" V1Prefix = "/v0.1" V2Prefix = "/v0.2" diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 9a4c862fce..fcdea8c87b 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -12,7 +12,6 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/network/models" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" "github.com/Microsoft/hcsshim/hcn" @@ -41,11 +40,18 @@ const ( // hcnIpamTypeStatic indicates the static type of ipam hcnIpamTypeStatic = "Static" - // apipaNetworkName indicates the name of the apipa network used for host container connectivity - apipaNetworkName = "apipa-network" + // hostNCApipaNetworkName indicates the name of the apipa network used for host container connectivity + hostNCApipaNetworkName = "HostNCApipaNetwork" - // apipaEndpointName indicates the name of the apipa endpoint used for host container connectivity - apipaEndpointName = "apipa-endpoint" + // hostNCApipaNetworkType indicates the type of hns network setup for host NC connectivity + hostNCApipaNetworkType = hcn.L2Bridge + + // hostNCApipaEndpointName indicates the prefix for the name of the apipa endpoint used for + // the host container connectivity + hostNCApipaEndpointNamePrefix = "HostNCApipaEndpoint" + + // Name of the loopback adapter needed to create Host NC apipa network + hostNCLoopbackAdapterName = "LoopbackAdapterHostNCConnectivity" ) // CreateHnsNetwork creates the HNS network with the provided configuration @@ -177,9 +183,9 @@ func deleteHnsNetwork(networkName string) error { return err } -func configureApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeNetwork, error) { - apipaNetwork := &hcn.HostComputeNetwork{ - Name: apipaNetworkName, +func configureHostNCApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeNetwork, error) { + network := &hcn.HostComputeNetwork{ + Name: hostNCApipaNetworkName, Ipams: []hcn.Ipam{ hcn.Ipam{ Type: hcnIpamTypeStatic, @@ -189,28 +195,29 @@ func configureApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostC Major: hcnSchemaVersionMajor, Minor: hcnSchemaVersionMinor, }, - Type: hcn.L2Bridge, + Type: hostNCApipaNetworkType, } - // TODO: How to get this string from the created loopback adapter? - // TODO: Create the loopback adapter using the LocalIPConfiguration passed in. - if netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy("Ethernet 6"); err == nil { - apipaNetwork.Policies = append(apipaNetwork.Policies, netAdapterNamePolicy) + if netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy(hostNCLoopbackAdapterName); err == nil { + network.Policies = append(network.Policies, netAdapterNamePolicy) } else { - log.Errorf("[Azure CNS] Failed to serialize network adapter policy due to error: %v", err) - return nil, err + return nil, fmt.Errorf("Failed to serialize network adapter policy. Error: %v", err) } // Calculate subnet prefix - var subnetPrefix net.IPNet - var subnetPrefixStr string - ipAddr := net.ParseIP(localIPConfiguration.IPSubnet.IPAddress) + var ( + subnetPrefix net.IPNet + subnetPrefixStr string + ipAddr net.IP + ) + + ipAddr = net.ParseIP(localIPConfiguration.IPSubnet.IPAddress) if ipAddr.To4() != nil { subnetPrefix = net.IPNet{Mask: net.CIDRMask(int(localIPConfiguration.IPSubnet.PrefixLength), 32)} } else if ipAddr.To16() != nil { subnetPrefix = net.IPNet{Mask: net.CIDRMask(int(localIPConfiguration.IPSubnet.PrefixLength), 128)} } else { - return nil, fmt.Errorf("[Azure CNS] Failed get subnet prefix for localIPConfiguration: %+v", localIPConfiguration) + return nil, fmt.Errorf("Failed get subnet prefix for localIPConfiguration: %+v", localIPConfiguration) } subnetPrefix.IP = ipAddr.Mask(subnetPrefix.Mask) @@ -218,7 +225,6 @@ func configureApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostC log.Printf("[tempdebug] configureApipaNetwork: subnetPrefixStr: %s, GW: %s", subnetPrefixStr, localIPConfiguration.GatewayIPAddress) subnet := hcn.Subnet{ - //IpAddressPrefix: "169.254.0.0/16", // TODO: this needs be calculated from LocalIPConfiguration passed in IpAddressPrefix: subnetPrefixStr, Routes: []hcn.Route{ hcn.Route{ @@ -228,57 +234,70 @@ func configureApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostC }, } - apipaNetwork.Ipams[0].Subnets = append(apipaNetwork.Ipams[0].Subnets, subnet) + network.Ipams[0].Subnets = append(network.Ipams[0].Subnets, subnet) - return apipaNetwork, nil + return network, nil } -func createApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeNetwork, error) { +func createHostNCApipaNetwork( + localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeNetwork, error) { var ( - apipaNetwork *hcn.HostComputeNetwork - err error + network *hcn.HostComputeNetwork + err error ) - // Check if the APIPA network exists - if apipaNetwork, err = hcn.GetNetworkByName(apipaNetworkName); err != nil { + // Check if the network exists for host NC connectivity + if network, err = hcn.GetNetworkByName(hostNCApipaNetworkName); err != nil { // If error is anything other than networkNotFound, mark this as error // TODO: why is following part not working? /* if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { - return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed due to error with GetNetworkByName: %v", err) + return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed. Error with GetNetworkByName: %v", err) } */ - // APIPA network doesn't exist. Create one. - if apipaNetwork, err = configureApipaNetwork(localIPConfiguration); err != nil { - log.Printf("[Azure CNS] Failed to configure apipa network due to error: %v", err) - return nil, err + // Network doesn't exist. Create one. + if network, err = configureHostNCApipaNetwork(localIPConfiguration); err != nil { + return nil, fmt.Errorf("Failed to configure network. Error: %v", err) + } + + // Create loopback adapter needed for this HNS network + if networkExists, _ := interfaceExists(hostNCLoopbackAdapterName); !networkExists { + ipconfig := cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: localIPConfiguration.GatewayIPAddress, + PrefixLength: localIPConfiguration.IPSubnet.PrefixLength, + }, + GatewayIPAddress: localIPConfiguration.GatewayIPAddress, + } + + if err = createLoopbackAdapter(hostNCLoopbackAdapterName, ipconfig); err != nil { + return nil, fmt.Errorf("Failed to create loopback adapter. Error: %v", err) + } } // Create the HNS network. - log.Printf("[net] Creating apipa network: %+v", apipaNetwork) - apipaNetwork, err = apipaNetwork.Create() + log.Printf("[Azure CNS] Creating HostNCApipaNetwork: %+v", network) - if err != nil { - log.Printf("[net] Failed to create apipa network due to error: %v", err) - return nil, fmt.Errorf("Failed to create apipa network: %s due to error: %v", apipaNetwork.Name, err) + if network, err = network.Create(); err != nil { + return nil, err } - log.Printf("[net] Successfully created apipa network for host container connectivity: %+v", apipaNetwork) + log.Printf("[Azure CNS] Successfully created apipa network for host container connectivity: %+v", network) } else { - log.Printf("[Azure CNS] Found existing APIPA network: %+v", apipaNetwork) + log.Printf("[Azure CNS] Found existing HostNCApipaNetwork: %+v", network) } - return apipaNetwork, err + return network, err } -func configureApipaEndpoint( - apipaNetworkID string, +func configureHostNCApipaEndpoint( + endpointName string, + networkID string, localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeEndpoint, error) { - //log.Printf("[tempdebug] configureApipaEndpoint ID: %+v", apipaNetwork) - apipaEndpoint := &hcn.HostComputeEndpoint{ - Name: apipaEndpointName, - HostComputeNetwork: apipaNetworkID, + endpoint := &hcn.HostComputeEndpoint{ + Name: endpointName, + HostComputeNetwork: networkID, SchemaVersion: hcn.SchemaVersion{ Major: hcnSchemaVersionMajor, Minor: hcnSchemaVersionMinor, @@ -287,7 +306,7 @@ func configureApipaEndpoint( localIP := localIPConfiguration.IPSubnet.IPAddress remoteIP := localIPConfiguration.GatewayIPAddress - log.Printf("[tempdebug] configureApipaEndpoint localIP: %s, remoteIP: %s", localIP, remoteIP) + log.Printf("[tempdebug] configureHostNCApipaEndpoint localIP: %s, remoteIP: %s", localIP, remoteIP) /********************************************************************************************************/ // Add ICMP ACLs { @@ -311,7 +330,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) // Add endpoint ACL for allowing out to host apipa aclOutAllowToHostOnly := hcn.AclPolicySetting{ @@ -334,7 +353,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) // Add endpoint ACL for allowing in from host apipa aclInBlockAll := hcn.AclPolicySetting{ @@ -356,7 +375,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) // Add endpoint ACL for allowing in from host apipa aclInAllowFromHostOnly := hcn.AclPolicySetting{ @@ -379,7 +398,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) } // Add TCP ACLs @@ -404,7 +423,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) // Add endpoint ACL for allowing out to host apipa aclOutAllowToHostOnly := hcn.AclPolicySetting{ @@ -427,7 +446,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) // Add endpoint ACL for allowing in from host apipa aclInBlockAll := hcn.AclPolicySetting{ @@ -449,7 +468,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) // Add endpoint ACL for allowing in from host apipa aclInAllowFromHostOnly := hcn.AclPolicySetting{ @@ -472,7 +491,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) } // Add UDP ACLs @@ -497,7 +516,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) // Add endpoint ACL for allowing out to host apipa aclOutAllowToHostOnly := hcn.AclPolicySetting{ @@ -520,7 +539,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) // Add endpoint ACL for allowing in from host apipa aclInBlockAll := hcn.AclPolicySetting{ @@ -542,7 +561,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) // Add endpoint ACL for allowing in from host apipa aclInAllowFromHostOnly := hcn.AclPolicySetting{ @@ -565,7 +584,7 @@ func configureApipaEndpoint( Settings: rawJSON, } - apipaEndpoint.Policies = append(apipaEndpoint.Policies, endpointPolicy) + endpoint.Policies = append(endpoint.Policies, endpointPolicy) } /********************************************************************************************************/ @@ -575,62 +594,84 @@ func configureApipaEndpoint( DestinationPrefix: "0.0.0.0/0", } - apipaEndpoint.Routes = append(apipaEndpoint.Routes, hcnRoute) + endpoint.Routes = append(endpoint.Routes, hcnRoute) ipConfiguration := hcn.IpConfig{ IpAddress: localIP, PrefixLength: localIPConfiguration.IPSubnet.PrefixLength, // TODO: this should come from the cns config } - apipaEndpoint.IpConfigurations = append(apipaEndpoint.IpConfigurations, ipConfiguration) + endpoint.IpConfigurations = append(endpoint.IpConfigurations, ipConfiguration) + + return endpoint, nil +} - return apipaEndpoint, nil +func getHostNCApipaEndpointName( + networkContainerID string) string { + return hostNCApipaEndpointNamePrefix + "-" + networkContainerID } //TODO: lock // CreateHostNCApipaEndpoint creates the endpoint in the apipa network for host container connectivity -func CreateHostNCApipaEndpoint(localIPConfiguration cns.IPConfiguration) (string, error) { +func CreateHostNCApipaEndpoint( + networkContainerID string, + localIPConfiguration cns.IPConfiguration) (string, error) { var ( - apipaNetwork *hcn.HostComputeNetwork - apipaEndpoint *hcn.HostComputeEndpoint - err error + network *hcn.HostComputeNetwork + endpoint *hcn.HostComputeEndpoint + endpointName = getHostNCApipaEndpointName(networkContainerID) + err error ) - //TODO: check if the endpoint exists - if apipaNetwork, err = createApipaNetwork(localIPConfiguration); err != nil { - log.Errorf("[Azure CNS] Failed to create apipa network for host container connectivity due to error: %v", err) + // Return if the endpoint already exists + if endpoint, err = hcn.GetEndpointByName(endpointName); err != nil { + // TODO: these are failing due to hcn bug https://github.com/microsoft/hcsshim/pull/519/files + // If error is anything other than EndpointNotFoundError, return error. + if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { + return "", fmt.Errorf("ERROR: Failed to query endpoint using GetEndpointByName "+ + "due to error: %v", err) + } + } + + if endpoint != nil { + log.Debugf("[Azure CNS] Found existing endpoint: %+v", endpoint) + return endpoint.Id, nil + } + + if network, err = createHostNCApipaNetwork(localIPConfiguration); err != nil { + log.Errorf("[Azure CNS] Failed to create HostNCApipaNetwork. Error: %v", err) return "", err } - if apipaEndpoint, err = configureApipaEndpoint(apipaNetwork.Id, localIPConfiguration); err != nil { - log.Errorf("[Azure CNS] Failed to configure apipa endpoint for host container connectivity due to error: %v", err) + if endpoint, err = configureHostNCApipaEndpoint(endpointName, network.Id, localIPConfiguration); err != nil { + log.Errorf("[Azure CNS] Failed to configure HostNCApipaEndpoint: %s. Error: %v", endpointName, err) return "", err } // Create the apipa endpoint - log.Printf("[Azure CNS] Creating apipa endpoint for host-container connectivity: %+v", apipaEndpoint) - if apipaEndpoint, err = apipaEndpoint.Create(); err != nil { - err = fmt.Errorf("Failed to create apipa endpoint: %s due to error: %v", apipaEndpoint.Name, err) + log.Printf("[Azure CNS] Creating HostNCApipaEndpoint for host container connectivity: %+v", endpoint) + if endpoint, err = endpoint.Create(); err != nil { + err = fmt.Errorf("Failed to create HostNCApipaEndpoint: %s. Error: %v", endpointName, err) log.Errorf("[Azure CNS] %s", err.Error()) return "", err } - log.Printf("[Azure CNS] Successfully created apipa endpoint for host-container connectivity: %+v", apipaEndpoint) + log.Printf("[Azure CNS] Successfully created HostNCApipaEndpoint: %+v", endpoint) - return apipaEndpoint.Id, nil + return endpoint.Id, nil } //TODO: lock -// DeleteApipaEndpoint deletes the endpoint in the apipa network created for host container connectivity +// DeleteHostNCApipaEndpoint deletes the endpoint in the apipa network created for host container connectivity // TODO: If you don't delete this APIPA network / if VM gets rebooted, how will you clean this upon restart? func DeleteHostNCApipaEndpoint(endpointID string) error { var ( - apipaEndpoint *hcn.HostComputeEndpoint - err error + endpoint *hcn.HostComputeEndpoint + err error ) // Check if the endpoint with the provided ID exists - if apipaEndpoint, err = hcn.GetEndpointByID(endpointID); err != nil { + if endpoint, err = hcn.GetEndpointByID(endpointID); err != nil { // If error is anything other than EndpointNotFoundError, return error. // else log the error but don't return error because endpoint is already deleted. if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { @@ -638,39 +679,39 @@ func DeleteHostNCApipaEndpoint(endpointID string) error { "error with GetEndpointByName: %v", err) } - log.Errorf("[Azure CNS] Failed to find endpoint: %s for deletion due to error: %v", endpointID, err) + log.Errorf("[Azure CNS] Failed to find endpoint: %s for deletion. Error: %v", endpointID, err) return nil } - //networkID := apipaEndpoint.HostComputeNetwork - - if err = apipaEndpoint.Delete(); err != nil { - err = fmt.Errorf("Failed to delete endpoint: %+v due to error: %v", apipaEndpoint, err) + if err = endpoint.Delete(); err != nil { + err = fmt.Errorf("Failed to delete endpoint: %+v. Error: %v", endpoint, err) log.Errorf("[Azure CNS] %v", err) return err } - log.Debugf("[Azure CNS] Successfully deleted apipa endpoint: %v", apipaNetworkName) + log.Debugf("[Azure CNS] Successfully deleted apipa endpoint: %v", hostNCApipaNetworkName) var endpoints []hcn.HostComputeEndpoint // Check if the network has any endpoints left - if endpoints, err = hcn.ListEndpointsOfNetwork(apipaEndpoint.HostComputeNetwork); err != nil { - log.Errorf("[Azure CNS] Failed to list endpoints in the network: %s due to error: %v", apipaNetworkName, err) + if endpoints, err = hcn.ListEndpointsOfNetwork(endpoint.HostComputeNetwork); err != nil { + log.Errorf("[Azure CNS] Failed to list endpoints in the network: %s. Error: %v", hostNCApipaNetworkName, err) return nil } // Delete network if it doesn't have any endpoints if len(endpoints) == 0 { - if err = DeleteApipaNetwork(apipaEndpoint.HostComputeNetwork); err == nil { + if err = DeleteApipaNetwork(endpoint.HostComputeNetwork); err == nil { // Delete the loopback adapter created for this network - deleteLoopbackAdapter("LoopbackAdapterHostNCConnectivity") + deleteLoopbackAdapter(hostNCLoopbackAdapterName) } } return nil } -func DeleteApipaNetwork(networkID string) error { +// TODO: this can be a generic function to call hns delete +func DeleteApipaNetwork( + networkID string) error { var ( network *hcn.HostComputeNetwork err error @@ -689,7 +730,7 @@ func DeleteApipaNetwork(networkID string) error { } if err = network.Delete(); err != nil { - err = fmt.Errorf("Failed to delete network: %+v due to error: %v", network, err) + err = fmt.Errorf("Failed to delete network: %+v. Error: %v", network, err) log.Errorf("[Azure CNS] %v", err) return err } @@ -699,162 +740,6 @@ func DeleteApipaNetwork(networkID string) error { return nil } -func CreateNewNetwork( - networkInfo models.NetworkInfo, - extInterface models.ExternalInterface) /**hcn.HostComputeNetwork, replace this by network.network*/ error { - var ( - apipaNetwork *hcn.HostComputeNetwork - err error - ) - - // Check if the APIPA network exists - if apipaNetwork, err = hcn.GetNetworkByName(networkInfo.Id); err != nil { - // If error is anything other than networkNotFound, mark this as error - // TODO: why is following part not working? - /* - if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { - return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed due to error with GetNetworkByName: %v", err) - } - */ - - // APIPA network doesn't exist. Create one. - if apipaNetwork, err = configureApipaNetwork2(networkInfo, extInterface); err != nil { - log.Printf("[Azure CNS] Failed to configure apipa network due to error: %v", err) - return err - } - - // Create the HNS network. - log.Printf("[net] Creating apipa network: %+v", apipaNetwork) - apipaNetwork, err = apipaNetwork.Create() - - if err != nil { - log.Printf("[net] Failed to create apipa network due to error: %v", err) - return fmt.Errorf("Failed to create apipa network: %s due to error: %v", apipaNetwork.Name, err) - } - - log.Printf("[net] Successfully created apipa network for host container connectivity: %+v", apipaNetwork) - } else { - log.Printf("[Azure CNS] Found existing APIPA network: %+v", apipaNetwork) - } - - return nil -} - -func configureApipaNetwork2( - networkInfo models.NetworkInfo, - extInterface models.ExternalInterface) (*hcn.HostComputeNetwork, error) { - // TODO: this needs to be the generic hnsv2 path implementation - apipaNetwork := &hcn.HostComputeNetwork{ - Name: networkInfo.Id, - Ipams: []hcn.Ipam{ - hcn.Ipam{ - Type: hcnIpamTypeStatic, - }, - }, - SchemaVersion: hcn.SchemaVersion{ - Major: hcnSchemaVersionMajor, - Minor: hcnSchemaVersionMinor, - }, - Type: hcn.L2Bridge, - } - - // Create loopback adapter if needed - // TODO: check the settings from the options in networkInfo and create the loopback adapter if needed. - ipconfig := cns.IPConfiguration{ - IPSubnet: cns.IPSubnet{ - IPAddress: "169.254.0.2", - PrefixLength: 16, - }, - GatewayIPAddress: "169.254.0.2", - } - - if exists, _ := interfaceExists("LoopbackAdapterHostNCConnectivity"); !exists { - if err := createLoopbackAdapter("LoopbackAdapterHostNCConnectivity", ipconfig); err != nil { - err = fmt.Errorf("Failed to create loopback adapter for host container connectivity due to error: %v", err) - log.Errorf("[Azure CNS] %v", err) - return nil, err - } - } - - if netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy( /*"Ethernet 6"*/ "LoopbackAdapterHostNCConnectivity"); err == nil { - apipaNetwork.Policies = append(apipaNetwork.Policies, netAdapterNamePolicy) - } else { - log.Errorf("[Azure CNS] Failed to serialize network adapter policy due to error: %v", err) - return nil, err - } - - /* - // Calculate subnet prefix - var subnetPrefix net.IPNet - var subnetPrefixStr string - ipAddr := net.ParseIP(localIPConfiguration.IPSubnet.IPAddress) - if ipAddr.To4() != nil { - subnetPrefix = net.IPNet{Mask: net.CIDRMask(int(localIPConfiguration.IPSubnet.PrefixLength), 32)} - } else if ipAddr.To16() != nil { - subnetPrefix = net.IPNet{Mask: net.CIDRMask(int(localIPConfiguration.IPSubnet.PrefixLength), 128)} - } else { - return nil, fmt.Errorf("[Azure CNS] Failed get subnet prefix for localIPConfiguration: %+v", localIPConfiguration) - } - - subnetPrefix.IP = ipAddr.Mask(subnetPrefix.Mask) - subnetPrefixStr = subnetPrefix.IP.String() + "/" + strconv.Itoa(int(localIPConfiguration.IPSubnet.PrefixLength)) - log.Printf("[tempdebug] configureApipaNetwork: subnetPrefixStr: %s, GW: %s", subnetPrefixStr, localIPConfiguration.GatewayIPAddress) - */ - subnet := hcn.Subnet{ - IpAddressPrefix: "169.254.0.0/16", // TODO: this needs be calculated from LocalIPConfiguration passed in - //IpAddressPrefix: subnetPrefixStr, - Routes: []hcn.Route{ - hcn.Route{ - //NextHop: localIPConfiguration.GatewayIPAddress, - NextHop: "169.254.0.2", - DestinationPrefix: "0.0.0.0/0", - }, - }, - } - - apipaNetwork.Ipams[0].Subnets = append(apipaNetwork.Ipams[0].Subnets, subnet) - - return apipaNetwork, nil -} - -func CreateNewEndpoint( - endpointInfo models.EndpointInfo, - localIPConfiguration cns.IPConfiguration) (string, error) { - var ( - //apipaNetwork *hcn.HostComputeNetwork - apipaEndpoint *hcn.HostComputeEndpoint - err error - ) - - //TODO: this needs to be generic implementation of create endpoint with v2 - - //TODO: check if the endpoint exists - - /* - if apipaNetwork, err = createApipaNetwork(localIPConfiguration); err != nil { - log.Errorf("[Azure CNS] Failed to create apipa network for host container connectivity due to error: %v", err) - return "", err - } - */ - - if apipaEndpoint, err = configureApipaEndpoint(endpointInfo.NetworkID, localIPConfiguration); err != nil { - log.Errorf("[Azure CNS] Failed to configure apipa endpoint for host container connectivity due to error: %v", err) - return "", err - } - - // Create the apipa endpoint - log.Printf("[Azure CNS] Creating apipa endpoint for host-container connectivity: %+v", apipaEndpoint) - if apipaEndpoint, err = apipaEndpoint.Create(); err != nil { - err = fmt.Errorf("Failed to create apipa endpoint: %s due to error: %v", apipaEndpoint.Name, err) - log.Errorf("[Azure CNS] %s", err.Error()) - return "", err - } - - log.Printf("[Azure CNS] Successfully created apipa endpoint for host-container connectivity: %+v", apipaEndpoint) - - return apipaEndpoint.Id, nil -} - func interfaceExists(iFaceName string) (bool, error) { _, err := net.InterfaceByName(iFaceName) if err != nil { diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 0d0a2c3d17..5eb7a388f1 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -161,8 +161,6 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores) listener.AddHandler(cns.CreateHostNCApipaEndpointPath, service.createHostNCApipaEndpoint) listener.AddHandler(cns.DeleteHostNCApipaEndpointPath, service.deleteHostNCApipaEndpoint) - listener.AddHandler(cns.CreateNewEndpointPath, service.createNewEndpoint) - listener.AddHandler(cns.CreateNewNetworkPath, service.createNewNetwork) // handlers for v0.2 listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment) @@ -1218,7 +1216,7 @@ func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkCo savedReq := containerDetails.CreateNetworkContainerRequest getNetworkContainerResponse = cns.GetNetworkContainerResponse{ - NetworkContainerID: savedReq.NetworkContainerid, + NetworkContainerID: savedReq.NetworkContainerid, IPConfiguration: savedReq.IPConfiguration, Routes: savedReq.Routes, CnetAddressSpace: savedReq.CnetAddressSpace, @@ -1623,7 +1621,7 @@ func (service *HTTPRestService) getNumberOfCPUCores(w http.ResponseWriter, r *ht log.Response(service.Name, numOfCPUCoresResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } -func (service *HTTPRestService) getNetworkContainerDetails(networkContainerID string) (containerStatus, bool) { +func (service *HTTPRestService) getNetworkContainerDetails(networkContainerID string) (containerstatus, bool) { service.lock.Lock() defer service.lock.Unlock() @@ -1652,26 +1650,26 @@ func (service *HTTPRestService) createHostNCApipaEndpoint(w http.ResponseWriter, switch r.Method { case "POST": /* - var req2 cns.GetNetworkContainerRequest - req2.NetworkContainerid = req.NetworkContainerid - req2.OrchestratorContext = req.OrchestratorContext - networkContainerGoalState := service.getNetworkContainerResponse(req2) - log.Printf("[tempdebug] restServer: networkContainerGoalState: %+v", networkContainerGoalState) - if endpointID, err = hnsclient.CreateHostNCApipaEndpoint(networkContainerGoalState.LocalIPConfiguration); err != nil { - returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: %v", err) - returnCode = UnexpectedError - } + var req2 cns.GetNetworkContainerRequest + req2.NetworkContainerid = req.NetworkContainerid + req2.OrchestratorContext = req.OrchestratorContext + networkContainerGoalState := service.getNetworkContainerResponse(req2) + log.Printf("[tempdebug] restServer: networkContainerGoalState: %+v", networkContainerGoalState) + if endpointID, err = hnsclient.CreateHostNCApipaEndpoint(networkContainerGoalState.LocalIPConfiguration); err != nil { + returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: %v", err) + returnCode = UnexpectedError + } */ networkContainerDetails, found := service.getNetworkContainerDetails(req.NetworkContainerID) if found { - if endpointID, err = hnsclient.CreateHostNCApipaEndpoint( + if endpointID, err = hnsclient.CreateHostNCApipaEndpoint(req.NetworkContainerID, networkContainerDetails.CreateNetworkContainerRequest.LocalIPConfiguration); err != nil { returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: %v", err) returnCode = UnexpectedError } } else { - returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: Unable to find goal state for" + - " Network Container: %s", req.NetworkContainerID) + returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: Unable to find goal state for"+ + " the given Network Container: %s", req.NetworkContainerID) returnCode = UnknownContainerID } default: @@ -1723,116 +1721,10 @@ func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter, Response: cns.Response{ ReturnCode: returnCode, Message: returnMessage, - } + }, } log.Printf("[tempdebug] deleteHostNCApipaEndpointResponse: %+v", response) err = service.Listener.Encode(w, &response) log.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err) } - -func (service *HTTPRestService) createNewNetwork(w http.ResponseWriter, r *http.Request) { - log.Printf("[Azure-CNS] createNewNetwork") - - var ( - returnCode int - err error - returnMessage string - req cns.CreateNewNetworkRequest - ) - - err = service.Listener.Decode(w, r, &req) - log.Request(service.Name, &req, err) - if err != nil { - return - } - - switch r.Method { - case "POST": - // Get the NC goal state from the NC identifier passed in request - /* - if req.OptionsNCIdentifier != nil { - if _, ok := req.OptionsNCIdentifier[OptOrchContext]; ok { - enableSnat = false - } - } - */ - if err = hnsclient.CreateNewNetwork(req.NetworkInfo, req.ExternalInterface); err != nil { - returnMessage = fmt.Sprintf("CreateNewNetwork failed with error: %v", err) - returnCode = UnexpectedError - } - default: - returnMessage = "CreateNewNetwork API expects a POST" - returnCode = UnsupportedVerb - } - - resp := cns.Response{ - ReturnCode: returnCode, - Message: returnMessage, - } - - log.Printf("[tempdebug] CreateNewNetwork: %+v", resp) - - err = service.Listener.Encode(w, &resp) - log.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) -} - -func (service *HTTPRestService) createNewEndpoint(w http.ResponseWriter, r *http.Request) { - log.Printf("[Azure-CNS] createNewEndpoint") - - var ( - returnCode int - err error - returnMessage string - req cns.CreateNewEndpointRequest - endpointID string - ) - - err = service.Listener.Decode(w, r, &req) - log.Request(service.Name, &req, err) - if err != nil { - return - } - - switch r.Method { - case "POST": - // Get the NC goal state from the NC identifier passed in request - /* - if req.OptionsNCIdentifier != nil { - if _, ok := req.OptionsNCIdentifier[OptOrchContext]; ok { - enableSnat = false - } - } - */ - - podInfo := cns.KubernetesPodInfo{PodName: req.EndpointInfo.PODName, PodNamespace: req.EndpointInfo.PODNameSpace} - orchestratorContext, _ := json.Marshal(podInfo) - - var req2 cns.GetNetworkContainerRequest - //req2.NetworkContainerid = req.NetworkContainerid - req2.OrchestratorContext = orchestratorContext - networkContainerGoalState := service.getNetworkContainerResponse(req2) - log.Printf("[tempdebug] restServer: networkContainerGoalState2: %+v", networkContainerGoalState) - if endpointID, err = hnsclient.CreateNewEndpoint(req.EndpointInfo, networkContainerGoalState.LocalIPConfiguration); err != nil { - returnMessage = fmt.Sprintf("CreateNewEndpoint failed with error: %v", err) - returnCode = UnexpectedError - } - default: - returnMessage = "CreateNewEndpoint API expects a POST" - returnCode = UnsupportedVerb - } - - resp := cns.Response{ - ReturnCode: returnCode, - Message: returnMessage, - } - - createApipaEndpointResp := cns.CreateApipaEndpointResponse{ - Response: resp, - EndpointID: endpointID, - } - log.Printf("[tempdebug] CreateNewEndpoint: %+v", createApipaEndpointResp) - - err = service.Listener.Encode(w, &createApipaEndpointResp) - log.Response(service.Name, createApipaEndpointResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) -} diff --git a/network/endpoint.go b/network/endpoint.go index 9ccdb5fe8d..21bab74aaa 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -36,6 +36,7 @@ type endpoint struct { AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool HostNCApipaEndpointID string + NetworkContainerID string NetworkNameSpace string `json:",omitempty"` ContainerID string PODName string `json:",omitempty"` diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 6d52194044..2729c34cbf 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -537,7 +537,7 @@ func (nw *network) configureApipaEndpoint(epInfo *EndpointInfo) (*hcn.HostComput return hcnEndpoint, nil } -func (nw *network) deleteApipaEndpoint(endpointID string) error { +func (nw *network) deleteHostNCApipaEndpoint(endpointID string) error { // TODO: cnsclient shouldn't be here. Need to move and encap this somewhere else. cnsClient, err := cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg if err != nil { @@ -546,7 +546,7 @@ func (nw *network) deleteApipaEndpoint(endpointID string) error { } log.Printf("[net] Deleting apipa hcn endpoint with id: %s", endpointID) - err = cnsClient.DeleteApipaEndpoint(endpointID) + err = cnsClient.DeleteHostNCApipaEndpoint(endpointID) log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", endpointID, err) return nil @@ -557,8 +557,8 @@ func (nw *network) createHostNCApipaEndpoint(epInfo *EndpointInfo) error { err error cnsClient *cnsclient.CNSClient hostNCApipaEndpointID string - hostNCApipaNetwork *hcn.HostComputeNetwork - namespace *hcn.HostComputeNamespace + //hostNCApipaNetwork *hcn.HostComputeNetwork + namespace *hcn.HostComputeNamespace ) if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { @@ -722,7 +722,7 @@ func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { log.Printf("[net] deleteEndpointImplHnsV2 DELETE id:%+v", ep) //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { { - if err = nw.deleteApipaEndpoint(ep.HostNCApipaEndpointID); err != nil { + if err = nw.deleteHostNCApipaEndpoint(ep.HostNCApipaEndpointID); err != nil { log.Errorf("[net] Failed to delete APIPA endpoint due to error: %v", err) return err } From 12bb8d8572fbdc79b5b9e61ff8060ae3de4ec7f3 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Mon, 14 Oct 2019 00:20:52 -0700 Subject: [PATCH 27/37] WIP-10-13-2 --- cni/network/multitenancy.go | 4 +- cni/network/network.go | 11 +- cns/cnsclient/cnsclient.go | 117 +++------- cns/hnsclient/hnsclient_windows.go | 220 ++++++------------ cns/networkcontainers/networkcontainers.go | 21 +- .../networkcontainers_windows.go | 77 +++--- cns/restserver/restserver.go | 14 +- network/endpoint_windows.go | 39 ++-- 8 files changed, 186 insertions(+), 317 deletions(-) diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index 2af4164da9..e37f979047 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -65,9 +65,9 @@ func getContainerNetworkConfigurationInternal( namespace string, podName string, ifName string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) { - cnsClient, err := cnsclient.NewCnsClient(address) + cnsClient, err := cnsclient.GetCnsClient() if err != nil { - log.Printf("Initializing CNS client error %v", err) + log.Printf("Failed to get CNS client. Error: %v", err) return nil, nil, net.IPNet{}, err } diff --git a/cni/network/network.go b/cni/network/network.go index 83ef22c12d..f1e82ae06c 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -242,6 +242,9 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { return err } + // Initialize CNSClient + cnsclient.InitCnsClient(nwCfg.CNSUrl) + k8sContainerID := args.ContainerID if len(k8sContainerID) == 0 { errMsg := "Container ID not specified in CNI Args" @@ -552,6 +555,9 @@ func (plugin *netPlugin) Get(args *cniSkel.CmdArgs) error { return err } + // Initialize CNSClient + cnsclient.InitCnsClient(nwCfg.CNSUrl) + // Initialize values from network config. if networkId, err = getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg); err != nil { log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err) @@ -627,6 +633,9 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Failed to get POD info due to error: %v", err) } + // Initialize CNSClient + cnsclient.InitCnsClient(nwCfg.CNSUrl) + // Initialize values from network config. if networkId, err = getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg); err != nil { log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err) @@ -772,7 +781,7 @@ func (plugin *netPlugin) Update(args *cniSkel.CmdArgs) error { // now query CNS to get the target routes that should be there in the networknamespace (as a result of update) log.Printf("Going to collect target routes for [name=%v, namespace=%v] from CNS.", k8sPodName, k8sNamespace) - if cnsClient, err = cnsclient.NewCnsClient(nwCfg.CNSUrl); err != nil { + if cnsClient, err = cnsclient.InitCnsClient(nwCfg.CNSUrl); err != nil { log.Printf("Initializing CNS client error in CNI Update%v", err) log.Printf(err.Error()) return plugin.Errorf(err.Error()) diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index c8f33c9e19..fb085f3229 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -8,75 +8,8 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/log" - //"github.com/Azure/azure-container-networking/network" ) -/* -// DNSInfo contains DNS information for a container network or endpoint. -type DNSInfo struct { - Suffix string - Servers []string - Options []string -} - -// NetworkInfo contains read-only information about a container network. -type NetworkInfo struct { - MasterIfName string - Id string - Mode string - Subnets []SubnetInfo - DNS DNSInfo - Policies []policy.Policy - BridgeName string - EnableSnatOnHost bool - NetNs string - Options map[string]interface{} -} - -// ExternalInterface is a host network interface that bridges containers to external networks. -type externalInterface struct { - Name string - Networks map[string]*network - Subnets []string - BridgeName string - DNSInfo DNSInfo - MacAddress net.HardwareAddr - IPAddresses []*net.IPNet - Routes []*route - IPv4Gateway net.IP - IPv6Gateway net.IP -} - -// EndpointInfo contains read-only information about an endpoint. -type EndpointInfo struct { - Id string - ContainerID string - NetNsPath string - IfName string - SandboxKey string - IfIndex int - MacAddress net.HardwareAddr - DNS DNSInfo - IPAddresses []net.IPNet - InfraVnetIP net.IPNet - Routes []RouteInfo - Policies []policy.Policy - Gateways []net.IP - EnableSnatOnHost bool - EnableInfraVnet bool - EnableMultiTenancy bool - AllowInboundFromHostToNC bool - AllowInboundFromNCToHost bool - HostNCApipaEndpointID string - PODName string - PODNameSpace string - Data map[string]interface{} - InfraVnetAddressSpace string - SkipHotAttachEp bool - NetworkID string -} -*/ - // CNSClient specifies a client to connect to Ipam Plugin. type CNSClient struct { connectionURL string @@ -86,15 +19,34 @@ const ( defaultCnsURL = "http://localhost:10090" ) -// NewCnsClient create a new cns client. -func NewCnsClient(url string) (*CNSClient, error) { - if url == "" { - url = defaultCnsURL +var ( + cnsClient *CNSClient +) + +// InitCnsClient initializes new cns client and returns the object +func InitCnsClient(url string) (*CNSClient, error) { + if cnsClient == nil { + if url == "" { + url = defaultCnsURL + } + + cnsClient = &CNSClient{ + connectionURL: url, + } + } + + return cnsClient, nil +} + +// GetCnsClient returns the cns client object +func GetCnsClient() (*CNSClient, error) { + var err error + + if cnsClient == nil { + err = fmt.Errorf("[Azure CNSClient] CNS Client not initialized") } - return &CNSClient{ - connectionURL: url, - }, nil + return cnsClient, err } // GetNetworkConfiguration Request to get network config. @@ -147,7 +99,7 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(orchestratorContext []byte) // CreateHostNCApipaEndpoint creates an endpoint in APIPA network for host container connectivity. func (cnsClient *CNSClient) CreateHostNCApipaEndpoint( - networkContainerID string /*podName, podNamespace string*/ /*orchestratorContext []byte*/) (string, error) { + networkContainerID string) (string, error) { var ( err error body bytes.Buffer @@ -157,21 +109,6 @@ func (cnsClient *CNSClient) CreateHostNCApipaEndpoint( url := cnsClient.connectionURL + cns.CreateHostNCApipaEndpointPath log.Printf("CreateHostNCApipaEndpoint url: %v", url) - /* - podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace} - orchestratorContext, err := json.Marshal(podInfo) - if err != nil { - log.Printf("Failed to marshall podInfo for orchestrator context due to error: %v", err) - return "", err - } - - - // TODO: What can be used here? can you pass ncid? - payload := &cns.CreateHostNCApipaEndpointRequest{ - OrchestratorContext: orchestratorContext, - } - */ - payload := &cns.CreateHostNCApipaEndpointRequest{ NetworkContainerID: networkContainerID, } diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index fcdea8c87b..7a9ac65590 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -2,15 +2,13 @@ package hnsclient import ( "encoding/json" - "errors" "fmt" "net" - "os" - "os/exec" "strconv" "strings" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/networkcontainers" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" @@ -43,7 +41,7 @@ const ( // hostNCApipaNetworkName indicates the name of the apipa network used for host container connectivity hostNCApipaNetworkName = "HostNCApipaNetwork" - // hostNCApipaNetworkType indicates the type of hns network setup for host NC connectivity + // hostNCApipaNetworkType indicates the type of hns network set up for host NC connectivity hostNCApipaNetworkType = hcn.L2Bridge // hostNCApipaEndpointName indicates the prefix for the name of the apipa endpoint used for @@ -262,7 +260,7 @@ func createHostNCApipaNetwork( } // Create loopback adapter needed for this HNS network - if networkExists, _ := interfaceExists(hostNCLoopbackAdapterName); !networkExists { + if networkExists, _ := networkcontainers.InterfaceExists(hostNCLoopbackAdapterName); !networkExists { ipconfig := cns.IPConfiguration{ IPSubnet: cns.IPSubnet{ IPAddress: localIPConfiguration.GatewayIPAddress, @@ -271,7 +269,11 @@ func createHostNCApipaNetwork( GatewayIPAddress: localIPConfiguration.GatewayIPAddress, } - if err = createLoopbackAdapter(hostNCLoopbackAdapterName, ipconfig); err != nil { + if err = networkcontainers.CreateLoopbackAdapter( + hostNCLoopbackAdapterName, + ipconfig, + false, /* Flag to setWeakHostOnInterface */ + "" /* Empty primary Interface Identifier as setWeakHostOnInterface is not needed*/); err != nil { return nil, fmt.Errorf("Failed to create loopback adapter. Error: %v", err) } } @@ -606,11 +608,6 @@ func configureHostNCApipaEndpoint( return endpoint, nil } -func getHostNCApipaEndpointName( - networkContainerID string) string { - return hostNCApipaEndpointNamePrefix + "-" + networkContainerID -} - //TODO: lock // CreateHostNCApipaEndpoint creates the endpoint in the apipa network for host container connectivity func CreateHostNCApipaEndpoint( @@ -661,56 +658,12 @@ func CreateHostNCApipaEndpoint( return endpoint.Id, nil } -//TODO: lock -// DeleteHostNCApipaEndpoint deletes the endpoint in the apipa network created for host container connectivity -// TODO: If you don't delete this APIPA network / if VM gets rebooted, how will you clean this upon restart? -func DeleteHostNCApipaEndpoint(endpointID string) error { - var ( - endpoint *hcn.HostComputeEndpoint - err error - ) - - // Check if the endpoint with the provided ID exists - if endpoint, err = hcn.GetEndpointByID(endpointID); err != nil { - // If error is anything other than EndpointNotFoundError, return error. - // else log the error but don't return error because endpoint is already deleted. - if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { - return fmt.Errorf("[Azure CNS] ERROR: DeleteHostNCApipaEndpoint failed due to "+ - "error with GetEndpointByName: %v", err) - } - - log.Errorf("[Azure CNS] Failed to find endpoint: %s for deletion. Error: %v", endpointID, err) - return nil - } - - if err = endpoint.Delete(); err != nil { - err = fmt.Errorf("Failed to delete endpoint: %+v. Error: %v", endpoint, err) - log.Errorf("[Azure CNS] %v", err) - return err - } - - log.Debugf("[Azure CNS] Successfully deleted apipa endpoint: %v", hostNCApipaNetworkName) - - var endpoints []hcn.HostComputeEndpoint - // Check if the network has any endpoints left - if endpoints, err = hcn.ListEndpointsOfNetwork(endpoint.HostComputeNetwork); err != nil { - log.Errorf("[Azure CNS] Failed to list endpoints in the network: %s. Error: %v", hostNCApipaNetworkName, err) - return nil - } - - // Delete network if it doesn't have any endpoints - if len(endpoints) == 0 { - if err = DeleteApipaNetwork(endpoint.HostComputeNetwork); err == nil { - // Delete the loopback adapter created for this network - deleteLoopbackAdapter(hostNCLoopbackAdapterName) - } - } - - return nil +func getHostNCApipaEndpointName( + networkContainerID string) string { + return hostNCApipaEndpointNamePrefix + "-" + networkContainerID } -// TODO: this can be a generic function to call hns delete -func DeleteApipaNetwork( +func deleteNetworkHnsV2( networkID string) error { var ( network *hcn.HostComputeNetwork @@ -721,18 +674,18 @@ func DeleteApipaNetwork( // If error is anything other than NetworkNotFoundError, return error. // else log the error but don't return error because network is already deleted. if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { - return fmt.Errorf("[Azure CNS] ERROR: DeleteApipaNetwork failed due to "+ + return fmt.Errorf("[Azure CNS] deleteNetworkHnsV2 failed due to "+ "error with GetNetworkByID: %v", err) } - log.Errorf("[Azure CNS] Failed to find network with ID: %s for deletion", networkID) + log.Errorf("[Azure CNS] Delete called on the Network: %s which doesn't exist. Error: %v", + networkID, err) + return nil } if err = network.Delete(); err != nil { - err = fmt.Errorf("Failed to delete network: %+v. Error: %v", network, err) - log.Errorf("[Azure CNS] %v", err) - return err + return fmt.Errorf("Failed to delete network: %+v. Error: %v", network, err) } log.Errorf("[Azure CNS] Successfully deleted network: %+v", network) @@ -740,107 +693,66 @@ func DeleteApipaNetwork( return nil } -func interfaceExists(iFaceName string) (bool, error) { - _, err := net.InterfaceByName(iFaceName) - if err != nil { - errMsg := fmt.Sprintf("[Azure CNS] Unable to get interface by name %s. Error: %v", iFaceName, err) - log.Printf(errMsg) - return false, fmt.Errorf(errMsg) - } +func deleteEndpointHnsV2( + endpointID string) error { + var ( + endpoint *hcn.HostComputeEndpoint + err error + ) - log.Printf("[Azure CNS] Found interface by name %s", iFaceName) + // Check if the endpoint with the provided ID exists + if endpoint, err = hcn.GetEndpointByID(endpointID); err != nil { + // If error is anything other than EndpointNotFoundError, return error. + // else log the error but don't return error because endpoint is already deleted. + if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { + return fmt.Errorf("[Azure CNS] deleteEndpointHnsV2 failed due to "+ + "error with GetEndpointByName: %v", err) + } - return true, nil -} + log.Errorf("[Azure CNS] Delete called on the Endpoint: %s which doesn't exist. Error: %v", + endpointID, err) -func createLoopbackAdapter( - adapterName string, - ipConfig cns.IPConfiguration) error { - if _, err := os.Stat("./AzureNetworkContainer.exe"); err != nil { - return fmt.Errorf("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") - } - - if ipConfig.IPSubnet.IPAddress == "" { - return fmt.Errorf("[Azure CNS] IPAddress in IPConfiguration is nil") - } - - ipv4AddrCidr := fmt.Sprintf("%v/%d", ipConfig.IPSubnet.IPAddress, ipConfig.IPSubnet.PrefixLength) - log.Printf("[Azure CNS] Created ipv4Cidr as %v", ipv4AddrCidr) - ipv4Addr, _, err := net.ParseCIDR(ipv4AddrCidr) - ipv4NetInt := net.CIDRMask((int)(ipConfig.IPSubnet.PrefixLength), 32) - log.Printf("[Azure CNS] Created netmask as %v", ipv4NetInt) - ipv4NetStr := fmt.Sprintf("%d.%d.%d.%d", ipv4NetInt[0], ipv4NetInt[1], ipv4NetInt[2], ipv4NetInt[3]) - log.Printf("[Azure CNS] Created netmask in string format %v", ipv4NetStr) - - args := []string{"/C", "AzureNetworkContainer.exe", "/logpath", log.GetLogDirectory(), - "/name", - adapterName, - "/operation", - "CREATE", - "/ip", - ipv4Addr.String(), - "/netmask", - ipv4NetStr, - "/gateway", - ipConfig.GatewayIPAddress, - "/weakhostsend", - "true", - "/weakhostreceive", - "true"} - - c := exec.Command("cmd", args...) - - //loopbackOperationLock.Lock() - log.Printf("[Azure CNS] Going to create/update network loopback adapter: %v", args) - bytes, err := c.Output() - /* - if err == nil { - err = setWeakHostOnInterface(createNetworkContainerRequest.PrimaryInterfaceIdentifier, - createNetworkContainerRequest.NetworkContainerid) - } - */ - //loopbackOperationLock.Unlock() + return nil + } - if err == nil { - log.Printf("[Azure CNS] Successfully created network loopback adapter for ipConfig: %+v. Output:%v.", - ipConfig, string(bytes)) - } else { - log.Printf("Failed to create loopback adapter for IP config: %+v. Error: %v. Output: %v", - ipConfig, err, string(bytes)) + if err = endpoint.Delete(); err != nil { + return fmt.Errorf("Failed to delete endpoint: %+v. Error: %v", endpoint, err) } - return err -} + log.Errorf("[Azure CNS] Successfully deleted endpoint: %+v", endpoint) -func deleteLoopbackAdapter(adapterName string) error { - if _, err := os.Stat("./AzureNetworkContainer.exe"); err != nil { - return fmt.Errorf("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") - } + return nil +} - if adapterName == "" { - return errors.New("[Azure CNS] Adapter name is not specified") +//TODO: lock +// DeleteHostNCApipaEndpoint deletes the endpoint in the apipa network created for host container connectivity +// TODO: If you don't delete this APIPA network / if VM gets rebooted, how will you clean this upon restart? +func DeleteHostNCApipaEndpoint( + endpointID string) error { + if err := deleteEndpointHnsV2(endpointID); err != nil { + log.Errorf("[Azure CNS] Failed to delete HostNCApipaEndpoint with ID: %s. Error: %v", endpointID, err) + return err } - args := []string{"/C", "AzureNetworkContainer.exe", "/logpath", log.GetLogDirectory(), - "/name", - adapterName, - "/operation", - "DELETE"} + log.Debugf("[Azure CNS] Successfully deleted HostNCApipaEndpoint with ID: %v", endpointID) - c := exec.Command("cmd", args...) - - //loopbackOperationLock.Lock() - log.Printf("[Azure CNS] Going to delete network loopback adapter: %v", args) - bytes, err := c.Output() - // loopbackOperationLock.Unlock() + // Check if hostNCApipaNetworkName has any endpoints left + if network, err := hcn.GetNetworkByName(hostNCApipaNetworkName); err == nil { + var endpoints []hcn.HostComputeEndpoint + if endpoints, err = hcn.ListEndpointsOfNetwork(network.Id); err != nil { + log.Errorf("[Azure CNS] Failed to list endpoints in the network: %s. Error: %v", + hostNCApipaNetworkName, err) + return nil + } - if err == nil { - log.Printf("[Azure CNS] Successfully deleted loopback adapter: %s. Output: %v.", - adapterName, string(bytes)) - } else { - log.Printf("Failed to delete loopback adapter: %s. Error: %v. Output: %v", - adapterName, err, string(bytes)) - //return err + // Delete network if it doesn't have any endpoints + if len(endpoints) == 0 { + if err = deleteNetworkHnsV2(network.Id); err == nil { + // Delete the loopback adapter created for this network + networkcontainers.DeleteLoopbackAdapter(hostNCLoopbackAdapterName) + } + } } - return err + + return nil } diff --git a/cns/networkcontainers/networkcontainers.go b/cns/networkcontainers/networkcontainers.go index d1218f8a71..87ea0b8efc 100644 --- a/cns/networkcontainers/networkcontainers.go +++ b/cns/networkcontainers/networkcontainers.go @@ -52,7 +52,7 @@ func NewNetPluginConfiguration(binPath, configPath string) *NetPluginConfigurati } } -func interfaceExists(iFaceName string) (bool, error) { +func InterfaceExists(iFaceName string) (bool, error) { _, err := net.InterfaceByName(iFaceName) if err != nil { errMsg := fmt.Sprintf("[Azure CNS] Unable to get interface by name %s. Error: %v", iFaceName, err) @@ -94,6 +94,25 @@ func (cn *NetworkContainers) Delete(networkContainerID string) error { return err } +// CreateLoopbackAdapter creates a loopback adapter with the specified settings +func CreateLoopbackAdapter( + adapterName string, + ipConfig cns.IPConfiguration, + setWeakHostOnInterface bool, + primaryInterfaceIdentifier string) error { + return createOrUpdateWithOperation( + adapterName, + ipConfig, + setWeakHostOnInterface, // Flag to setWeakHostOnInterface + primaryInterfaceIdentifier, + "CREATE") +} + +// DeleteLoopbackAdapter deletes loopback adapter with the specified name +func DeleteLoopbackAdapter(adapterName string) error { + return deleteInterface(adapterName) +} + // This function gets the flattened network configuration (compliant with azure cni) in byte array format func getNetworkConfig(configFilePath string) ([]byte, error) { content, err := ioutil.ReadFile(configFilePath) diff --git a/cns/networkcontainers/networkcontainers_windows.go b/cns/networkcontainers/networkcontainers_windows.go index b3b8b912ae..f5439b951b 100644 --- a/cns/networkcontainers/networkcontainers_windows.go +++ b/cns/networkcontainers/networkcontainers_windows.go @@ -27,11 +27,21 @@ func createOrUpdateInterface(createNetworkContainerRequest cns.CreateNetworkCont return nil } - if exists, _ := interfaceExists(createNetworkContainerRequest.NetworkContainerid); !exists { - return createOrUpdateWithOperation(createNetworkContainerRequest, "CREATE") + if exists, _ := InterfaceExists(createNetworkContainerRequest.NetworkContainerid); !exists { + return createOrUpdateWithOperation( + createNetworkContainerRequest.NetworkContainerid, + createNetworkContainerRequest.IPConfiguration, + true, // Flag to setWeakHostOnInterface + createNetworkContainerRequest.PrimaryInterfaceIdentifier, + "CREATE") } - return createOrUpdateWithOperation(createNetworkContainerRequest, "UPDATE") + return createOrUpdateWithOperation( + createNetworkContainerRequest.NetworkContainerid, + createNetworkContainerRequest.IPConfiguration, + true, // Flag to setWeakHostOnInterface + createNetworkContainerRequest.PrimaryInterfaceIdentifier, + "UPDATE") } func updateInterface(createNetworkContainerRequest cns.CreateNetworkContainerRequest, netpluginConfig *NetPluginConfiguration) error { @@ -102,28 +112,31 @@ func setWeakHostOnInterface(ipAddress, ncID string) error { return nil } -func createOrUpdateWithOperation(createNetworkContainerRequest cns.CreateNetworkContainerRequest, operation string) error { +func createOrUpdateWithOperation( + adapterName string, + ipConfig cns.IPConfiguration, + setWeakHost bool, + primaryInterfaceIdentifier string, + operation string) error { if _, err := os.Stat("./AzureNetworkContainer.exe"); err != nil { - if os.IsNotExist(err) { - return errors.New("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") - } + return fmt.Errorf("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") } - if createNetworkContainerRequest.IPConfiguration.IPSubnet.IPAddress == "" { - return errors.New("[Azure CNS] IPAddress in IPConfiguration of createNetworkContainerRequest is nil") + if ipConfig.IPSubnet.IPAddress == "" { + return fmt.Errorf("[Azure CNS] IPAddress in IPConfiguration is nil") } - ipv4AddrCidr := fmt.Sprintf("%v/%d", createNetworkContainerRequest.IPConfiguration.IPSubnet.IPAddress, createNetworkContainerRequest.IPConfiguration.IPSubnet.PrefixLength) + ipv4AddrCidr := fmt.Sprintf("%v/%d", ipConfig.IPSubnet.IPAddress, ipConfig.IPSubnet.PrefixLength) log.Printf("[Azure CNS] Created ipv4Cidr as %v", ipv4AddrCidr) ipv4Addr, _, err := net.ParseCIDR(ipv4AddrCidr) - ipv4NetInt := net.CIDRMask((int)(createNetworkContainerRequest.IPConfiguration.IPSubnet.PrefixLength), 32) + ipv4NetInt := net.CIDRMask((int)(ipConfig.IPSubnet.PrefixLength), 32) log.Printf("[Azure CNS] Created netmask as %v", ipv4NetInt) ipv4NetStr := fmt.Sprintf("%d.%d.%d.%d", ipv4NetInt[0], ipv4NetInt[1], ipv4NetInt[2], ipv4NetInt[3]) log.Printf("[Azure CNS] Created netmask in string format %v", ipv4NetStr) args := []string{"/C", "AzureNetworkContainer.exe", "/logpath", log.GetLogDirectory(), "/name", - createNetworkContainerRequest.NetworkContainerid, + adapterName, "/operation", operation, "/ip", @@ -131,7 +144,7 @@ func createOrUpdateWithOperation(createNetworkContainerRequest cns.CreateNetwork "/netmask", ipv4NetStr, "/gateway", - createNetworkContainerRequest.IPConfiguration.GatewayIPAddress, + ipConfig.GatewayIPAddress, "/weakhostsend", "true", "/weakhostreceive", @@ -142,38 +155,34 @@ func createOrUpdateWithOperation(createNetworkContainerRequest cns.CreateNetwork loopbackOperationLock.Lock() log.Printf("[Azure CNS] Going to create/update network loopback adapter: %v", args) bytes, err := c.Output() - if err == nil { - err = setWeakHostOnInterface(createNetworkContainerRequest.PrimaryInterfaceIdentifier, - createNetworkContainerRequest.NetworkContainerid) + if err == nil && setWeakHost { + err = setWeakHostOnInterface(primaryInterfaceIdentifier, adapterName) } loopbackOperationLock.Unlock() if err == nil { - log.Printf("[Azure CNS] Successfully created network loopback adapter for NC: %s. Output:%v.", - createNetworkContainerRequest.NetworkContainerid, string(bytes)) + log.Printf("[Azure CNS] Successfully created network loopback adapter with name: %s and IP config: %+v. Output:%v.", + adapterName, ipConfig, string(bytes)) } else { - log.Printf("Failed to create/update Network Container: %s. Error: %v. Output: %v", - createNetworkContainerRequest.NetworkContainerid, err.Error(), string(bytes)) + log.Printf("[Azure CNS] Failed to create network loopback adapter with name: %s and IP config: %+v."+ + " Error: %v. Output: %v", adapterName, ipConfig, err, string(bytes)) } return err } -func deleteInterface(networkContainerID string) error { - +func deleteInterface(interfaceName string) error { if _, err := os.Stat("./AzureNetworkContainer.exe"); err != nil { - if os.IsNotExist(err) { - return errors.New("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") - } + return fmt.Errorf("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") } - if networkContainerID == "" { - return errors.New("[Azure CNS] networkContainerID is nil") + if interfaceName == "" { + return fmt.Errorf("[Azure CNS] Interface name is nil") } args := []string{"/C", "AzureNetworkContainer.exe", "/logpath", log.GetLogDirectory(), "/name", - networkContainerID, + interfaceName, "/operation", "DELETE"} @@ -185,14 +194,14 @@ func deleteInterface(networkContainerID string) error { loopbackOperationLock.Unlock() if err == nil { - log.Printf("[Azure CNS] Successfully deleted network container: %s. Output: %v.", - networkContainerID, string(bytes)) + log.Printf("[Azure CNS] Successfully deleted loopack adapter with name: %s. Output: %v.", + interfaceName, string(bytes)) } else { - log.Printf("Failed to delete Network Container: %s. Error:%v. Output:%v", - networkContainerID, err.Error(), string(bytes)) - return err + log.Printf("[Azure CNS] Failed to delete loopback adapter with name: %s. Error:%v. Output:%v", + interfaceName, err.Error(), string(bytes)) } - return nil + + return err } func configureNetworkContainerNetworking(operation, podName, podNamespace, dockerContainerid string, netPluginConfig *NetPluginConfiguration) (err error) { diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 5eb7a388f1..05cf883363 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -1621,6 +1621,7 @@ func (service *HTTPRestService) getNumberOfCPUCores(w http.ResponseWriter, r *ht log.Response(service.Name, numOfCPUCoresResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } +//TODO: Use this in other places where this is being used. func (service *HTTPRestService) getNetworkContainerDetails(networkContainerID string) (containerstatus, bool) { service.lock.Lock() defer service.lock.Unlock() @@ -1649,17 +1650,6 @@ func (service *HTTPRestService) createHostNCApipaEndpoint(w http.ResponseWriter, switch r.Method { case "POST": - /* - var req2 cns.GetNetworkContainerRequest - req2.NetworkContainerid = req.NetworkContainerid - req2.OrchestratorContext = req.OrchestratorContext - networkContainerGoalState := service.getNetworkContainerResponse(req2) - log.Printf("[tempdebug] restServer: networkContainerGoalState: %+v", networkContainerGoalState) - if endpointID, err = hnsclient.CreateHostNCApipaEndpoint(networkContainerGoalState.LocalIPConfiguration); err != nil { - returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: %v", err) - returnCode = UnexpectedError - } - */ networkContainerDetails, found := service.getNetworkContainerDetails(req.NetworkContainerID) if found { if endpointID, err = hnsclient.CreateHostNCApipaEndpoint(req.NetworkContainerID, @@ -1684,7 +1674,6 @@ func (service *HTTPRestService) createHostNCApipaEndpoint(w http.ResponseWriter, }, EndpointID: endpointID, } - log.Printf("[tempdebug] CreateHostNCApipaEndpointResponse: %+v", response) err = service.Listener.Encode(w, &response) log.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err) @@ -1723,7 +1712,6 @@ func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter, Message: returnMessage, }, } - log.Printf("[tempdebug] deleteHostNCApipaEndpointResponse: %+v", response) err = service.Listener.Encode(w, &response) log.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 2729c34cbf..8e5b5de96f 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -537,28 +537,29 @@ func (nw *network) configureApipaEndpoint(epInfo *EndpointInfo) (*hcn.HostComput return hcnEndpoint, nil } -func (nw *network) deleteHostNCApipaEndpoint(endpointID string) error { - // TODO: cnsclient shouldn't be here. Need to move and encap this somewhere else. - cnsClient, err := cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg +func (nw *network) deleteHostNCApipaEndpoint(hostNCApipaEndpointID string) error { + cnsClient, err := cnsclient.GetCnsClient() if err != nil { - log.Errorf("Initializing CNS client error %v", err) + log.Errorf("Failed to get CNS client. Error %v", err) return err } - log.Printf("[net] Deleting apipa hcn endpoint with id: %s", endpointID) - err = cnsClient.DeleteHostNCApipaEndpoint(endpointID) - log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", endpointID, err) + log.Printf("[net] Deleting HostNCApipaEndpoint with id: %s", hostNCApipaEndpointID) + err = cnsClient.DeleteHostNCApipaEndpoint(hostNCApipaEndpointID) + log.Printf("[net] Completed HostNCApipaEndpoint deletion for ID: %s with error: %v", + hostNCApipaEndpointID, err) return nil } +// createHostNCApipaEndpoint creates a new endpoint in the HostNCApipaNetwork +// for host container connectivity func (nw *network) createHostNCApipaEndpoint(epInfo *EndpointInfo) error { var ( err error cnsClient *cnsclient.CNSClient hostNCApipaEndpointID string - //hostNCApipaNetwork *hcn.HostComputeNetwork - namespace *hcn.HostComputeNamespace + namespace *hcn.HostComputeNamespace ) if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { @@ -566,26 +567,21 @@ func (nw *network) createHostNCApipaEndpoint(epInfo *EndpointInfo) error { " due to error: %v", epInfo.NetNsPath, err) } - // TODO: This should be changed to GetCnsClient() - cnsClient, err = cnsclient.NewCnsClient("") //TODO: Need to pass the CNS url from nwCfg - if err != nil { - log.Errorf("Initializing CNS client error %v", err) - return err // upfate this to meaningful error + if cnsClient, err = cnsclient.GetCnsClient(); err != nil { + log.Errorf("Failed to get CNS client. Error %v", err) + return err } - log.Printf("[Azure CNS] Creating endpoint for host container connectivity") + log.Printf("[net] Creating HostNCApipaEndpoint for host container connectivity") if hostNCApipaEndpointID, err = - cnsClient.CreateHostNCApipaEndpoint(epInfo.NetworkContainerID /*epInfo.PODName, epInfo.PODNameSpace*/); err != nil { + cnsClient.CreateHostNCApipaEndpoint(epInfo.NetworkContainerID); err != nil { return err } defer func() { if err != nil { - log.Printf("[net] Deleting hcn endpoint with id: %s", hostNCApipaEndpointID) - // TODO: when this becomes generic, localIP can be passed to delete the endpoint - err = cnsClient.DeleteHostNCApipaEndpoint(hostNCApipaEndpointID) - log.Printf("[net] Completed hcn endpoint deletion for id: %s with error: %v", hostNCApipaEndpointID, err) + nw.deleteHostNCApipaEndpoint(hostNCApipaEndpointID) } }() @@ -643,11 +639,10 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) } }() - // If the host <-> container connectivity is requested, create endpoint in HostNCApipaNetwork + // If the Host - container connectivity is requested, create endpoint in HostNCApipaNetwork //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { if err = nw.createHostNCApipaEndpoint(epInfo); err != nil { return nil, fmt.Errorf("Failed to create HostNCApipaEndpoint due to error: %v", err) - // TODO: delete the endpoint created above and return appropriate error. } //} From a7d06c9d87bcca08229f8aad37d3a46caaec1470 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Mon, 14 Oct 2019 00:48:42 -0700 Subject: [PATCH 28/37] WIP-10-13-3 --- cni/network/multitenancy.go | 9 +- cni/network/network_windows.go | 1 - cns/api.go | 83 ------- cns/hnsclient/hnsclient_windows.go | 8 +- network/endpoint.go | 1 - network/endpoint_windows.go | 333 +---------------------------- network/manager.go | 1 - network/models/models.go | 140 ------------ network/network_windows.go | 102 --------- 9 files changed, 9 insertions(+), 669 deletions(-) delete mode 100644 network/models/models.go diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index e37f979047..fdb68a4ac7 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -29,11 +29,11 @@ func SetupRoutingForMultitenancy( // if snat enabled, add 169.254.0.1 as default gateway if nwCfg.EnableSnatOnHost { log.Printf("add default route for multitenancy.snat on host enabled with .2") - addDefaultRoute("169.254.0.2" /*cnsNetworkConfig.LocalIPConfiguration.GatewayIPAddress*/, epInfo, result) + addDefaultRoute(cnsNetworkConfig.LocalIPConfiguration.GatewayIPAddress, epInfo, result) } else { _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") dstIP := net.IPNet{IP: net.ParseIP("0.0.0.0"), Mask: defaultIPNet.Mask} - gwIP := net.ParseIP("169.254.0.2") //cnsNetworkConfig.IPConfiguration.GatewayIPAddress) + gwIP := net.ParseIP(cnsNetworkConfig.IPConfiguration.GatewayIPAddress) epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP}) result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) } @@ -253,8 +253,3 @@ func CleanupMultitenancyResources(enableInfraVnet bool, nwCfg *cni.NetworkConfig cleanupInfraVnetIP(enableInfraVnet, &azIpamResult.IPs[0].Address, nwCfg, plugin) } } - -/* -func CreateApipaEndpoint2(podName, podNamespace string) error { - return nil -}*/ diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 9eb67955f6..036390dd11 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -93,7 +93,6 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne cnetAddressMap = append(cnetAddressMap, ipSubnet.IPAddress+"/"+strconv.Itoa(int(ipSubnet.PrefixLength))) } epInfo.Data[network.CnetAddressSpace] = cnetAddressMap - epInfo.Data[network.LocalIPKey] = cnsNwConfig.LocalIPConfiguration.IPSubnet.IPAddress epInfo.AllowInboundFromHostToNC = cnsNwConfig.AllowHostToNCCommunication epInfo.AllowInboundFromNCToHost = cnsNwConfig.AllowNCToHostCommunication epInfo.NetworkContainerID = cnsNwConfig.NetworkContainerID diff --git a/cns/api.go b/cns/api.go index 8fcbe4c03c..648be68dec 100644 --- a/cns/api.go +++ b/cns/api.go @@ -5,8 +5,6 @@ package cns import ( "encoding/json" - //"github.com/Azure/azure-container-networking/network" - models "github.com/Azure/azure-container-networking/network/models" ) // Container Network Service remote API Contract @@ -27,77 +25,8 @@ const ( DeleteHostNCApipaEndpointPath = "/network/deletehostncapipaendpoint" V1Prefix = "/v0.1" V2Prefix = "/v0.2" - - OptOrchContext = "OrchestratorContext" - OptNCID = "NCID" ) -/* -// DNSInfo contains DNS information for a container network or endpoint. -type DNSInfo struct { - Suffix string - Servers []string - Options []string -} - -// NetworkInfo contains read-only information about a container network. -type NetworkInfo struct { - MasterIfName string - Id string - Mode string - Subnets []SubnetInfo - DNS DNSInfo - Policies []policy.Policy - BridgeName string - EnableSnatOnHost bool - NetNs string - Options map[string]interface{} -} - -// ExternalInterface is a host network interface that bridges containers to external networks. -type externalInterface struct { - Name string - Networks map[string]*network - Subnets []string - BridgeName string - DNSInfo DNSInfo - MacAddress net.HardwareAddr - IPAddresses []*net.IPNet - Routes []*route - IPv4Gateway net.IP - IPv6Gateway net.IP -} - -// EndpointInfo contains read-only information about an endpoint. -type EndpointInfo struct { - Id string - ContainerID string - NetNsPath string - IfName string - SandboxKey string - IfIndex int - MacAddress net.HardwareAddr - DNS DNSInfo - IPAddresses []net.IPNet - InfraVnetIP net.IPNet - Routes []RouteInfo - Policies []policy.Policy - Gateways []net.IP - EnableSnatOnHost bool - EnableInfraVnet bool - EnableMultiTenancy bool - AllowInboundFromHostToNC bool - AllowInboundFromNCToHost bool - HostNCApipaEndpointID string - PODName string - PODNameSpace string - Data map[string]interface{} - InfraVnetAddressSpace string - SkipHotAttachEp bool - NetworkID string -} -*/ - // SetEnvironmentRequest describes the Request to set the environment in CNS. type SetEnvironmentRequest struct { Location string @@ -229,22 +158,10 @@ type errorResponse struct { Err string } -// CreateNewNetworkRequest describes request to create new network. -type CreateNewNetworkRequest struct { - NetworkInfo models.NetworkInfo - ExternalInterface models.ExternalInterface -} - -// CreateNewEndpointRequest describes request to create new endpoint. -type CreateNewEndpointRequest struct { - EndpointInfo models.EndpointInfo -} - // CreateHostNCApipaEndpointRequest describes request for create apipa endpoint // for host container connectivity for the given network container type CreateHostNCApipaEndpointRequest struct { NetworkContainerID string - //OrchestratorContext json.RawMessage } // CreateHostNCApipaEndpointResponse describes response for create apipa endpoint request diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 7a9ac65590..b2f0e57967 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -220,7 +220,6 @@ func configureHostNCApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn subnetPrefix.IP = ipAddr.Mask(subnetPrefix.Mask) subnetPrefixStr = subnetPrefix.IP.String() + "/" + strconv.Itoa(int(localIPConfiguration.IPSubnet.PrefixLength)) - log.Printf("[tempdebug] configureApipaNetwork: subnetPrefixStr: %s, GW: %s", subnetPrefixStr, localIPConfiguration.GatewayIPAddress) subnet := hcn.Subnet{ IpAddressPrefix: subnetPrefixStr, @@ -234,6 +233,8 @@ func configureHostNCApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn network.Ipams[0].Subnets = append(network.Ipams[0].Subnets, subnet) + log.Printf("[Azure CNS] Configured HostNCApipaNetwork: %+v", network) + return network, nil } @@ -308,7 +309,6 @@ func configureHostNCApipaEndpoint( localIP := localIPConfiguration.IPSubnet.IPAddress remoteIP := localIPConfiguration.GatewayIPAddress - log.Printf("[tempdebug] configureHostNCApipaEndpoint localIP: %s, remoteIP: %s", localIP, remoteIP) /********************************************************************************************************/ // Add ICMP ACLs { @@ -600,11 +600,13 @@ func configureHostNCApipaEndpoint( ipConfiguration := hcn.IpConfig{ IpAddress: localIP, - PrefixLength: localIPConfiguration.IPSubnet.PrefixLength, // TODO: this should come from the cns config + PrefixLength: localIPConfiguration.IPSubnet.PrefixLength, } endpoint.IpConfigurations = append(endpoint.IpConfigurations, ipConfiguration) + log.Printf("[Azure CNS] Configured HostNCApipaEndpoint: %+v", endpoint) + return endpoint, nil } diff --git a/network/endpoint.go b/network/endpoint.go index 21bab74aaa..6e19f17c25 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -124,7 +124,6 @@ func (nw *network) deleteEndpoint(endpointId string) error { // Look up the endpoint. ep, err := nw.getEndpoint(endpointId) - log.Printf("[net] tempdebug:EP %+v.", ep) if err != nil { log.Printf("[net] Endpoint %v not found. Not Returning error", endpointId) return nil diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 8e5b5de96f..97f2eea6da 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -209,334 +209,6 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE return hcnEndpoint, nil } -// configureApipaEndpoint configures hcn endpoint for creation -func (nw *network) configureApipaEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeEndpoint, error) { - //infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName) - var hcnNetwork *hcn.HostComputeNetwork - var err error - if hcnNetwork, err = hcn.GetNetworkByName("secondary-nw"); err != nil { - log.Printf("[net] Failed to get temp nw due to error: %v", err) - return nil, fmt.Errorf("Failed to get hcn network with id: %s due to err: %v", nw.HnsId, err) - } - - name := "secondaryepwin" - - hcnEndpoint := &hcn.HostComputeEndpoint{ - Name: name, - HostComputeNetwork: hcnNetwork.Id, - SchemaVersion: hcn.SchemaVersion{ - Major: hcnSchemaVersionMajor, - Minor: hcnSchemaVersionMinor, - }, - } - - // TODO: below code can be handled by passing the context to the CNS and CNS looking up localIP from the CNS config - var localIP string - if epInfo.Data != nil { - if localIPData, ok := epInfo.Data[LocalIPKey]; ok { - localIP = localIPData.(string) - } - } - - /********************************************************************************************************/ - // Add ICMP ACLs - { - // Add endpoint ACL for preventing the comm to other apipa - aclOutBlockAll := hcn.AclPolicySetting{ - Protocols: "1", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err := json.Marshal(aclOutBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy := hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing out to host apipa - aclOutAllowToHostOnly := hcn.AclPolicySetting{ - Protocols: "1", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RemoteAddresses: "169.254.0.2", - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclOutAllowToHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInBlockAll := hcn.AclPolicySetting{ - Protocols: "1", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err = json.Marshal(aclInBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInAllowFromHostOnly := hcn.AclPolicySetting{ - Protocols: "1", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RemoteAddresses: "169.254.0.2", - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclInAllowFromHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - } - - // Add TCP ACLs - { - // Add endpoint ACL for preventing the comm to other apipa - aclOutBlockAll := hcn.AclPolicySetting{ - Protocols: "6", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err := json.Marshal(aclOutBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy := hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing out to host apipa - aclOutAllowToHostOnly := hcn.AclPolicySetting{ - Protocols: "6", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RemoteAddresses: "169.254.0.2", - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclOutAllowToHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInBlockAll := hcn.AclPolicySetting{ - Protocols: "6", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err = json.Marshal(aclInBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInAllowFromHostOnly := hcn.AclPolicySetting{ - Protocols: "6", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RemoteAddresses: "169.254.0.2", - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclInAllowFromHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - } - - // Add UDP ACLs - { - // Add endpoint ACL for preventing the comm to other apipa - aclOutBlockAll := hcn.AclPolicySetting{ - Protocols: "17", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err := json.Marshal(aclOutBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy := hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing out to host apipa - aclOutAllowToHostOnly := hcn.AclPolicySetting{ - Protocols: "17", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RemoteAddresses: "169.254.0.2", - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclOutAllowToHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInBlockAll := hcn.AclPolicySetting{ - Protocols: "17", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err = json.Marshal(aclInBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInAllowFromHostOnly := hcn.AclPolicySetting{ - Protocols: "17", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RemoteAddresses: "169.254.0.2", - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclInAllowFromHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - hcnEndpoint.Policies = append(hcnEndpoint.Policies, endpointPolicy) - } - /********************************************************************************************************/ - - nexthop := "169.254.0.2" - hcnRoute := hcn.Route{ - NextHop: nexthop, - DestinationPrefix: "0.0.0.0/0", - } - - hcnEndpoint.Routes = append(hcnEndpoint.Routes, hcnRoute) - - ipConfiguration := hcn.IpConfig{ - IpAddress: localIP, - PrefixLength: 16, // TODO: this should come from the cns config - } - - hcnEndpoint.IpConfigurations = append(hcnEndpoint.IpConfigurations, ipConfiguration) - - return hcnEndpoint, nil -} - func (nw *network) deleteHostNCApipaEndpoint(hostNCApipaEndpointID string) error { cnsClient, err := cnsclient.GetCnsClient() if err != nil { @@ -658,8 +330,6 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) gateway = net.ParseIP(hnsResponse.Routes[0].NextHop) } - log.Errorf("[net] tempdebug: HostNCApipaEndpointID: %s", epInfo.HostNCApipaEndpointID) - // Create the endpoint object. ep := &endpoint{ Id: hcnEndpoint.Name, @@ -715,10 +385,11 @@ func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { ) log.Printf("[net] deleteEndpointImplHnsV2 DELETE id:%+v", ep) + //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { { if err = nw.deleteHostNCApipaEndpoint(ep.HostNCApipaEndpointID); err != nil { - log.Errorf("[net] Failed to delete APIPA endpoint due to error: %v", err) + log.Errorf("[net] Failed to delete HostNCApipaEndpoint due to error: %v", err) return err } } diff --git a/network/manager.go b/network/manager.go index 966b6b2a2a..98bcbf5ace 100644 --- a/network/manager.go +++ b/network/manager.go @@ -17,7 +17,6 @@ const ( // Network store key. storeKey = "Network" VlanIDKey = "VlanID" - LocalIPKey = "localIP" genericData = "com.docker.network.generic" ) diff --git a/network/models/models.go b/network/models/models.go deleted file mode 100644 index 3a39cbb4ce..0000000000 --- a/network/models/models.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2017 Microsoft. All rights reserved. -// MIT License - -package models - -import ( - "net" - - "github.com/Azure/azure-container-networking/network/policy" - "github.com/Azure/azure-container-networking/platform" -) - -//todo: linux has different def for this. -type route interface{} - -// ExternalInterface is a host network interface that bridges containers to external networks. -type ExternalInterface struct { - Name string - Networks map[string]*network - Subnets []string - BridgeName string - DNSInfo DNSInfo - MacAddress net.HardwareAddr - IPAddresses []*net.IPNet - Routes []*route - IPv4Gateway net.IP - IPv6Gateway net.IP -} - -// A container network is a set of endpoints allowed to communicate with each other. -type network struct { - Id string - HnsId string `json:",omitempty"` - Mode string - VlanId int - Subnets []SubnetInfo - Endpoints map[string]*endpoint - extIf *ExternalInterface - DNS DNSInfo - EnableSnatOnHost bool - NetNs string - SnatBridgeIP string -} - -// NetworkInfo contains read-only information about a container network. -type NetworkInfo struct { - MasterIfName string - Id string - Mode string - Subnets []SubnetInfo - DNS DNSInfo - Policies []policy.Policy - BridgeName string - EnableSnatOnHost bool - NetNs string - Options map[string]interface{} -} - -// SubnetInfo contains subnet information for a container network. -type SubnetInfo struct { - Family platform.AddressFamily - Prefix net.IPNet - Gateway net.IP -} - -// DNSInfo contains DNS information for a container network or endpoint. -type DNSInfo struct { - Suffix string - Servers []string - Options []string -} - -// Endpoint represents a container network interface. -type endpoint struct { - Id string - HnsId string `json:",omitempty"` - SandboxKey string - IfName string - HostIfName string - MacAddress net.HardwareAddr - InfraVnetIP net.IPNet - LocalIP string - IPAddresses []net.IPNet - Gateways []net.IP - DNS DNSInfo - Routes []RouteInfo - VlanID int - EnableSnatOnHost bool - EnableInfraVnet bool - EnableMultitenancy bool - AllowInboundFromHostToNC bool - AllowInboundFromNCToHost bool - HostNCApipaEndpointID string - NetworkNameSpace string `json:",omitempty"` - ContainerID string - PODName string `json:",omitempty"` - PODNameSpace string `json:",omitempty"` - InfraVnetAddressSpace string `json:",omitempty"` - NetNs string `json:",omitempty"` - NetworkID string -} - -// EndpointInfo contains read-only information about an endpoint. -type EndpointInfo struct { - Id string - ContainerID string - NetNsPath string - IfName string - SandboxKey string - IfIndex int - MacAddress net.HardwareAddr - DNS DNSInfo - IPAddresses []net.IPNet - InfraVnetIP net.IPNet - Routes []RouteInfo - Policies []policy.Policy - Gateways []net.IP - EnableSnatOnHost bool - EnableInfraVnet bool - EnableMultiTenancy bool - AllowInboundFromHostToNC bool - AllowInboundFromNCToHost bool - HostNCApipaEndpointID string - PODName string - PODNameSpace string - Data map[string]interface{} - InfraVnetAddressSpace string - SkipHotAttachEp bool - NetworkID string -} - -// RouteInfo contains information about an IP route. -type RouteInfo struct { - Dst net.IPNet - Src net.IP - Gw net.IP - Protocol int - DevName string - Scope int -} diff --git a/network/network_windows.go b/network/network_windows.go index dbc1573884..0269ab5d6b 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -214,21 +214,6 @@ func (nm *networkManager) configureHcnNetwork(nwInfo *NetworkInfo, extIf *extern // newNetworkImplHnsV2 creates a new container network for HNSv2. func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { - // Do this only if there hostToCont / ContToHost is set - // if hostToCont / ContToHost - /* - { - apipaNw, err := nm.createApipaNw() - if err != nil { - err := fmt.Errorf("Failed to create APIPA bridge network for host to container connectivity due to error: %v", err) - log.Errorf("[net] %s", err.Error()) - return nil, err - } - - log.Printf("[net] Successfully setup APIPA bridge network for host to container connectivity: %+v", apipaNw) - } - */ - hcnNetwork, err := nm.configureHcnNetwork(nwInfo, extIf) if err != nil { log.Printf("[net] Failed to configure hcn network due to error: %v", err) @@ -267,93 +252,6 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern return nw, nil } -/* -// configureHcnEndpoint configures hcn endpoint for creation -func (nm *networkManager) configureApipaNetwork() (*hcn.HostComputeNetwork, error) { - // Initialize HNS network. - hcnNetwork := &hcn.HostComputeNetwork{ - Name: "secondary-nw", - Ipams: []hcn.Ipam{ - hcn.Ipam{ - Type: hcnIpamTypeStatic, - }, - }, - SchemaVersion: hcn.SchemaVersion{ - Major: hcnSchemaVersionMajor, - Minor: hcnSchemaVersionMinor, - }, - } - - // TODO: How to get this string from the created loopback adapter? - netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy("Ethernet 6") - if err != nil { - log.Printf("[net] Failed to serialize network adapter policy due to error: %v", err) - return nil, err - } - - hcnNetwork.Policies = append(hcnNetwork.Policies, netAdapterNamePolicy) - - // Set hcn subnet policy - var vlanid int - vlanid = 0 - - var subnetPolicy []byte - - hcnNetwork.Type = hcn.L2Bridge - - // Populate subnets. - hnsSubnet := hcn.Subnet{ - IpAddressPrefix: "169.254.0.0/16", - Routes: []hcn.Route{ - hcn.Route{ - NextHop: "169.254.0.2", - DestinationPrefix: "0.0.0.0/0", - }, - }, - } - - // Set the subnet policy - if vlanid > 0 { - hnsSubnet.Policies = append(hnsSubnet.Policies, subnetPolicy) - } - - hcnNetwork.Ipams[0].Subnets = append(hcnNetwork.Ipams[0].Subnets, hnsSubnet) - - return hcnNetwork, nil -} -*/ - -/* -// createApipaNw creates a new container network for HNSv2. -func (nm *networkManager) createApipaNw() (*hcn.HostComputeNetwork, error) { - var hcnNetwork *hcn.HostComputeNetwork - var err error - if hcnNetwork, err = hcn.GetNetworkByName("secondary-nw"); err == nil { - log.Printf("[net] Found existing APIPA network: %+v", hcnNetwork) - return nil, nil - } - - hcnNetwork, err = nm.configureApipaNetwork() - if err != nil { - log.Printf("[net] Failed to configure hcn network due to error: %v", err) - return nil, err - } - - // Create the HNS network. - log.Printf("[net] Creating temp hcn network: %+v", hcnNetwork) - hnsResponse, err := hcnNetwork.Create() - - if err != nil { - log.Printf("[net] Failed to create temp hcn network due to error: %v", err) - return nil, fmt.Errorf("Failed to create hcn network: %s due to error: %v", hcnNetwork.Name, err) - } - - log.Printf("[net] Successfully created temp hcn network with response: %+v", hnsResponse) - - return hnsResponse, nil -} -*/ - // NewNetworkImpl creates a new container network. func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) { if useHnsV2, err := UseHnsV2(nwInfo.NetNs); useHnsV2 { From 1da8f85b8698a0f13f4e5af5d3101497ec19cd19 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Mon, 14 Oct 2019 12:35:54 -0700 Subject: [PATCH 29/37] WIP-10-14-1 --- cni/network/multitenancy.go | 3 +-- cns/api.go | 4 +--- cns/hnsclient/hnsclient_windows.go | 1 + network/endpoint.go | 3 --- network/endpoint_windows.go | 6 +----- network/policy/policy_windows.go | 18 +++--------------- 6 files changed, 7 insertions(+), 28 deletions(-) diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index fdb68a4ac7..16728e252f 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -28,7 +28,7 @@ func SetupRoutingForMultitenancy( if nwCfg.MultiTenancy { // if snat enabled, add 169.254.0.1 as default gateway if nwCfg.EnableSnatOnHost { - log.Printf("add default route for multitenancy.snat on host enabled with .2") + log.Printf("add default route for multitenancy.snat on host enabled") addDefaultRoute(cnsNetworkConfig.LocalIPConfiguration.GatewayIPAddress, epInfo, result) } else { _, defaultIPNet, _ := net.ParseCIDR("0.0.0.0/0") @@ -59,7 +59,6 @@ func getContainerNetworkConfiguration( return getContainerNetworkConfigurationInternal(nwCfg.CNSUrl, podNamespace, podNameWithoutSuffix, ifName) } -//TODO: there is no need for this internal function - maybe called from other place func getContainerNetworkConfigurationInternal( address string, namespace string, diff --git a/cns/api.go b/cns/api.go index 648be68dec..111aabcdc0 100644 --- a/cns/api.go +++ b/cns/api.go @@ -3,9 +3,7 @@ package cns -import ( - "encoding/json" -) +import "encoding/json" // Container Network Service remote API Contract const ( diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index b2f0e57967..7814e253b1 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -194,6 +194,7 @@ func configureHostNCApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn Minor: hcnSchemaVersionMinor, }, Type: hostNCApipaNetworkType, + Flags: hcn.EnableNonPersistent,// Set up the network in non-persistent mode } if netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy(hostNCLoopbackAdapterName); err == nil { diff --git a/network/endpoint.go b/network/endpoint.go index 6e19f17c25..6710cf6347 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -43,7 +43,6 @@ type endpoint struct { PODNameSpace string `json:",omitempty"` InfraVnetAddressSpace string `json:",omitempty"` NetNs string `json:",omitempty"` - NetworkID string } // EndpointInfo contains read-only information about an endpoint. @@ -66,7 +65,6 @@ type EndpointInfo struct { EnableMultiTenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool - NetworkID string //TODO: check if this needed NetworkContainerID string HostNCApipaEndpointID string PODName string @@ -106,7 +104,6 @@ func (nw *network) newEndpoint(epInfo *EndpointInfo) (*endpoint, error) { nw.Endpoints[epInfo.Id] = ep log.Printf("[net] Created endpoint %+v.", ep) - log.Printf("[net] Created endpoint2 %+v.", nw.Endpoints[epInfo.Id]) return ep, nil } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 97f2eea6da..bffb34b297 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -25,9 +25,6 @@ const ( // hcnIpamTypeStatic indicates the static type of ipam hcnIpamTypeStatic = "Static" - - // HostNCApipaNetworkName indicates the name of the apipa network used for host container connectivity - HostNCApipaNetworkName = "host-nc-apipa-network" ) // HotAttachEndpoint is a wrapper of hcsshim's HotAttachEndpoint. @@ -178,7 +175,7 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE MacAddress: epInfo.MacAddress.String(), } - if endpointPolicies, err := policy.GetHcnEndpointPolicies(false, policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { + if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil { for _, epPolicy := range endpointPolicies { hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy) } @@ -343,7 +340,6 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) EnableSnatOnHost: epInfo.EnableSnatOnHost, NetNs: epInfo.NetNsPath, HostNCApipaEndpointID: epInfo.HostNCApipaEndpointID, - NetworkID: epInfo.NetworkID, } for _, route := range epInfo.Routes { diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index eb8dd04704..4ad4d3008b 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -327,7 +327,7 @@ func GetHcnPortMappingPolicy(policy Policy) (hcn.EndpointPolicy, error) { } // GetHcnEndpointPolicies returns array of all endpoint policies. -func GetHcnEndpointPolicies(onlyPortMapping bool, policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}) ([]hcn.EndpointPolicy, error) { +func GetHcnEndpointPolicies(policyType CNIPolicyType, policies []Policy, epInfoData map[string]interface{}) ([]hcn.EndpointPolicy, error) { var hcnEndPointPolicies []hcn.EndpointPolicy for _, policy := range policies { if policy.Type == policyType { @@ -336,23 +336,11 @@ func GetHcnEndpointPolicies(onlyPortMapping bool, policyType CNIPolicyType, poli switch GetPolicyType(policy) { case OutBoundNatPolicy: - //if !onlyPortMapping { endpointPolicy, err = GetHcnOutBoundNATPolicy(policy, epInfoData) - hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) - log.Printf("Successfully set the policy: %+v", endpointPolicy) - //} case RoutePolicy: - //if !onlyPortMapping { endpointPolicy, err = GetHcnRoutePolicy(policy) - hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) - log.Printf("Successfully set the policy: %+v", endpointPolicy) - //} case PortMappingPolicy: - //if onlyPortMapping { endpointPolicy, err = GetHcnPortMappingPolicy(policy) - hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) - log.Printf("Successfully set the policy: %+v", endpointPolicy) - //} default: // return error as we should be able to parse all the policies specified return hcnEndPointPolicies, fmt.Errorf("Failed to set Policy: Type: %s, Data: %s", policy.Type, policy.Data) @@ -363,8 +351,8 @@ func GetHcnEndpointPolicies(onlyPortMapping bool, policyType CNIPolicyType, poli return hcnEndPointPolicies, err } - //hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) - //log.Printf("Successfully set the policy: %+v", endpointPolicy) + hcnEndPointPolicies = append(hcnEndPointPolicies, endpointPolicy) + log.Printf("Successfully set the policy: %+v", endpointPolicy) } } From f6732cbf65299d7e975a9b09fcea27dd63dcadc9 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Mon, 14 Oct 2019 16:27:30 -0700 Subject: [PATCH 30/37] WIP-10-14-2 --- cns/hnsclient/hnsclient_windows.go | 364 +++++++---------------------- 1 file changed, 88 insertions(+), 276 deletions(-) diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 7814e253b1..cb4cf3bcb2 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -50,6 +50,15 @@ const ( // Name of the loopback adapter needed to create Host NC apipa network hostNCLoopbackAdapterName = "LoopbackAdapterHostNCConnectivity" + + // protocolTCP indicates the TCP protocol identifier in HCN + protocolTCP = "6" + + // protocolUDP indicates the UDP protocol identifier in HCN + protocolUDP = "17" + + // protocolICMPv4 indicates the ICMPv4 protocol identifier in HCN + protocolICMPv4 = "1" ) // CreateHnsNetwork creates the HNS network with the provided configuration @@ -193,8 +202,8 @@ func configureHostNCApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn Major: hcnSchemaVersionMajor, Minor: hcnSchemaVersionMinor, }, - Type: hostNCApipaNetworkType, - Flags: hcn.EnableNonPersistent,// Set up the network in non-persistent mode + Type: hostNCApipaNetworkType, + Flags: hcn.EnableNonPersistent, // Set up the network in non-persistent mode } if netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy(hostNCLoopbackAdapterName); err == nil { @@ -295,10 +304,33 @@ func createHostNCApipaNetwork( return network, err } +func addAclToEndpointPolicy( + aclPolicySetting hcn.AclPolicySetting, + endpointPolicies []hcn.EndpointPolicy) error { + var ( + rawJSON []byte + err error + ) + + if rawJSON, err = json.Marshal(aclPolicySetting); err != nil { + return fmt.Errorf("Failed to marshal endpoint ACL: %+v", aclPolicySetting) + } + + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + endpointPolicies = append(endpointPolicies, endpointPolicy) + + return nil +} + func configureHostNCApipaEndpoint( endpointName string, networkID string, localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeEndpoint, error) { + var err error endpoint := &hcn.HostComputeEndpoint{ Name: endpointName, HostComputeNetwork: networkID, @@ -308,299 +340,79 @@ func configureHostNCApipaEndpoint( }, } - localIP := localIPConfiguration.IPSubnet.IPAddress - remoteIP := localIPConfiguration.GatewayIPAddress - /********************************************************************************************************/ - // Add ICMP ACLs - { - // Add endpoint ACL for preventing the comm to other apipa - aclOutBlockAll := hcn.AclPolicySetting{ - Protocols: "1", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err := json.Marshal(aclOutBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } + networkContainerApipaIP := localIPConfiguration.IPSubnet.IPAddress + hostApipaIP := localIPConfiguration.GatewayIPAddress + protocolList := fmt.Sprintf("%s,%s,%s", protocolICMPv4, protocolTCP, protocolUDP) - endpointPolicy := hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - endpoint.Policies = append(endpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing out to host apipa - aclOutAllowToHostOnly := hcn.AclPolicySetting{ - Protocols: "1", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RemoteAddresses: remoteIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclOutAllowToHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - endpoint.Policies = append(endpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInBlockAll := hcn.AclPolicySetting{ - Protocols: "1", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err = json.Marshal(aclInBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - endpoint.Policies = append(endpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInAllowFromHostOnly := hcn.AclPolicySetting{ - Protocols: "1", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RemoteAddresses: remoteIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclInAllowFromHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - endpoint.Policies = append(endpoint.Policies, endpointPolicy) + // Endpoint ACL to block all outbound traffic from the Apipa IP of the container + outBlockAll := hcn.AclPolicySetting{ + Protocols: protocolList, + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + LocalAddresses: networkContainerApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, } - // Add TCP ACLs - { - // Add endpoint ACL for preventing the comm to other apipa - aclOutBlockAll := hcn.AclPolicySetting{ - Protocols: "6", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err := json.Marshal(aclOutBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy := hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - endpoint.Policies = append(endpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing out to host apipa - aclOutAllowToHostOnly := hcn.AclPolicySetting{ - Protocols: "6", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RemoteAddresses: remoteIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclOutAllowToHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - endpoint.Policies = append(endpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInBlockAll := hcn.AclPolicySetting{ - Protocols: "6", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err = json.Marshal(aclInBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - endpoint.Policies = append(endpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInAllowFromHostOnly := hcn.AclPolicySetting{ - Protocols: "6", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RemoteAddresses: remoteIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclInAllowFromHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - endpoint.Policies = append(endpoint.Policies, endpointPolicy) + if err = addAclToEndpointPolicy(outBlockAll, endpoint.Policies); err != nil { + return nil, err } - // Add UDP ACLs - { - // Add endpoint ACL for preventing the comm to other apipa - aclOutBlockAll := hcn.AclPolicySetting{ - Protocols: "17", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err := json.Marshal(aclOutBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy := hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - endpoint.Policies = append(endpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing out to host apipa - aclOutAllowToHostOnly := hcn.AclPolicySetting{ - Protocols: "17", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeOut, - LocalAddresses: localIP, - RemoteAddresses: remoteIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - rawJSON, err = json.Marshal(aclOutAllowToHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } - - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } - - endpoint.Policies = append(endpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInBlockAll := hcn.AclPolicySetting{ - Protocols: "17", - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - rawJSON, err = json.Marshal(aclInBlockAll) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } + // Endpoint ACL to allow the outbound traffic from the Apipa IP of the container to + // Apipa IP of the host only + outAllowToHostOnly := hcn.AclPolicySetting{ + Protocols: protocolList, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: networkContainerApipaIP, + RemoteAddresses: hostApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } + if err = addAclToEndpointPolicy(outAllowToHostOnly, endpoint.Policies); err != nil { + return nil, err + } - endpoint.Policies = append(endpoint.Policies, endpointPolicy) - - // Add endpoint ACL for allowing in from host apipa - aclInAllowFromHostOnly := hcn.AclPolicySetting{ - Protocols: "17", - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeIn, - LocalAddresses: localIP, - RemoteAddresses: remoteIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } + // Endpoint ACL to block all inbound traffic to the Apipa IP of the container + inBlockAll := hcn.AclPolicySetting{ + Protocols: protocolList, + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeIn, + LocalAddresses: networkContainerApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } - rawJSON, err = json.Marshal(aclInAllowFromHostOnly) - if err != nil { - return nil, fmt.Errorf("Failed to marshal the endpoint ACL") - } + if err = addAclToEndpointPolicy(inBlockAll, endpoint.Policies); err != nil { + return nil, err + } - endpointPolicy = hcn.EndpointPolicy{ - Type: hcn.ACL, - Settings: rawJSON, - } + // Endpoint ACL to allow the inbound traffic from the apipa IP of the host to + // the apipa IP of the container only + inAllowFromHostOnly := hcn.AclPolicySetting{ + Protocols: protocolList, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: networkContainerApipaIP, + RemoteAddresses: hostApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } - endpoint.Policies = append(endpoint.Policies, endpointPolicy) + if err = addAclToEndpointPolicy(inAllowFromHostOnly, endpoint.Policies); err != nil { + return nil, err } - /********************************************************************************************************/ - nexthop := remoteIP hcnRoute := hcn.Route{ - NextHop: nexthop, + NextHop: hostApipaIP, DestinationPrefix: "0.0.0.0/0", } endpoint.Routes = append(endpoint.Routes, hcnRoute) ipConfiguration := hcn.IpConfig{ - IpAddress: localIP, + IpAddress: networkContainerApipaIP, PrefixLength: localIPConfiguration.IPSubnet.PrefixLength, } From 0fb1572a2c002ae55ba1ab4546ee794c1390e8c2 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Mon, 14 Oct 2019 18:27:16 -0700 Subject: [PATCH 31/37] WIP-10-14-3 --- cns/api.go | 2 +- cns/cnsclient/cnsclient.go | 4 +- cns/hnsclient/hnsclient_windows.go | 41 +++++++++++----- cns/restserver/restserver.go | 5 +- common/namedlock.go | 75 ++++++++++++++++++++++++++++++ network/endpoint.go | 15 +++--- network/endpoint_windows.go | 38 +++++++-------- 7 files changed, 134 insertions(+), 46 deletions(-) create mode 100644 common/namedlock.go diff --git a/cns/api.go b/cns/api.go index 111aabcdc0..e933d42b56 100644 --- a/cns/api.go +++ b/cns/api.go @@ -172,7 +172,7 @@ type CreateHostNCApipaEndpointResponse struct { // DeleteHostNCApipaEndpointRequest describes request for deleting apipa endpoint created // for host NC connectivity. type DeleteHostNCApipaEndpointRequest struct { - EndpointID string + NetworkContainerID string } // DeleteHostNCApipaEndpointResponse describes response for delete host NC apipa endpoint request. diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index fb085f3229..781c7cc5a0 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -150,7 +150,7 @@ func (cnsClient *CNSClient) CreateHostNCApipaEndpoint( } // DeleteHostNCApipaEndpoint deletes the endpoint in APIPA network created for host container connectivity. -func (cnsClient *CNSClient) DeleteHostNCApipaEndpoint(endpointID string) error { +func (cnsClient *CNSClient) DeleteHostNCApipaEndpoint(networkContainerID string) error { var body bytes.Buffer // TODO: Move this to create a reusable http client. @@ -159,7 +159,7 @@ func (cnsClient *CNSClient) DeleteHostNCApipaEndpoint(endpointID string) error { log.Printf("DeleteHostNCApipaEndpoint url: %v", url) payload := &cns.DeleteHostNCApipaEndpointRequest{ - EndpointID: endpointID, + NetworkContainerID: networkContainerID, } err := json.NewEncoder(&body).Encode(payload) diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index cb4cf3bcb2..42daaacf1b 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -9,6 +9,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/networkcontainers" + "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" @@ -61,6 +62,11 @@ const ( protocolICMPv4 = "1" ) +var ( + // Named Lock for network and endpoint creation/deletion + namedLock = common.InitNamedLock() +) + // CreateHnsNetwork creates the HNS network with the provided configuration func CreateHnsNetwork(nwConfig cns.CreateHnsNetworkRequest) error { log.Printf("[Azure CNS] CreateHnsNetwork") @@ -255,6 +261,9 @@ func createHostNCApipaNetwork( err error ) + namedLock.LockAcquire(hostNCApipaNetworkName) + defer namedLock.LockRelease(hostNCApipaNetworkName) + // Check if the network exists for host NC connectivity if network, err = hcn.GetNetworkByName(hostNCApipaNetworkName); err != nil { // If error is anything other than networkNotFound, mark this as error @@ -423,7 +432,6 @@ func configureHostNCApipaEndpoint( return endpoint, nil } -//TODO: lock // CreateHostNCApipaEndpoint creates the endpoint in the apipa network for host container connectivity func CreateHostNCApipaEndpoint( networkContainerID string, @@ -435,6 +443,9 @@ func CreateHostNCApipaEndpoint( err error ) + namedLock.LockAcquire(endpointName) + defer namedLock.LockRelease(endpointName) + // Return if the endpoint already exists if endpoint, err = hcn.GetEndpointByName(endpointName); err != nil { // TODO: these are failing due to hcn bug https://github.com/microsoft/hcsshim/pull/519/files @@ -508,15 +519,15 @@ func deleteNetworkHnsV2( return nil } -func deleteEndpointHnsV2( - endpointID string) error { +func deleteEndpointByNameHnsV2( + endpointName string) error { var ( endpoint *hcn.HostComputeEndpoint err error ) - // Check if the endpoint with the provided ID exists - if endpoint, err = hcn.GetEndpointByID(endpointID); err != nil { + // Check if the endpoint exists + if endpoint, err = hcn.GetEndpointByName(endpointName); err != nil { // If error is anything other than EndpointNotFoundError, return error. // else log the error but don't return error because endpoint is already deleted. if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { @@ -525,7 +536,7 @@ func deleteEndpointHnsV2( } log.Errorf("[Azure CNS] Delete called on the Endpoint: %s which doesn't exist. Error: %v", - endpointID, err) + endpointName, err) return nil } @@ -539,17 +550,23 @@ func deleteEndpointHnsV2( return nil } -//TODO: lock // DeleteHostNCApipaEndpoint deletes the endpoint in the apipa network created for host container connectivity -// TODO: If you don't delete this APIPA network / if VM gets rebooted, how will you clean this upon restart? func DeleteHostNCApipaEndpoint( - endpointID string) error { - if err := deleteEndpointHnsV2(endpointID); err != nil { - log.Errorf("[Azure CNS] Failed to delete HostNCApipaEndpoint with ID: %s. Error: %v", endpointID, err) + networkContainerID string) error { + endpointName := getHostNCApipaEndpointName(networkContainerID) + + namedLock.LockAcquire(endpointName) + defer namedLock.LockRelease(endpointName) + + if err := deleteEndpointByNameHnsV2(endpointName); err != nil { + log.Errorf("[Azure CNS] Failed to delete HostNCApipaEndpoint: %s. Error: %v", endpointName, err) return err } - log.Debugf("[Azure CNS] Successfully deleted HostNCApipaEndpoint with ID: %v", endpointID) + log.Debugf("[Azure CNS] Successfully deleted HostNCApipaEndpoint: %v", endpointName) + + namedLock.LockAcquire(hostNCApipaNetworkName) + defer namedLock.LockRelease(hostNCApipaNetworkName) // Check if hostNCApipaNetworkName has any endpoints left if network, err := hcn.GetNetworkByName(hostNCApipaNetworkName); err == nil { diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 05cf883363..87833bf386 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -1697,8 +1697,9 @@ func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter, switch r.Method { case "POST": - if err = hnsclient.DeleteHostNCApipaEndpoint(req.EndpointID); err != nil { - returnMessage = fmt.Sprintf("Failed to delete endpoint: %s due to error: %v", req.EndpointID, err) + if err = hnsclient.DeleteHostNCApipaEndpoint(req.NetworkContainerID); err != nil { + returnMessage = fmt.Sprintf("Failed to delete endpoint for Network Container: %s "+ + "due to error: %v", req.NetworkContainerID, err) returnCode = UnexpectedError } default: diff --git a/common/namedlock.go b/common/namedlock.go new file mode 100644 index 0000000000..3736ec340d --- /dev/null +++ b/common/namedlock.go @@ -0,0 +1,75 @@ +package common + +import ( + "sync" +) + +// NamedLock holds a mutex and a map of locks. Mutex is used to +// get exclusive lock on the map while initializing the lock in the +// map. +type NamedLock struct { + mutex sync.Mutex + lockMap map[string]*refCountedLock +} + +// refCountedLock holds the lock and ref count for it +type refCountedLock struct { + mutex sync.RWMutex + refCount int +} + +// InitNamedLock initializes the named lock struct +func InitNamedLock() *NamedLock { + return &NamedLock{ + mutex: sync.Mutex{}, + lockMap: make(map[string]*refCountedLock), + } +} + +// LockAcquire acquires the lock with specified name +func (namedLock *NamedLock) LockAcquire(lockName string) { + namedLock.mutex.Lock() + _, ok := namedLock.lockMap[lockName] + if !ok { + namedLock.lockMap[lockName] = &refCountedLock{refCount: 0} + } + + namedLock.lockMap[lockName].AddRef() + namedLock.mutex.Unlock() + namedLock.lockMap[lockName].Lock() +} + +// LockRelease releases the lock with specified name +func (namedLock *NamedLock) LockRelease(lockName string) { + namedLock.mutex.Lock() + defer namedLock.mutex.Unlock() + + lock, ok := namedLock.lockMap[lockName] + if ok { + lock.Unlock() + lock.RemoveRef() + if lock.refCount == 0 { + delete(namedLock.lockMap, lockName) + } + } +} + +// AddRef increments the ref count on the lock +func (refCountedLock *refCountedLock) AddRef() { + refCountedLock.refCount++ +} + +// RemoveRef decrements the ref count on the lock +func (refCountedLock *refCountedLock) RemoveRef() { + refCountedLock.refCount-- +} + +// Lock locks the named lock +func (refCountedLock *refCountedLock) Lock() { + refCountedLock.mutex.Lock() +} + +// Unlock unlocks the named lock +func (refCountedLock *refCountedLock) Unlock() { + refCountedLock.mutex.Unlock() +} diff --git a/network/endpoint.go b/network/endpoint.go index 6710cf6347..8464105c55 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -35,7 +35,6 @@ type endpoint struct { EnableMultitenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool - HostNCApipaEndpointID string NetworkContainerID string NetworkNameSpace string `json:",omitempty"` ContainerID string @@ -66,7 +65,6 @@ type EndpointInfo struct { AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool NetworkContainerID string - HostNCApipaEndpointID string PODName string PODNameSpace string Data map[string]interface{} @@ -206,13 +204,12 @@ func (ep *endpoint) getInfo() *EndpointInfo { EnableMultiTenancy: ep.EnableMultitenancy, AllowInboundFromHostToNC: ep.AllowInboundFromHostToNC, AllowInboundFromNCToHost: ep.AllowInboundFromNCToHost, - IfName: ep.IfName, - ContainerID: ep.ContainerID, - NetNsPath: ep.NetworkNameSpace, - PODName: ep.PODName, - PODNameSpace: ep.PODNameSpace, - HostNCApipaEndpointID: ep.HostNCApipaEndpointID, - NetworkContainerID: ep.NetworkContainerID, + IfName: ep.IfName, + ContainerID: ep.ContainerID, + NetNsPath: ep.NetworkNameSpace, + PODName: ep.PODName, + PODNameSpace: ep.PODNameSpace, + NetworkContainerID: ep.NetworkContainerID, } for _, route := range ep.Routes { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index bffb34b297..bc8d21243e 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -206,17 +206,17 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE return hcnEndpoint, nil } -func (nw *network) deleteHostNCApipaEndpoint(hostNCApipaEndpointID string) error { +func (nw *network) deleteHostNCApipaEndpoint(networkContainerID string) error { cnsClient, err := cnsclient.GetCnsClient() if err != nil { log.Errorf("Failed to get CNS client. Error %v", err) return err } - log.Printf("[net] Deleting HostNCApipaEndpoint with id: %s", hostNCApipaEndpointID) - err = cnsClient.DeleteHostNCApipaEndpoint(hostNCApipaEndpointID) - log.Printf("[net] Completed HostNCApipaEndpoint deletion for ID: %s with error: %v", - hostNCApipaEndpointID, err) + log.Printf("[net] Deleting HostNCApipaEndpoint for network container: %s", networkContainerID) + err = cnsClient.DeleteHostNCApipaEndpoint(networkContainerID) + log.Printf("[net] Completed HostNCApipaEndpoint deletion for network container: %s"+ + " with error: %v", networkContainerID, err) return nil } @@ -250,7 +250,7 @@ func (nw *network) createHostNCApipaEndpoint(epInfo *EndpointInfo) error { defer func() { if err != nil { - nw.deleteHostNCApipaEndpoint(hostNCApipaEndpointID) + nw.deleteHostNCApipaEndpoint(epInfo.NetworkContainerID) } }() @@ -259,8 +259,6 @@ func (nw *network) createHostNCApipaEndpoint(epInfo *EndpointInfo) error { hostNCApipaEndpointID, namespace.Id, err) } - epInfo.HostNCApipaEndpointID = hostNCApipaEndpointID - return nil } @@ -329,17 +327,17 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) // Create the endpoint object. ep := &endpoint{ - Id: hcnEndpoint.Name, - HnsId: hnsResponse.Id, - SandboxKey: epInfo.ContainerID, - IfName: epInfo.IfName, - IPAddresses: epInfo.IPAddresses, - Gateways: []net.IP{gateway}, - DNS: epInfo.DNS, - VlanID: vlanid, - EnableSnatOnHost: epInfo.EnableSnatOnHost, - NetNs: epInfo.NetNsPath, - HostNCApipaEndpointID: epInfo.HostNCApipaEndpointID, + Id: hcnEndpoint.Name, + HnsId: hnsResponse.Id, + SandboxKey: epInfo.ContainerID, + IfName: epInfo.IfName, + IPAddresses: epInfo.IPAddresses, + Gateways: []net.IP{gateway}, + DNS: epInfo.DNS, + VlanID: vlanid, + EnableSnatOnHost: epInfo.EnableSnatOnHost, + NetNs: epInfo.NetNsPath, + NetworkContainerID: epInfo.NetworkContainerID, } for _, route := range epInfo.Routes { @@ -384,7 +382,7 @@ func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { { - if err = nw.deleteHostNCApipaEndpoint(ep.HostNCApipaEndpointID); err != nil { + if err = nw.deleteHostNCApipaEndpoint(ep.NetworkContainerID); err != nil { log.Errorf("[net] Failed to delete HostNCApipaEndpoint due to error: %v", err) return err } From 6456379c1662b203e75d65e9b56f67530163a093 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Tue, 15 Oct 2019 10:22:22 -0700 Subject: [PATCH 32/37] WIP-10-15-1 --- cns/cnsclient/cnsclient.go | 1 - cns/hnsclient/hnsclient_windows.go | 96 +++++++++++++++++------------- cns/restserver/api.go | 1 + cns/restserver/restserver.go | 36 +++++------ network/endpoint_windows.go | 35 +++++------ 5 files changed, 91 insertions(+), 78 deletions(-) diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index 781c7cc5a0..ade05a4f5b 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -153,7 +153,6 @@ func (cnsClient *CNSClient) CreateHostNCApipaEndpoint( func (cnsClient *CNSClient) DeleteHostNCApipaEndpoint(networkContainerID string) error { var body bytes.Buffer - // TODO: Move this to create a reusable http client. httpc := &http.Client{} url := cnsClient.connectionURL + cns.DeleteHostNCApipaEndpointPath log.Printf("DeleteHostNCApipaEndpoint url: %v", url) diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 42daaacf1b..1d844fe98a 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -264,15 +264,12 @@ func createHostNCApipaNetwork( namedLock.LockAcquire(hostNCApipaNetworkName) defer namedLock.LockRelease(hostNCApipaNetworkName) - // Check if the network exists for host NC connectivity + // Check if the network exists for Host NC connectivity if network, err = hcn.GetNetworkByName(hostNCApipaNetworkName); err != nil { // If error is anything other than networkNotFound, mark this as error - // TODO: why is following part not working? - /* - if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { - return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed. Error with GetNetworkByName: %v", err) - } - */ + if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { + return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed. Error with GetNetworkByName: %v", err) + } // Network doesn't exist. Create one. if network, err = configureHostNCApipaNetwork(localIPConfiguration); err != nil { @@ -280,7 +277,7 @@ func createHostNCApipaNetwork( } // Create loopback adapter needed for this HNS network - if networkExists, _ := networkcontainers.InterfaceExists(hostNCLoopbackAdapterName); !networkExists { + if interfaceExists, _ := networkcontainers.InterfaceExists(hostNCLoopbackAdapterName); !interfaceExists { ipconfig := cns.IPConfiguration{ IPSubnet: cns.IPSubnet{ IPAddress: localIPConfiguration.GatewayIPAddress, @@ -338,7 +335,9 @@ func addAclToEndpointPolicy( func configureHostNCApipaEndpoint( endpointName string, networkID string, - localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeEndpoint, error) { + localIPConfiguration cns.IPConfiguration, + allowNCToHostCommunication bool, + allowHostToNCCommunication bool) (*hcn.HostComputeEndpoint, error) { var err error endpoint := &hcn.HostComputeEndpoint{ Name: endpointName, @@ -367,20 +366,23 @@ func configureHostNCApipaEndpoint( return nil, err } - // Endpoint ACL to allow the outbound traffic from the Apipa IP of the container to - // Apipa IP of the host only - outAllowToHostOnly := hcn.AclPolicySetting{ - Protocols: protocolList, - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeOut, - LocalAddresses: networkContainerApipaIP, - RemoteAddresses: hostApipaIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } + if allowNCToHostCommunication { + log.Printf("[Azure CNS] Allowing NC to Host connectivity") + // Endpoint ACL to allow the outbound traffic from the Apipa IP of the container to + // Apipa IP of the host only + outAllowToHostOnly := hcn.AclPolicySetting{ + Protocols: protocolList, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: networkContainerApipaIP, + RemoteAddresses: hostApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } - if err = addAclToEndpointPolicy(outAllowToHostOnly, endpoint.Policies); err != nil { - return nil, err + if err = addAclToEndpointPolicy(outAllowToHostOnly, endpoint.Policies); err != nil { + return nil, err + } } // Endpoint ACL to block all inbound traffic to the Apipa IP of the container @@ -397,20 +399,23 @@ func configureHostNCApipaEndpoint( return nil, err } - // Endpoint ACL to allow the inbound traffic from the apipa IP of the host to - // the apipa IP of the container only - inAllowFromHostOnly := hcn.AclPolicySetting{ - Protocols: protocolList, - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeIn, - LocalAddresses: networkContainerApipaIP, - RemoteAddresses: hostApipaIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } + if allowHostToNCCommunication { + log.Printf("[Azure CNS] Allowing Host to NC connectivity") + // Endpoint ACL to allow the inbound traffic from the apipa IP of the host to + // the apipa IP of the container only + inAllowFromHostOnly := hcn.AclPolicySetting{ + Protocols: protocolList, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: networkContainerApipaIP, + RemoteAddresses: hostApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } - if err = addAclToEndpointPolicy(inAllowFromHostOnly, endpoint.Policies); err != nil { - return nil, err + if err = addAclToEndpointPolicy(inAllowFromHostOnly, endpoint.Policies); err != nil { + return nil, err + } } hcnRoute := hcn.Route{ @@ -435,7 +440,9 @@ func configureHostNCApipaEndpoint( // CreateHostNCApipaEndpoint creates the endpoint in the apipa network for host container connectivity func CreateHostNCApipaEndpoint( networkContainerID string, - localIPConfiguration cns.IPConfiguration) (string, error) { + localIPConfiguration cns.IPConfiguration, + allowNCToHostCommunication bool, + allowHostToNCCommunication bool) (string, error) { var ( network *hcn.HostComputeNetwork endpoint *hcn.HostComputeEndpoint @@ -448,7 +455,6 @@ func CreateHostNCApipaEndpoint( // Return if the endpoint already exists if endpoint, err = hcn.GetEndpointByName(endpointName); err != nil { - // TODO: these are failing due to hcn bug https://github.com/microsoft/hcsshim/pull/519/files // If error is anything other than EndpointNotFoundError, return error. if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { return "", fmt.Errorf("ERROR: Failed to query endpoint using GetEndpointByName "+ @@ -466,12 +472,16 @@ func CreateHostNCApipaEndpoint( return "", err } - if endpoint, err = configureHostNCApipaEndpoint(endpointName, network.Id, localIPConfiguration); err != nil { + if endpoint, err = configureHostNCApipaEndpoint( + endpointName, + network.Id, + localIPConfiguration, + allowNCToHostCommunication, + allowHostToNCCommunication); err != nil { log.Errorf("[Azure CNS] Failed to configure HostNCApipaEndpoint: %s. Error: %v", endpointName, err) return "", err } - // Create the apipa endpoint log.Printf("[Azure CNS] Creating HostNCApipaEndpoint for host container connectivity: %+v", endpoint) if endpoint, err = endpoint.Create(); err != nil { err = fmt.Errorf("Failed to create HostNCApipaEndpoint: %s. Error: %v", endpointName, err) @@ -489,7 +499,7 @@ func getHostNCApipaEndpointName( return hostNCApipaEndpointNamePrefix + "-" + networkContainerID } -func deleteNetworkHnsV2( +func deleteNetworkByIDHnsV2( networkID string) error { var ( network *hcn.HostComputeNetwork @@ -500,7 +510,7 @@ func deleteNetworkHnsV2( // If error is anything other than NetworkNotFoundError, return error. // else log the error but don't return error because network is already deleted. if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { - return fmt.Errorf("[Azure CNS] deleteNetworkHnsV2 failed due to "+ + return fmt.Errorf("[Azure CNS] deleteNetworkByIDHnsV2 failed due to "+ "error with GetNetworkByID: %v", err) } @@ -531,7 +541,7 @@ func deleteEndpointByNameHnsV2( // If error is anything other than EndpointNotFoundError, return error. // else log the error but don't return error because endpoint is already deleted. if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { - return fmt.Errorf("[Azure CNS] deleteEndpointHnsV2 failed due to "+ + return fmt.Errorf("[Azure CNS] deleteEndpointByNameHnsV2 failed due to "+ "error with GetEndpointByName: %v", err) } @@ -579,7 +589,7 @@ func DeleteHostNCApipaEndpoint( // Delete network if it doesn't have any endpoints if len(endpoints) == 0 { - if err = deleteNetworkHnsV2(network.Id); err == nil { + if err = deleteNetworkByIDHnsV2(network.Id); err == nil { // Delete the loopback adapter created for this network networkcontainers.DeleteLoopbackAdapter(hostNCLoopbackAdapterName) } diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 25d60e3ca2..f2a4fcbf2d 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -23,6 +23,7 @@ const ( DockerContainerNotSpecified = 20 UnsupportedVerb = 21 UnsupportedNetworkContainerType = 22 + InvalidRequest = 23 UnexpectedError = 99 ) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 87833bf386..af00747a9f 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -1099,9 +1099,7 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr case "POST": if req.NetworkContainerType == cns.WebApps { // try to get the saved nc state if it exists - service.lock.Lock() - existing, ok := service.state.ContainerStatus[req.NetworkContainerid] - service.lock.Unlock() + existing, ok := service.getNetworkContainerDetails(req.NetworkContainerid) // create/update nc only if it doesn't exist or it exists and the requested version is different from the saved version if !ok || (ok && existing.VMVersion != req.Version) { @@ -1114,9 +1112,7 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr } } else if req.NetworkContainerType == cns.AzureContainerInstance { // try to get the saved nc state if it exists - service.lock.Lock() - existing, ok := service.state.ContainerStatus[req.NetworkContainerid] - service.lock.Unlock() + existing, ok := service.getNetworkContainerDetails(req.NetworkContainerid) // create/update nc only if it doesn't exist or it exists and the requested version is different from the saved version if ok && existing.VMVersion != req.Version { @@ -1278,9 +1274,7 @@ func (service *HTTPRestService) deleteNetworkContainer(w http.ResponseWriter, r var containerStatus containerstatus var ok bool - service.lock.Lock() - containerStatus, ok = service.state.ContainerStatus[req.NetworkContainerid] - service.lock.Unlock() + containerStatus, ok = service.getNetworkContainerDetails(req.NetworkContainerid) if !ok { log.Printf("Not able to retrieve network container details for this container id %v", req.NetworkContainerid) @@ -1541,9 +1535,8 @@ func (service *HTTPRestService) attachOrDetachHelper(req cns.ConfigureContainerN Message: "[Azure CNS] Error. NetworkContainerid is empty"} } - service.lock.Lock() - existing, ok := service.state.ContainerStatus[cns.SwiftPrefix+req.NetworkContainerid] - service.lock.Unlock() + existing, ok := service.getNetworkContainerDetails(cns.SwiftPrefix + req.NetworkContainerid) + if !ok { return cns.Response{ ReturnCode: NotFound, @@ -1621,7 +1614,6 @@ func (service *HTTPRestService) getNumberOfCPUCores(w http.ResponseWriter, r *ht log.Response(service.Name, numOfCPUCoresResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } -//TODO: Use this in other places where this is being used. func (service *HTTPRestService) getNetworkContainerDetails(networkContainerID string) (containerstatus, bool) { service.lock.Lock() defer service.lock.Unlock() @@ -1652,10 +1644,20 @@ func (service *HTTPRestService) createHostNCApipaEndpoint(w http.ResponseWriter, case "POST": networkContainerDetails, found := service.getNetworkContainerDetails(req.NetworkContainerID) if found { - if endpointID, err = hnsclient.CreateHostNCApipaEndpoint(req.NetworkContainerID, - networkContainerDetails.CreateNetworkContainerRequest.LocalIPConfiguration); err != nil { - returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: %v", err) - returnCode = UnexpectedError + if !networkContainerDetails.CreateNetworkContainerRequest.AllowNCToHostCommunication && + !networkContainerDetails.CreateNetworkContainerRequest.AllowHostToNCCommunication { + returnMessage = fmt.Sprintf("HostNCApipaEndpoint creation is not supported unless " + + "AllowNCToHostCommunication or AllowHostToNCCommunication is set to true") + returnCode = InvalidRequest + } else { + if endpointID, err = hnsclient.CreateHostNCApipaEndpoint( + req.NetworkContainerID, + networkContainerDetails.CreateNetworkContainerRequest.LocalIPConfiguration, + networkContainerDetails.CreateNetworkContainerRequest.AllowNCToHostCommunication, + networkContainerDetails.CreateNetworkContainerRequest.AllowHostToNCCommunication); err != nil { + returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: %v", err) + returnCode = UnexpectedError + } } } else { returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: Unable to find goal state for"+ diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index bc8d21243e..cc8e38cea9 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -307,11 +307,11 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) }() // If the Host - container connectivity is requested, create endpoint in HostNCApipaNetwork - //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { - if err = nw.createHostNCApipaEndpoint(epInfo); err != nil { - return nil, fmt.Errorf("Failed to create HostNCApipaEndpoint due to error: %v", err) + if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { + if err = nw.createHostNCApipaEndpoint(epInfo); err != nil { + return nil, fmt.Errorf("Failed to create HostNCApipaEndpoint due to error: %v", err) + } } - //} var vlanid int if epInfo.Data != nil { @@ -327,17 +327,19 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) // Create the endpoint object. ep := &endpoint{ - Id: hcnEndpoint.Name, - HnsId: hnsResponse.Id, - SandboxKey: epInfo.ContainerID, - IfName: epInfo.IfName, - IPAddresses: epInfo.IPAddresses, - Gateways: []net.IP{gateway}, - DNS: epInfo.DNS, - VlanID: vlanid, - EnableSnatOnHost: epInfo.EnableSnatOnHost, - NetNs: epInfo.NetNsPath, - NetworkContainerID: epInfo.NetworkContainerID, + Id: hcnEndpoint.Name, + HnsId: hnsResponse.Id, + SandboxKey: epInfo.ContainerID, + IfName: epInfo.IfName, + IPAddresses: epInfo.IPAddresses, + Gateways: []net.IP{gateway}, + DNS: epInfo.DNS, + VlanID: vlanid, + EnableSnatOnHost: epInfo.EnableSnatOnHost, + NetNs: epInfo.NetNsPath, + AllowInboundFromNCToHost: epInfo.AllowInboundFromNCToHost, + AllowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, + NetworkContainerID: epInfo.NetworkContainerID, } for _, route := range epInfo.Routes { @@ -380,8 +382,7 @@ func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { log.Printf("[net] deleteEndpointImplHnsV2 DELETE id:%+v", ep) - //if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { - { + if ep.AllowInboundFromHostToNC || ep.AllowInboundFromNCToHost { if err = nw.deleteHostNCApipaEndpoint(ep.NetworkContainerID); err != nil { log.Errorf("[net] Failed to delete HostNCApipaEndpoint due to error: %v", err) return err From 1e1866ff177429fc2b1e96711f450be5ab36cf9d Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Tue, 15 Oct 2019 10:46:45 -0700 Subject: [PATCH 33/37] WIP-10-15-2 --- cns/hnsclient/hnsclient_linux.go | 19 +++++++++++++++++++ .../networkcontainers_linux.go | 9 +++++++++ 2 files changed, 28 insertions(+) diff --git a/cns/hnsclient/hnsclient_linux.go b/cns/hnsclient/hnsclient_linux.go index 186dec2eb3..a78acb9fb8 100644 --- a/cns/hnsclient/hnsclient_linux.go +++ b/cns/hnsclient/hnsclient_linux.go @@ -30,3 +30,22 @@ func CreateHnsNetwork(nwConfig cns.CreateHnsNetworkRequest) error { func DeleteHnsNetwork(networkName string) error { return fmt.Errorf("DeleteHnsNetwork shouldn't be called for linux platform") } + +// CreateHostNCApipaEndpoint creates the endpoint in the apipa network +// for host container connectivity +// This is windows platform specific. +func CreateHostNCApipaEndpoint( + networkContainerID string, + localIPConfiguration cns.IPConfiguration, + allowNCToHostCommunication bool, + allowHostToNCCommunication bool) (string, error) { + return "", nil +} + +// DeleteHostNCApipaEndpoint deletes the endpoint in the apipa network +// created for host container connectivity +// This is windows platform specific. +func DeleteHostNCApipaEndpoint( + networkContainerID string) error { + return nil +} diff --git a/cns/networkcontainers/networkcontainers_linux.go b/cns/networkcontainers/networkcontainers_linux.go index 0da7ce76a3..e776988c4b 100644 --- a/cns/networkcontainers/networkcontainers_linux.go +++ b/cns/networkcontainers/networkcontainers_linux.go @@ -90,3 +90,12 @@ func deleteInterface(networkContainerID string) error { func configureNetworkContainerNetworking(operation, podName, podNamespace, dockerContainerid string, netPluginConfig *NetPluginConfiguration) (err error) { return fmt.Errorf("[Azure CNS] Operation is not supported in linux.") } + +func createOrUpdateWithOperation( + adapterName string, + ipConfig cns.IPConfiguration, + setWeakHost bool, + primaryInterfaceIdentifier string, + operation string) error { + return nil +} From c9019b02abc5e21ab07d0729e05a607461f1df14 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Tue, 15 Oct 2019 16:26:05 -0700 Subject: [PATCH 34/37] WIP-10-15-3 --- cns/hnsclient/hnsclient_windows.go | 167 ++++++++++++++++++----------- 1 file changed, 102 insertions(+), 65 deletions(-) diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 1d844fe98a..7d581ee629 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -312,7 +312,7 @@ func createHostNCApipaNetwork( func addAclToEndpointPolicy( aclPolicySetting hcn.AclPolicySetting, - endpointPolicies []hcn.EndpointPolicy) error { + endpointPolicies *[]hcn.EndpointPolicy) error { var ( rawJSON []byte err error @@ -327,18 +327,106 @@ func addAclToEndpointPolicy( Settings: rawJSON, } - endpointPolicies = append(endpointPolicies, endpointPolicy) + *endpointPolicies = append(*endpointPolicies, endpointPolicy) return nil } +func configureAclSettingHostNCApipaEndpoint( + protocolList []string, + networkContainerApipaIP string, + hostApipaIP string, + allowNCToHostCommunication bool, + allowHostToNCCommunication bool) ([]hcn.EndpointPolicy, error) { + var ( + err error + endpointPolicies []hcn.EndpointPolicy + ) + + if allowNCToHostCommunication { + log.Printf("[Azure CNS] Allowing NC to Host connectivity") + } + + if allowHostToNCCommunication { + log.Printf("[Azure CNS] Allowing Host to NC connectivity") + } + + // Iterate thru the protocol list and add ACL for each + for _, protocol := range protocolList { + // Endpoint ACL to block all outbound traffic from the Apipa IP of the container + outBlockAll := hcn.AclPolicySetting{ + Protocols: protocol, + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + LocalAddresses: networkContainerApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + if err = addAclToEndpointPolicy(outBlockAll, &endpointPolicies); err != nil { + return nil, err + } + + if allowNCToHostCommunication { + // Endpoint ACL to allow the outbound traffic from the Apipa IP of the container to + // Apipa IP of the host only + outAllowToHostOnly := hcn.AclPolicySetting{ + Protocols: protocol, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: networkContainerApipaIP, + RemoteAddresses: hostApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + if err = addAclToEndpointPolicy(outAllowToHostOnly, &endpointPolicies); err != nil { + return nil, err + } + } + + // Endpoint ACL to block all inbound traffic to the Apipa IP of the container + inBlockAll := hcn.AclPolicySetting{ + Protocols: protocol, + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeIn, + LocalAddresses: networkContainerApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 2000, + } + + if err = addAclToEndpointPolicy(inBlockAll, &endpointPolicies); err != nil { + return nil, err + } + + if allowHostToNCCommunication { + // Endpoint ACL to allow the inbound traffic from the apipa IP of the host to + // the apipa IP of the container only + inAllowFromHostOnly := hcn.AclPolicySetting{ + Protocols: protocol, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: networkContainerApipaIP, + RemoteAddresses: hostApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: 200, + } + + if err = addAclToEndpointPolicy(inAllowFromHostOnly, &endpointPolicies); err != nil { + return nil, err + } + } + } + + return endpointPolicies, nil +} + func configureHostNCApipaEndpoint( endpointName string, networkID string, localIPConfiguration cns.IPConfiguration, allowNCToHostCommunication bool, allowHostToNCCommunication bool) (*hcn.HostComputeEndpoint, error) { - var err error endpoint := &hcn.HostComputeEndpoint{ Name: endpointName, HostComputeNetwork: networkID, @@ -350,72 +438,21 @@ func configureHostNCApipaEndpoint( networkContainerApipaIP := localIPConfiguration.IPSubnet.IPAddress hostApipaIP := localIPConfiguration.GatewayIPAddress - protocolList := fmt.Sprintf("%s,%s,%s", protocolICMPv4, protocolTCP, protocolUDP) - - // Endpoint ACL to block all outbound traffic from the Apipa IP of the container - outBlockAll := hcn.AclPolicySetting{ - Protocols: protocolList, - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeOut, - LocalAddresses: networkContainerApipaIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } - - if err = addAclToEndpointPolicy(outBlockAll, endpoint.Policies); err != nil { - return nil, err - } - - if allowNCToHostCommunication { - log.Printf("[Azure CNS] Allowing NC to Host connectivity") - // Endpoint ACL to allow the outbound traffic from the Apipa IP of the container to - // Apipa IP of the host only - outAllowToHostOnly := hcn.AclPolicySetting{ - Protocols: protocolList, - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeOut, - LocalAddresses: networkContainerApipaIP, - RemoteAddresses: hostApipaIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } + protocolList := []string{protocolICMPv4, protocolTCP, protocolUDP} - if err = addAclToEndpointPolicy(outAllowToHostOnly, endpoint.Policies); err != nil { - return nil, err - } - } - - // Endpoint ACL to block all inbound traffic to the Apipa IP of the container - inBlockAll := hcn.AclPolicySetting{ - Protocols: protocolList, - Action: hcn.ActionTypeBlock, - Direction: hcn.DirectionTypeIn, - LocalAddresses: networkContainerApipaIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 2000, - } + endpointPolicies, err := configureAclSettingHostNCApipaEndpoint( + protocolList, + networkContainerApipaIP, + hostApipaIP, + allowNCToHostCommunication, + allowHostToNCCommunication) - if err = addAclToEndpointPolicy(inBlockAll, endpoint.Policies); err != nil { - return nil, err + if err != nil { + log.Errorf("[Azure CNS] Failed to configure ACL for HostNCApipaEndpoint. Error: %v", err) } - if allowHostToNCCommunication { - log.Printf("[Azure CNS] Allowing Host to NC connectivity") - // Endpoint ACL to allow the inbound traffic from the apipa IP of the host to - // the apipa IP of the container only - inAllowFromHostOnly := hcn.AclPolicySetting{ - Protocols: protocolList, - Action: hcn.ActionTypeAllow, - Direction: hcn.DirectionTypeIn, - LocalAddresses: networkContainerApipaIP, - RemoteAddresses: hostApipaIP, - RuleType: hcn.RuleTypeSwitch, - Priority: 200, - } - - if err = addAclToEndpointPolicy(inAllowFromHostOnly, endpoint.Policies); err != nil { - return nil, err - } + for _, endpointPolicy := range endpointPolicies { + endpoint.Policies = append(endpoint.Policies, endpointPolicy) } hcnRoute := hcn.Route{ From a10d7dfd65115e6d117c68865d7cf5d0984cfccf Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Wed, 16 Oct 2019 16:31:51 -0700 Subject: [PATCH 35/37] Address review comments --- network/network_windows.go | 32 +++++++++++++-------- network/policy/policy_windows.go | 48 ++++++++++++++++---------------- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/network/network_windows.go b/network/network_windows.go index 0269ab5d6b..9a07b9306f 100644 --- a/network/network_windows.go +++ b/network/network_windows.go @@ -19,15 +19,21 @@ import ( const ( // HNS network types. - hnsL2bridge = "l2bridge" - hnsL2tunnel = "l2tunnel" - CnetAddressSpace = "cnetAddressSpace" + hnsL2bridge = "l2bridge" + hnsL2tunnel = "l2tunnel" + CnetAddressSpace = "cnetAddressSpace" + vEthernetAdapterPrefix = "vEthernet" + baseDecimal = 10 + bitSize = 32 + defaultRouteCIDR = "0.0.0.0/0" ) // Windows implementation of route. type route interface{} // UseHnsV2 indicates whether to use HNSv1 or HNSv2 +// HNSv2 should be used if the NetNs is a valid GUID and if the platform +// has HCN which supports HNSv2 API. func UseHnsV2(netNs string) (bool, error) { // Check if the netNs is a valid GUID to decide on HNSv1 or HNSv2 useHnsV2 := false @@ -47,7 +53,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern var vlanid int networkAdapterName := extIf.Name // FixMe: Find a better way to check if a nic that is selected is not part of a vSwitch - if strings.HasPrefix(networkAdapterName, "vEthernet") { + if strings.HasPrefix(networkAdapterName, vEthernetAdapterPrefix) { networkAdapterName = "" } // Initialize HNS network. @@ -64,7 +70,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern vlanPolicy := hcsshim.VlanPolicy{ Type: "VLAN", } - vlanID, _ := strconv.ParseUint(opt[VlanIDKey].(string), 10, 32) + vlanID, _ := strconv.ParseUint(opt[VlanIDKey].(string), baseDecimal, bitSize) vlanPolicy.VLAN = uint(vlanID) serializedVlanPolicy, _ := json.Marshal(vlanPolicy) @@ -153,7 +159,7 @@ func (nm *networkManager) configureHcnNetwork(nwInfo *NetworkInfo, extIf *extern // Set hcn network adaptor name policy // FixMe: Find a better way to check if a nic that is selected is not part of a vSwitch - if !strings.HasPrefix(extIf.Name, "vEthernet") { + if !strings.HasPrefix(extIf.Name, vEthernetAdapterPrefix) { netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy(extIf.Name) if err != nil { log.Printf("[net] Failed to serialize network adapter policy due to error: %v", err) @@ -164,12 +170,15 @@ func (nm *networkManager) configureHcnNetwork(nwInfo *NetworkInfo, extIf *extern } // Set hcn subnet policy - var vlanid int - var subnetPolicy []byte + var ( + vlanid int + subnetPolicy []byte + ) + opt, _ := nwInfo.Options[genericData].(map[string]interface{}) if opt != nil && opt[VlanIDKey] != nil { var err error - vlanID, _ := strconv.ParseUint(opt[VlanIDKey].(string), 10, 32) + vlanID, _ := strconv.ParseUint(opt[VlanIDKey].(string), baseDecimal, bitSize) subnetPolicy, err = policy.SerializeHcnSubnetVlanPolicy((uint32)(vlanID)) if err != nil { log.Printf("[net] Failed to serialize subnet vlan policy due to error: %v", err) @@ -193,10 +202,11 @@ func (nm *networkManager) configureHcnNetwork(nwInfo *NetworkInfo, extIf *extern for _, subnet := range nwInfo.Subnets { hnsSubnet := hcn.Subnet{ IpAddressPrefix: subnet.Prefix.String(), + // Set the Gateway route Routes: []hcn.Route{ hcn.Route{ NextHop: subnet.Gateway.String(), - DestinationPrefix: "0.0.0.0/0", + DestinationPrefix: defaultRouteCIDR, }, }, } @@ -233,7 +243,7 @@ func (nm *networkManager) newNetworkImplHnsV2(nwInfo *NetworkInfo, extIf *extern var vlanid int opt, _ := nwInfo.Options[genericData].(map[string]interface{}) if opt != nil && opt[VlanIDKey] != nil { - vlanID, _ := strconv.ParseInt(opt[VlanIDKey].(string), 10, 32) + vlanID, _ := strconv.ParseInt(opt[VlanIDKey].(string), baseDecimal, bitSize) vlanid = (int)(vlanID) } diff --git a/network/policy/policy_windows.go b/network/policy/policy_windows.go index 4ad4d3008b..92ff473e8a 100644 --- a/network/policy/policy_windows.go +++ b/network/policy/policy_windows.go @@ -18,9 +18,15 @@ const ( protocolUdp = 17 // CnetAddressSpace indicates constant for the key string - cnetAddressSpace = "cnetAddressSpace" + CnetAddressSpace = "cnetAddressSpace" ) +type KVPairRoutePolicy struct { + Type CNIPolicyType `json:"Type"` + DestinationPrefix json.RawMessage `json:"DestinationPrefix"` + NeedEncap json.RawMessage `json:"NeedEncap"` +} + type KVPairPortMapping struct { Type CNIPolicyType `json:"Type"` ExternalPort uint16 `json:"ExternalPort"` @@ -111,8 +117,8 @@ func SerializeOutBoundNATPolicy(policy Policy, epInfoData map[string]interface{} } } - if epInfoData[cnetAddressSpace] != nil { - if cnetAddressSpace := epInfoData[cnetAddressSpace].([]string); cnetAddressSpace != nil { + if epInfoData[CnetAddressSpace] != nil { + if cnetAddressSpace := epInfoData[CnetAddressSpace].([]string); cnetAddressSpace != nil { for _, ipAddress := range cnetAddressSpace { outBoundNatPolicy.Exceptions = append(outBoundNatPolicy.Exceptions, ipAddress) } @@ -164,22 +170,22 @@ func SerializeHcnSubnetVlanPolicy(vlanID uint32) ([]byte, error) { IsolationId: vlanID, } - vlanPolicySettingJSON, err := json.Marshal(vlanPolicySetting) + vlanPolicySettingBytes, err := json.Marshal(vlanPolicySetting) if err != nil { return nil, err } vlanSubnetPolicy := &hcn.SubnetPolicy{ Type: hcn.VLAN, - Settings: vlanPolicySettingJSON, + Settings: vlanPolicySettingBytes, } - vlanSubnetPolicyJSON, err := json.Marshal(vlanSubnetPolicy) + vlanSubnetPolicyBytes, err := json.Marshal(vlanSubnetPolicy) if err != nil { return nil, err } - return vlanSubnetPolicyJSON, nil + return vlanSubnetPolicyBytes, nil } // GetHcnNetAdapterPolicy returns network adapter name policy. @@ -192,12 +198,12 @@ func GetHcnNetAdapterPolicy(networkAdapterName string) (hcn.NetworkPolicy, error NetworkAdapterName: networkAdapterName, } - netAdapterNamePolicySettingJSON, err := json.Marshal(netAdapterNamePolicySetting) + netAdapterNamePolicySettingBytes, err := json.Marshal(netAdapterNamePolicySetting) if err != nil { return networkAdapterNamePolicy, err } - networkAdapterNamePolicy.Settings = netAdapterNamePolicySettingJSON + networkAdapterNamePolicy.Settings = netAdapterNamePolicySettingBytes return networkAdapterNamePolicy, nil } @@ -221,8 +227,8 @@ func GetHcnOutBoundNATPolicy(policy Policy, epInfoData map[string]interface{}) ( } } - if epInfoData[cnetAddressSpace] != nil { - if cnetAddressSpace := epInfoData[cnetAddressSpace].([]string); cnetAddressSpace != nil { + if epInfoData[CnetAddressSpace] != nil { + if cnetAddressSpace := epInfoData[CnetAddressSpace].([]string); cnetAddressSpace != nil { for _, ipAddress := range cnetAddressSpace { outBoundNATPolicySetting.Exceptions = append(outBoundNATPolicySetting.Exceptions, ipAddress) } @@ -230,12 +236,12 @@ func GetHcnOutBoundNATPolicy(policy Policy, epInfoData map[string]interface{}) ( } if outBoundNATPolicySetting.Exceptions != nil { - outBoundNATPolicySettingJSON, err := json.Marshal(outBoundNATPolicySetting) + outBoundNATPolicySettingBytes, err := json.Marshal(outBoundNATPolicySetting) if err != nil { return outBoundNATPolicy, err } - outBoundNATPolicy.Settings = outBoundNATPolicySettingJSON + outBoundNATPolicy.Settings = outBoundNATPolicySettingBytes return outBoundNATPolicy, nil } @@ -248,13 +254,7 @@ func GetHcnRoutePolicy(policy Policy) (hcn.EndpointPolicy, error) { Type: hcn.SDNRoute, } - type KVPair struct { - Type CNIPolicyType `json:"Type"` - DestinationPrefix json.RawMessage `json:"DestinationPrefix"` - NeedEncap json.RawMessage `json:"NeedEncap"` - } - - var data KVPair + var data KVPairRoutePolicy if err := json.Unmarshal(policy.Data, &data); err != nil { return routePolicy, err } @@ -276,12 +276,12 @@ func GetHcnRoutePolicy(policy Policy) (hcn.EndpointPolicy, error) { NeedEncap: needEncap, } - routePolicySettingJSON, err := json.Marshal(sdnRoutePolicySetting) + routePolicySettingBytes, err := json.Marshal(sdnRoutePolicySetting) if err != nil { return routePolicy, err } - routePolicy.Settings = routePolicySettingJSON + routePolicy.Settings = routePolicySettingBytes return routePolicy, nil } @@ -316,12 +316,12 @@ func GetHcnPortMappingPolicy(policy Policy) (hcn.EndpointPolicy, error) { return portMappingPolicy, fmt.Errorf("Invalid protocol: %s for port mapping", protocol) } - portMappingPolicySettingJSON, err := json.Marshal(portMappingPolicySetting) + portMappingPolicySettingBytes, err := json.Marshal(portMappingPolicySetting) if err != nil { return portMappingPolicy, err } - portMappingPolicy.Settings = portMappingPolicySettingJSON + portMappingPolicy.Settings = portMappingPolicySettingBytes return portMappingPolicy, nil } From 5f00ac155f8a9f9574575d8fe6f4a45e07fcc91c Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Fri, 18 Oct 2019 07:38:21 -0700 Subject: [PATCH 36/37] address review comments --- cni/network/network.go | 18 +++++++++++------ cns/cnsclient/cnsclient.go | 4 ++-- cns/hnsclient/hnsclient_windows.go | 31 +++++++++++++++++++++++------- common/namedlock.go | 4 ++++ network/endpoint_windows.go | 3 ++- 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/cni/network/network.go b/cni/network/network.go index f1e82ae06c..ed345af664 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -242,8 +242,10 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { return err } - // Initialize CNSClient - cnsclient.InitCnsClient(nwCfg.CNSUrl) + if nwCfg.MultiTenancy { + // Initialize CNSClient + cnsclient.InitCnsClient(nwCfg.CNSUrl) + } k8sContainerID := args.ContainerID if len(k8sContainerID) == 0 { @@ -555,8 +557,10 @@ func (plugin *netPlugin) Get(args *cniSkel.CmdArgs) error { return err } - // Initialize CNSClient - cnsclient.InitCnsClient(nwCfg.CNSUrl) + if nwCfg.MultiTenancy { + // Initialize CNSClient + cnsclient.InitCnsClient(nwCfg.CNSUrl) + } // Initialize values from network config. if networkId, err = getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg); err != nil { @@ -633,8 +637,10 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Failed to get POD info due to error: %v", err) } - // Initialize CNSClient - cnsclient.InitCnsClient(nwCfg.CNSUrl) + if nwCfg.MultiTenancy { + // Initialize CNSClient + cnsclient.InitCnsClient(nwCfg.CNSUrl) + } // Initialize values from network config. if networkId, err = getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg); err != nil { diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index ade05a4f5b..613b226fd9 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -107,7 +107,7 @@ func (cnsClient *CNSClient) CreateHostNCApipaEndpoint( httpc := &http.Client{} url := cnsClient.connectionURL + cns.CreateHostNCApipaEndpointPath - log.Printf("CreateHostNCApipaEndpoint url: %v", url) + log.Printf("CreateHostNCApipaEndpoint url: %v for NC: %s", url, networkContainerID) payload := &cns.CreateHostNCApipaEndpointRequest{ NetworkContainerID: networkContainerID, @@ -155,7 +155,7 @@ func (cnsClient *CNSClient) DeleteHostNCApipaEndpoint(networkContainerID string) httpc := &http.Client{} url := cnsClient.connectionURL + cns.DeleteHostNCApipaEndpointPath - log.Printf("DeleteHostNCApipaEndpoint url: %v", url) + log.Printf("DeleteHostNCApipaEndpoint url: %v for NC: %s", url, networkContainerID) payload := &cns.DeleteHostNCApipaEndpointRequest{ NetworkContainerID: networkContainerID, diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 7d581ee629..1fe9333fa8 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -60,6 +60,12 @@ const ( // protocolICMPv4 indicates the ICMPv4 protocol identifier in HCN protocolICMPv4 = "1" + + // aclPriority2000 indicates the ACL priority of 2000 + aclPriority2000 = 2000 + + // aclPriority200 indicates the ACL priority of 200 + aclPriority200 = 200 ) var ( @@ -219,6 +225,10 @@ func configureHostNCApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn } // Calculate subnet prefix + // Following code calculates the subnet prefix from localIPConfiguration IP + // e.g. IP: 169.254.128.7 Prefix length: 17 then resulting subnet prefix: 169.254.128.0/17 + // subnetPrefix: ffff8000 + // subnetPrefix.IP: 169.254.128.0 var ( subnetPrefix net.IPNet subnetPrefixStr string @@ -344,11 +354,11 @@ func configureAclSettingHostNCApipaEndpoint( ) if allowNCToHostCommunication { - log.Printf("[Azure CNS] Allowing NC to Host connectivity") + log.Printf("[Azure CNS] Allowing NC (%s) to Host (%s) connectivity", networkContainerApipaIP, hostApipaIP) } if allowHostToNCCommunication { - log.Printf("[Azure CNS] Allowing Host to NC connectivity") + log.Printf("[Azure CNS] Allowing Host (%s) to NC (%s) connectivity", hostApipaIP, networkContainerApipaIP) } // Iterate thru the protocol list and add ACL for each @@ -360,7 +370,7 @@ func configureAclSettingHostNCApipaEndpoint( Direction: hcn.DirectionTypeOut, LocalAddresses: networkContainerApipaIP, RuleType: hcn.RuleTypeSwitch, - Priority: 2000, + Priority: aclPriority2000, } if err = addAclToEndpointPolicy(outBlockAll, &endpointPolicies); err != nil { @@ -377,7 +387,7 @@ func configureAclSettingHostNCApipaEndpoint( LocalAddresses: networkContainerApipaIP, RemoteAddresses: hostApipaIP, RuleType: hcn.RuleTypeSwitch, - Priority: 200, + Priority: aclPriority200, } if err = addAclToEndpointPolicy(outAllowToHostOnly, &endpointPolicies); err != nil { @@ -392,7 +402,7 @@ func configureAclSettingHostNCApipaEndpoint( Direction: hcn.DirectionTypeIn, LocalAddresses: networkContainerApipaIP, RuleType: hcn.RuleTypeSwitch, - Priority: 2000, + Priority: aclPriority2000, } if err = addAclToEndpointPolicy(inBlockAll, &endpointPolicies); err != nil { @@ -409,7 +419,7 @@ func configureAclSettingHostNCApipaEndpoint( LocalAddresses: networkContainerApipaIP, RemoteAddresses: hostApipaIP, RuleType: hcn.RuleTypeSwitch, - Priority: 200, + Priority: aclPriority200, } if err = addAclToEndpointPolicy(inAllowFromHostOnly, &endpointPolicies); err != nil { @@ -449,6 +459,7 @@ func configureHostNCApipaEndpoint( if err != nil { log.Errorf("[Azure CNS] Failed to configure ACL for HostNCApipaEndpoint. Error: %v", err) + return nil, err } for _, endpointPolicy := range endpointPolicies { @@ -509,6 +520,9 @@ func CreateHostNCApipaEndpoint( return "", err } + log.Printf("[Azure CNS] Configuring HostNCApipaEndpoint: %s, in network: %s with localIPConfig: %+v", + endpointName, network.Id, localIPConfiguration) + if endpoint, err = configureHostNCApipaEndpoint( endpointName, network.Id, @@ -605,12 +619,14 @@ func DeleteHostNCApipaEndpoint( namedLock.LockAcquire(endpointName) defer namedLock.LockRelease(endpointName) + log.Debugf("[Azure CNS] Deleting HostNCApipaEndpoint: %s", endpointName) + if err := deleteEndpointByNameHnsV2(endpointName); err != nil { log.Errorf("[Azure CNS] Failed to delete HostNCApipaEndpoint: %s. Error: %v", endpointName, err) return err } - log.Debugf("[Azure CNS] Successfully deleted HostNCApipaEndpoint: %v", endpointName) + log.Debugf("[Azure CNS] Successfully deleted HostNCApipaEndpoint: %s", endpointName) namedLock.LockAcquire(hostNCApipaNetworkName) defer namedLock.LockRelease(hostNCApipaNetworkName) @@ -626,6 +642,7 @@ func DeleteHostNCApipaEndpoint( // Delete network if it doesn't have any endpoints if len(endpoints) == 0 { + log.Debugf("[Azure CNS] Deleting network with ID: %s", network.Id) if err = deleteNetworkByIDHnsV2(network.Id); err == nil { // Delete the loopback adapter created for this network networkcontainers.DeleteLoopbackAdapter(hostNCLoopbackAdapterName) diff --git a/common/namedlock.go b/common/namedlock.go index 3736ec340d..68f0a8b625 100644 --- a/common/namedlock.go +++ b/common/namedlock.go @@ -2,6 +2,8 @@ package common import ( "sync" + + "github.com/Azure/azure-container-networking/log" ) // NamedLock holds a mutex and a map of locks. Mutex is used to @@ -51,6 +53,8 @@ func (namedLock *NamedLock) LockRelease(lockName string) { if lock.refCount == 0 { delete(namedLock.lockMap, lockName) } + } else { + log.Printf("[Azure CNS] Attempt to unlock: %s without acquiring the lock", lockName) } } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index cc8e38cea9..a754e2ad4c 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -241,7 +241,8 @@ func (nw *network) createHostNCApipaEndpoint(epInfo *EndpointInfo) error { return err } - log.Printf("[net] Creating HostNCApipaEndpoint for host container connectivity") + log.Printf("[net] Creating HostNCApipaEndpoint for host container connectivity for NC: %s", + epInfo.NetworkContainerID) if hostNCApipaEndpointID, err = cnsClient.CreateHostNCApipaEndpoint(epInfo.NetworkContainerID); err != nil { From 3c1d2c7b9150300acc424857943e0fbe5b142225 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Fri, 18 Oct 2019 07:59:04 -0700 Subject: [PATCH 37/37] Remove extra log line --- network/endpoint_windows.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index a754e2ad4c..22a5ccefb6 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -381,8 +381,6 @@ func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { err error ) - log.Printf("[net] deleteEndpointImplHnsV2 DELETE id:%+v", ep) - if ep.AllowInboundFromHostToNC || ep.AllowInboundFromNCToHost { if err = nw.deleteHostNCApipaEndpoint(ep.NetworkContainerID); err != nil { log.Errorf("[net] Failed to delete HostNCApipaEndpoint due to error: %v", err)