Skip to content

Commit

Permalink
EnableNonPersistent flag for Windows Overlay networks
Browse files Browse the repository at this point in the history
  • Loading branch information
@ksubrmnn
ksubrmnn committed May 21, 2019
1 parent ba49cd4 commit 5c7091b
Showing 1 changed file with 83 additions and 134 deletions.
217 changes: 83 additions & 134 deletions backend/vxlan/device_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,7 @@ package vxlan

import (
"encoding/json"
"fmt"
"github.com/Microsoft/hcsshim"
"github.com/Microsoft/hcsshim/hcn"
"github.com/buger/jsonparser"
"github.com/coreos/flannel/pkg/ip"
log "github.com/golang/glog"
"github.com/juju/errors"
Expand All @@ -37,19 +34,52 @@ type vxlanDeviceAttrs struct {
}

type vxlanDevice struct {
link *hcsshim.HNSNetwork
link *hcn.HostComputeNetwork
macPrefix string
directRouting bool
}

func newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) {
hnsNetwork := &hcsshim.HNSNetwork{
Name: devAttrs.name,
Type: "Overlay",
Subnets: make([]hcsshim.Subnet, 0, 1),
subnet := createSubnet(devAttrs.addressPrefix.String(), (devAttrs.addressPrefix.IP + 1).String(), "0.0.0.0/0")
network := &hcn.HostComputeNetwork{
Type: "Overlay",
Name: devAttrs.name,
Ipams: []hcn.Ipam{
{
Type: "Static",
Subnets: []hcn.Subnet{
*subnet,
},
},
},
Flags: hcn.EnableNonPersistent,
SchemaVersion: hcn.SchemaVersion{
Major: 2,
Minor: 0,
},
}

vsid := &hcn.VsidPolicySetting{
IsolationId: devAttrs.vni,
}
vsidJson, err := json.Marshal(vsid)
if err != nil {
return nil, err
}

sp := &hcn.SubnetPolicy{
Type: hcn.VSID,
}
sp.Settings = vsidJson

hnsNetwork, err := ensureNetwork(hnsNetwork, int64(devAttrs.vni), devAttrs.addressPrefix.String(), (devAttrs.addressPrefix.IP + 1).String())
spJson, err := json.Marshal(sp)
if err != nil {
return nil, err
}

network.Ipams[0].Subnets[0].Policies = append(network.Ipams[0].Subnets[0].Policies, spJson)

hnsNetwork, err := ensureNetwork(network, devAttrs.addressPrefix.String())
if err != nil {
return nil, err
}
Expand All @@ -59,87 +89,65 @@ func newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) {
}, nil
}

func ensureNetwork(expectedNetwork *hcsshim.HNSNetwork, expectedVSID int64, expectedAddressPrefix, expectedGW string) (*hcsshim.HNSNetwork, error) {
func ensureNetwork(expectedNetwork *hcn.HostComputeNetwork, expectedAddressPrefix string) (*hcn.HostComputeNetwork, error) {
createNetwork := true
networkName := expectedNetwork.Name

// 1. Check if the HNSNetwork exists and has the expected settings
existingNetwork, err := hcsshim.GetHNSNetworkByName(networkName)
// 1. Check if the HostComputeNetwork exists and has the expected settings
existingNetwork, err := hcn.GetNetworkByName(networkName)
if err == nil {
if existingNetwork.Type == expectedNetwork.Type {
for _, existingSubnet := range existingNetwork.Subnets {
if existingSubnet.AddressPrefix == expectedAddressPrefix && existingSubnet.GatewayAddress == expectedGW {
createNetwork = false
log.Infof("Found existing HNSNetwork %s", networkName)
break
}
if existingNetwork.Ipams[0].Subnets[0].IpAddressPrefix == expectedAddressPrefix {
createNetwork = false
log.Infof("Found existing HostComputeNetwork %s", networkName)
}
}
}

// 2. Create a new HNSNetwork
if createNetwork {
if existingNetwork != nil {
if _, err := existingNetwork.Delete(); err != nil {
return nil, errors.Annotatef(err, "failed to delete existing HNSNetwork %s", networkName)
if err := existingNetwork.Delete(); err != nil {
return nil, errors.Annotatef(err, "failed to delete existing HostComputeNetwork %s", networkName)
}
log.Infof("Deleted stale HNSNetwork %s", networkName)
log.Infof("Deleted stale HostComputeNetwork %s", networkName)
}

// Add a VxLan subnet
expectedNetwork.Subnets = append(expectedNetwork.Subnets, hcsshim.Subnet{
AddressPrefix: expectedAddressPrefix,
GatewayAddress: expectedGW,
Policies: []json.RawMessage{
[]byte(fmt.Sprintf(`{"Type":"VSID","VSID":%d}`, expectedVSID)),
},
})

// Config request params
jsonRequest, err := json.Marshal(expectedNetwork)
log.Infof("Attempting to create HostComputeNetwork %v", expectedNetwork)
newNetwork, err := expectedNetwork.Create()
if err != nil {
return nil, errors.Annotatef(err, "failed to marshal %+v", expectedNetwork)
}

log.Infof("Attempting to create HNSNetwork %s", string(jsonRequest))
newNetwork, err := hcsshim.HNSNetworkRequest("POST", "", string(jsonRequest))
if err != nil {
return nil, errors.Annotatef(err, "failed to create HNSNetwork %s", networkName)
return nil, errors.Annotatef(err, "failed to create HostComputeNetwork %s", networkName)
}

var waitErr, lastErr error
// Wait for the network to populate Management IP
log.Infof("Waiting to get ManagementIP from HNSNetwork %s", networkName)
log.Infof("Waiting to get ManagementIP from HostComputeNetwork %s", networkName)
waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
newNetwork, lastErr = hcsshim.HNSNetworkRequest("GET", newNetwork.Id, "")
return newNetwork != nil && len(newNetwork.ManagementIP) != 0, nil
newNetwork, lastErr = hcn.GetNetworkByID(newNetwork.Id)
return newNetwork != nil && len(getManagementIP(newNetwork)) != 0, nil
})
if waitErr == wait.ErrWaitTimeout {
return nil, errors.Annotatef(lastErr, "timeout, failed to get management IP from HNSNetwork %s", networkName)
return nil, errors.Annotatef(lastErr, "timeout, failed to get management IP from HostComputeNetwork %s", networkName)
}

managementIP := getManagementIP(newNetwork)
// Wait for the interface with the management IP
netshHelper := netsh.New(utilexec.New())
log.Infof("Waiting to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
log.Infof("Waiting to get net interface for HostComputeNetwork %s (%s)", networkName, managementIP)
waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
_, lastErr = netshHelper.GetInterfaceByIP(newNetwork.ManagementIP)
_, lastErr = netshHelper.GetInterfaceByIP(managementIP)
return lastErr == nil, nil
})
if waitErr == wait.ErrWaitTimeout {
return nil, errors.Annotatef(lastErr, "timeout, failed to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
return nil, errors.Annotatef(lastErr, "timeout, failed to get net interface for HostComputeNetwork %s (%s)", networkName, managementIP)
}

log.Infof("Created HNSNetwork %s", networkName)
log.Infof("Created HostComputeNetwork %s", networkName)
existingNetwork = newNetwork
}

existingNetworkV2, err := hcn.GetNetworkByID(existingNetwork.Id)
if err != nil {
return nil, errors.Annotatef(err, "Could not find vxlan0 in V2")
}

addHostRoute := true
for _, policy := range existingNetworkV2.Policies {
for _, policy := range existingNetwork.Policies {
if policy.Type == hcn.HostRoute {
addHostRoute = false
}
Expand All @@ -153,96 +161,37 @@ func ensureNetwork(expectedNetwork *hcsshim.HNSNetwork, expectedVSID int64, expe
networkRequest := hcn.PolicyNetworkRequest{
Policies: []hcn.NetworkPolicy{hostRoutePolicy},
}
existingNetworkV2.AddPolicy(networkRequest)
err = existingNetwork.AddPolicy(networkRequest)
if err != nil {
log.Infof("Could not apply HostRoute policy for local host to local pod connectivity. This policy requires Windows 18321.1000.19h1_release.190117-1502 or newer")
}
}

return existingNetwork, nil
}

type neighbor struct {
MAC string
IP ip.IP4
ManagementAddress string
}

func (dev *vxlanDevice) AddEndpoint(n *neighbor) error {
endpointName := createEndpointName(n.IP)

// 1. Check if the HNSEndpoint exists and has the expected settings
existingEndpoint, err := hcsshim.GetHNSEndpointByName(endpointName)
if err == nil && existingEndpoint.VirtualNetwork == dev.link.Id {
// Check policies if there is PA type
targetType := "PA"
for _, policy := range existingEndpoint.Policies {
policyType, _ := jsonparser.GetUnsafeString(policy, "Type")
if policyType == targetType {
actualPaIP, _ := jsonparser.GetUnsafeString(policy, targetType)
if actualPaIP == n.ManagementAddress {
log.Infof("Found existing remote HNSEndpoint %s", endpointName)
return nil
}
func getManagementIP(network *hcn.HostComputeNetwork) string {
for _, policy := range network.Policies {
if policy.Type == hcn.ProviderAddress {
policySettings := hcn.ProviderAddressEndpointPolicySetting{}
err := json.Unmarshal(policy.Settings, &policySettings)
if err != nil {
return ""
}
return policySettings.ProviderAddress
}
}

// 2. Create a new HNSNetwork
if existingEndpoint != nil {
if _, err := existingEndpoint.Delete(); err != nil {
return errors.Annotatef(err, "failed to delete existing remote HNSEndpoint %s", endpointName)
}
log.V(4).Infof("Deleted stale HNSEndpoint %s", endpointName)
}

newEndpoint := &hcsshim.HNSEndpoint{
Name: endpointName,
IPAddress: n.IP.ToIP(),
MacAddress: n.MAC,
VirtualNetwork: dev.link.Id,
IsRemoteEndpoint: true,
Policies: []json.RawMessage{
[]byte(fmt.Sprintf(`{"Type":"PA","PA":"%s"}`, n.ManagementAddress)),
},
}
if _, err := newEndpoint.Create(); err != nil {
return errors.Annotatef(err, "failed to create remote HNSEndpoint %s", endpointName)
}
log.V(4).Infof("Created HNSEndpoint %s", endpointName)

return nil
return ""
}

func (dev *vxlanDevice) DelEndpoint(n *neighbor) error {
endpointName := createEndpointName(n.IP)

existingEndpoint, err := hcsshim.GetHNSEndpointByName(endpointName)
if err == nil && existingEndpoint.VirtualNetwork == dev.link.Id {
// Check policies if there is PA type
targetType := "PA"
for _, policy := range existingEndpoint.Policies {
policyType, _ := jsonparser.GetUnsafeString(policy, "Type")
if policyType == targetType {
actualPaIP, _ := jsonparser.GetUnsafeString(policy, targetType)
if actualPaIP == n.ManagementAddress {
// Found it and delete
if _, err := existingEndpoint.Delete(); err != nil {
return errors.Annotatef(err, "failed to delete remote HNSEndpoint %s", endpointName)
}

log.V(4).Infof("Deleted HNSEndpoint %s", endpointName)
break
}
}
}
func createSubnet(AddressPrefix string, NextHop string, DestPrefix string) *hcn.Subnet {
return &hcn.Subnet{
IpAddressPrefix: AddressPrefix,
Routes: []hcn.Route{
{
NextHop: NextHop,
DestinationPrefix: DestPrefix,
},
},
}

return nil
}

func (dev *vxlanDevice) ConjureMac(targetIP ip.IP4) string {
a, b, c, d := targetIP.Octets()
return fmt.Sprintf("%v-%02x-%02x-%02x-%02x", dev.macPrefix, a, b, c, d)
}

func createEndpointName(targetIP ip.IP4) string {
return "remote_" + targetIP.String()
}

0 comments on commit 5c7091b

Please sign in to comment.