Merge pull request #8208 from hashicorp/f-multi-network

multi-interface network support

api/compose_test.go

@@ -20,7 +20,7 @@ func TestCompose(t *testing.T) {
 				{
 					CIDR:          "0.0.0.0/0",
 					MBits:         intToPtr(100),
-					ReservedPorts: []Port{{"", 80, 0}, {"", 443, 0}},
+					ReservedPorts: []Port{{"", 80, 0, ""}, {"", 443, 0, ""}},
 				},
 			},
 		})
@@ -111,8 +111,8 @@ func TestCompose(t *testing.T) {
 									CIDR:  "0.0.0.0/0",
 									MBits: intToPtr(100),
 									ReservedPorts: []Port{
-										{"", 80, 0},
-										{"", 443, 0},
+										{"", 80, 0, ""},
+										{"", 443, 0, ""},
 									},
 								},
 							},

api/resources.go

@@ -84,9 +84,10 @@ func (r *Resources) Merge(other *Resources) {
 }
 
 type Port struct {
-	Label string
-	Value int `mapstructure:"static"`
-	To    int `mapstructure:"to"`
+	Label       string
+	Value       int    `mapstructure:"static"`
+	To          int    `mapstructure:"to"`
+	HostNetwork string `mapstructure:"host_network"`
 }
 
 type DNSConfig struct {

api/tasks_test.go

@@ -269,7 +269,7 @@ func TestTask_Require(t *testing.T) {
 			{
 				CIDR:          "0.0.0.0/0",
 				MBits:         intToPtr(100),
-				ReservedPorts: []Port{{"", 80, 0}, {"", 443, 0}},
+				ReservedPorts: []Port{{"", 80, 0, ""}, {"", 443, 0, ""}},
 			},
 		},
 	}

client/allocrunner/networking_cni.go

@@ -164,16 +164,33 @@ func (c *cniNetworkConfigurator) ensureCNIInitialized() error {
 // portmapping capability arguments for the portmap CNI plugin
 func getPortMapping(alloc *structs.Allocation) []cni.PortMapping {
 	ports := []cni.PortMapping{}
-	for _, network := range alloc.AllocatedResources.Shared.Networks {
-		for _, port := range append(network.DynamicPorts, network.ReservedPorts...) {
+
+	if len(alloc.AllocatedResources.Shared.Ports) == 0 && len(alloc.AllocatedResources.Shared.Networks) > 0 {
+		for _, network := range alloc.AllocatedResources.Shared.Networks {
+			for _, port := range append(network.DynamicPorts, network.ReservedPorts...) {
+				if port.To < 1 {
+					port.To = port.Value
+				}
+				for _, proto := range []string{"tcp", "udp"} {
+					ports = append(ports, cni.PortMapping{
+						HostPort:      int32(port.Value),
+						ContainerPort: int32(port.To),
+						Protocol:      proto,
+					})
+				}
+			}
+		}
+	} else {
+		for _, port := range alloc.AllocatedResources.Shared.Ports {
 			if port.To < 1 {
-				continue
+				port.To = port.Value
 			}
 			for _, proto := range []string{"tcp", "udp"} {
 				ports = append(ports, cni.PortMapping{
 					HostPort:      int32(port.Value),
 					ContainerPort: int32(port.To),
 					Protocol:      proto,
+					HostIP:        port.HostIP,
 				})
 			}
 		}

client/client_test.go

@@ -1113,8 +1113,9 @@ func TestClient_UpdateNodeFromDevicesAccumulates(t *testing.T) {
 	// initial check
 	expectedResources := &structs.NodeResources{
 		// computed through test client initialization
-		Networks: client.configCopy.Node.NodeResources.Networks,
-		Disk:     client.configCopy.Node.NodeResources.Disk,
+		Networks:     client.configCopy.Node.NodeResources.Networks,
+		NodeNetworks: client.configCopy.Node.NodeResources.NodeNetworks,
+		Disk:         client.configCopy.Node.NodeResources.Disk,
 
 		// injected
 		Cpu:    structs.NodeCpuResources{CpuShares: 123},
@@ -1150,8 +1151,9 @@ func TestClient_UpdateNodeFromDevicesAccumulates(t *testing.T) {
 
 	expectedResources2 := &structs.NodeResources{
 		// computed through test client initialization
-		Networks: client.configCopy.Node.NodeResources.Networks,
-		Disk:     client.configCopy.Node.NodeResources.Disk,
+		Networks:     client.configCopy.Node.NodeResources.Networks,
+		NodeNetworks: client.configCopy.Node.NodeResources.NodeNetworks,
+		Disk:         client.configCopy.Node.NodeResources.Disk,
 
 		// injected
 		Cpu:    structs.NodeCpuResources{CpuShares: 123},

client/config/config.go

@@ -254,6 +254,9 @@ type Config struct {
 
 	// HostVolumes is a map of the configured host volumes by name.
 	HostVolumes map[string]*structs.ClientHostVolumeConfig
+
+	// HostNetworks is a map of the conigured host networks by name.
+	HostNetworks map[string]*structs.ClientHostNetworkConfig
 }
 
 type ClientTemplateConfig struct {
@@ -313,6 +316,7 @@ func DefaultConfig() *Config {
 		CNIPath:                    "/opt/cni/bin",
 		CNIConfigDir:               "/opt/cni/config",
 		CNIInterfacePrefix:         "eth",
+		HostNetworks:               map[string]*structs.ClientHostNetworkConfig{},
 	}
 }
 

client/fingerprint/bridge_linux.go

@@ -23,6 +23,12 @@ func (f *BridgeFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerpri
 				Mode: "bridge",
 			},
 		},
+		NodeNetworks: []*structs.NodeNetworkResource{
+			{
+				Mode:   "bridge",
+				Device: req.Config.BridgeNetworkName,
+			},
+		},
 	}
 	resp.Detected = true
 	return nil

client/fingerprint/cni.go

@@ -58,16 +58,22 @@ func (f *CNIFingerprint) Fingerprint(req *FingerprintRequest, resp *FingerprintR
 	}
 
 	var nodeNetworks structs.Networks
+	var newNodeNetworks []*structs.NodeNetworkResource
 
 	for name := range networks {
+		mode := fmt.Sprintf("cni/%s", name)
 		nodeNetworks = append(nodeNetworks, &structs.NetworkResource{
-			Mode: fmt.Sprintf("cni/%s", name),
+			Mode: mode,
+		})
+		newNodeNetworks = append(newNodeNetworks, &structs.NodeNetworkResource{
+			Mode: mode,
 		})
 		f.logger.Debug("detected CNI network", "name", name)
 	}
 
 	resp.NodeResources = &structs.NodeResources{
-		Networks: nodeNetworks,
+		Networks:     nodeNetworks,
+		NodeNetworks: newNodeNetworks,
 	}
 
 	resp.Detected = true

client/fingerprint/network.go

@@ -3,9 +3,12 @@ package fingerprint
 import (
 	"fmt"
 	"net"
+	"strings"
 
 	log "github.com/hashicorp/go-hclog"
 	sockaddr "github.com/hashicorp/go-sockaddr"
+	"github.com/hashicorp/go-sockaddr/template"
+	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -113,11 +116,135 @@ func (f *NetworkFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerpr
 	if len(nwResources) > 0 {
 		resp.AddAttribute("unique.network.ip-address", nwResources[0].IP)
 	}
+
+	ifaces, err := f.interfaceDetector.Interfaces()
+	if err != nil {
+		return err
+	}
+	nodeNetResources, err := f.createNodeNetworkResources(ifaces, disallowLinkLocal, req.Config)
+	if err != nil {
+		return err
+	}
+	resp.NodeResources.NodeNetworks = nodeNetResources
+
 	resp.Detected = true
 
 	return nil
 }
 
+func (f *NetworkFingerprint) createNodeNetworkResources(ifaces []net.Interface, disallowLinkLocal bool, conf *config.Config) ([]*structs.NodeNetworkResource, error) {
+	nets := make([]*structs.NodeNetworkResource, 0)
+	for _, iface := range ifaces {
+		speed := f.linkSpeed(iface.Name)
+		if speed == 0 {
+			speed = defaultNetworkSpeed
+			f.logger.Debug("link speed could not be detected, falling back to default speed", "mbits", defaultNetworkSpeed)
+		}
+
+		newNetwork := &structs.NodeNetworkResource{
+			Mode:       "host",
+			Device:     iface.Name,
+			MacAddress: iface.HardwareAddr.String(),
+			Speed:      speed,
+		}
+		addrs, err := f.interfaceDetector.Addrs(&iface)
+		if err != nil {
+			return nil, err
+		}
+		var networkAddrs, linkLocalAddrs []structs.NodeNetworkAddress
+		for _, addr := range addrs {
+			// Find the IP Addr and the CIDR from the Address
+			var ip net.IP
+			var family structs.NodeNetworkAF
+			switch v := (addr).(type) {
+			case *net.IPNet:
+				ip = v.IP
+			case *net.IPAddr:
+				ip = v.IP
+			}
+
+			if ip.To4() != nil {
+				family = structs.NodeNetworkAF_IPv4
+			} else {
+				family = structs.NodeNetworkAF_IPv6
+			}
+			newAddr := structs.NodeNetworkAddress{
+				Address: ip.String(),
+				Family:  family,
+				Alias:   deriveAddressAlias(iface, ip, conf),
+			}
+
+			if newAddr.Alias != "" {
+				if ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
+					linkLocalAddrs = append(linkLocalAddrs, newAddr)
+				} else {
+					networkAddrs = append(networkAddrs, newAddr)
+				}
+			}
+		}
+
+		if len(networkAddrs) == 0 && len(linkLocalAddrs) > 0 {
+			if disallowLinkLocal {
+				f.logger.Debug("ignoring detected link-local address on interface", "interface", iface.Name)
+			} else {
+				newNetwork.Addresses = linkLocalAddrs
+			}
+		} else {
+			newNetwork.Addresses = networkAddrs
+		}
+
+		if len(newNetwork.Addresses) > 0 {
+			nets = append(nets, newNetwork)
+		}
+	}
+	return nets, nil
+}
+
+func deriveAddressAlias(iface net.Interface, addr net.IP, config *config.Config) string {
+	for name, conf := range config.HostNetworks {
+		var cidrMatch, ifaceMatch bool
+		if conf.CIDR != "" {
+			for _, cidr := range strings.Split(conf.CIDR, ",") {
+				_, ipnet, err := net.ParseCIDR(cidr)
+				if err != nil {
+					continue
+				}
+
+				if ipnet.Contains(addr) {
+					cidrMatch = true
+					break
+				}
+			}
+		} else {
+			cidrMatch = true
+		}
+		if conf.Interface != "" {
+			ifaceName, err := template.Parse(conf.Interface)
+			if err != nil {
+				continue
+			}
+
+			if ifaceName == iface.Name {
+				ifaceMatch = true
+			}
+		} else {
+			ifaceMatch = true
+		}
+		if cidrMatch && ifaceMatch {
+			return name
+		}
+	}
+
+	ri, err := sockaddr.NewRouteInfo()
+	if err == nil {
+		defaultIface, err := ri.GetDefaultInterfaceName()
+		if err == nil && iface.Name == defaultIface {
+			return "default"
+		}
+	}
+	return ""
+}
+
 // createNetworkResources creates network resources for every IP
 func (f *NetworkFingerprint) createNetworkResources(throughput int, intf *net.Interface, disallowLinkLocal bool) ([]*structs.NetworkResource, error) {
 	// Find the interface with the name

client/fingerprint/network_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -197,6 +198,8 @@ func TestNetworkFingerprint_basic(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
+	spew.Dump(response)
+	os.Exit(0)
 	if !response.Detected {
 		t.Fatalf("expected response to be applicable")
 	}

client/taskenv/env.go

@@ -70,15 +70,21 @@ const (
 	// The ip:port are always the host's.
 	AddrPrefix = "NOMAD_ADDR_"
 
+	HostAddrPrefix = "NOMAD_HOST_ADDR_"
+
 	// IpPrefix is the prefix for passing the host IP of a port allocation
 	// to a task.
 	IpPrefix = "NOMAD_IP_"
 
+	HostIpPrefix = "NOMAD_HOST_IP_"
+
 	// PortPrefix is the prefix for passing the port allocation to a task.
 	// It will be the task's port if a port map is specified. Task's should
 	// bind to this port.
 	PortPrefix = "NOMAD_PORT_"
 
+	AllocPortPrefix = "NOMAD_ALLOC_PORT_"
+
 	// HostPortPrefix is the prefix for passing the host port when a port
 	// map is specified.
 	HostPortPrefix = "NOMAD_HOST_PORT_"
@@ -620,6 +626,7 @@ func (b *Builder) setAlloc(alloc *structs.Allocation) *Builder {
 			}
 		}
 
+		// COMPAT(1.0): remove in 1.0 when AllocatedPorts can be used exclusively
 		// Add ports from other tasks
 		for taskName, resources := range alloc.AllocatedResources.Tasks {
 			// Add ports from other tasks
@@ -637,6 +644,7 @@ func (b *Builder) setAlloc(alloc *structs.Allocation) *Builder {
 			}
 		}
 
+		// COMPAT(1.0): remove in 1.0 when AllocatedPorts can be used exclusively
 		// Add ports from group networks
 		//TODO Expose IPs but possibly only via variable interpolation
 		for _, nw := range alloc.AllocatedResources.Shared.Networks {
@@ -647,6 +655,11 @@ func (b *Builder) setAlloc(alloc *structs.Allocation) *Builder {
 				addGroupPort(b.otherPorts, p)
 			}
 		}
+
+		// Add any allocated host ports
+		if alloc.AllocatedResources.Shared.Ports != nil {
+			addPorts(b.otherPorts, alloc.AllocatedResources.Shared.Ports)
+		}
 	}
 
 	upstreams := []structs.ConsulUpstream{}
@@ -857,3 +870,23 @@ func addGroupPort(m map[string]string, port structs.Port) {
 
 	m[HostPortPrefix+port.Label] = strconv.Itoa(port.Value)
 }
+
+func addPorts(m map[string]string, ports structs.AllocatedPorts) {
+	for _, p := range ports {
+		m[AddrPrefix+p.Label] = fmt.Sprintf("%s:%d", p.HostIP, p.Value)
+		m[HostAddrPrefix+p.Label] = fmt.Sprintf("%s:%d", p.HostIP, p.Value)
+		m[IpPrefix+p.Label] = p.HostIP
+		m[HostIpPrefix+p.Label] = p.HostIP
+		if p.To > 0 {
+			val := strconv.Itoa(p.To)
+			m[PortPrefix+p.Label] = val
+			m[AllocPortPrefix+p.Label] = val
+		} else {
+			val := strconv.Itoa(p.Value)
+			m[PortPrefix+p.Label] = val
+			m[AllocPortPrefix+p.Label] = val
+		}
+
+		m[HostPortPrefix+p.Label] = strconv.Itoa(p.Value)
+	}
+}