From 4d8061bee00d59ed23cc3f00fbd7b2b183ee9cfd Mon Sep 17 00:00:00 2001
From: Onur Filiz <onur.filiz@microsoft.com>
Date: Wed, 7 Dec 2016 17:52:45 -0800
Subject: [PATCH 1/9] Separated source files for CNI and CNM targets

---
 Makefile | 31 ++++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index 77eb69086b..0c8e127c28 100644
--- a/Makefile
+++ b/Makefile
@@ -1,12 +1,5 @@
-SOURCEFILES = \
-	$(wildcard cni/*.go) \
-	$(wildcard cni/ipam/*.go) \
-	$(wildcard cni/network/*.go) \
-	$(wildcard cni/plugin/*.go) \
-	$(wildcard cnm/*.go) \
-	$(wildcard cnm/ipam/*.go) \
-	$(wildcard cnm/network/*.go) \
-	$(wildcard cnm/plugin/*.go) \
+# Source files common to all targets.
+COREFILES = \
 	$(wildcard common/*.go) \
 	$(wildcard ebtables/*.go) \
 	$(wildcard ipam/*.go) \
@@ -15,6 +8,22 @@ SOURCEFILES = \
 	$(wildcard network/*.go) \
 	$(wildcard store/*.go)
 
+# Source files for building CNM plugin.
+CNMFILES = \
+	$(wildcard cnm/*.go) \
+	$(wildcard cnm/ipam/*.go) \
+	$(wildcard cnm/network/*.go) \
+	$(wildcard cnm/plugin/*.go) \
+	$(COREFILES)
+
+# Source files for building CNI plugin.
+CNIFILES = \
+	$(wildcard cni/*.go) \
+	$(wildcard cni/ipam/*.go) \
+	$(wildcard cni/network/*.go) \
+	$(wildcard cni/plugin/*.go) \
+	$(COREFILES)
+
 CNMDIR = cnm/plugin
 
 CNIDIR = cni/plugin
@@ -35,11 +44,11 @@ clean:
 	rm -rf $(OUTPUTDIR)
 
 # Build the Azure CNM plugin.
-$(OUTPUTDIR)/azure-cnm-plugin: $(SOURCEFILES)
+$(OUTPUTDIR)/azure-cnm-plugin: $(CNMFILES)
 	go build -v -o $(OUTPUTDIR)/azure-cnm-plugin -ldflags "-X main.version=$(VERSION) -s -w" $(CNMDIR)/*.go
 
 # Build the Azure CNI plugin.
-$(OUTPUTDIR)/azure-cni-plugin: $(SOURCEFILES)
+$(OUTPUTDIR)/azure-cni-plugin: $(CNIFILES)
 	go build -v -o $(OUTPUTDIR)/azure-cni-plugin -ldflags "-X main.version=$(VERSION) -s -w" $(CNIDIR)/*.go
 
 install:

From 9a04ccd86683458bc3f881851d61c15d60ed5c63 Mon Sep 17 00:00:00 2001
From: Onur Filiz <onur.filiz@microsoft.com>
Date: Fri, 9 Dec 2016 16:53:12 -0800
Subject: [PATCH 2/9] Added GetPoolInfo API to expose DNS and GW information

---
 ipam/manager.go | 20 ++++++++++++++++++++
 ipam/pool.go    | 36 +++++++++++++++++++++++++++++++++++-
 ipam/utils.go   | 21 +++++++++++++++++++++
 3 files changed, 76 insertions(+), 1 deletion(-)
 create mode 100644 ipam/utils.go

diff --git a/ipam/manager.go b/ipam/manager.go
index a8cb4410ed..a5047bb1d4 100644
--- a/ipam/manager.go
+++ b/ipam/manager.go
@@ -34,6 +34,8 @@ type AddressManager interface {
 	GetDefaultAddressSpaces() (string, string)
 	RequestPool(asId, poolId, subPoolId string, options map[string]string, v6 bool) (string, string, error)
 	ReleasePool(asId, poolId string) error
+	GetPoolInfo(asId, poolId string) (*AddressPoolInfo, error)
+
 	RequestAddress(asId, poolId, address string, options map[string]string) (string, error)
 	ReleaseAddress(asId, poolId, address string) error
 }
@@ -254,6 +256,24 @@ func (am *addressManager) ReleasePool(asId string, poolId string) error {
 	return nil
 }
 
+// GetPoolInfo returns information about the given address pool.
+func (am *addressManager) GetPoolInfo(asId string, poolId string) (*AddressPoolInfo, error) {
+	am.Lock()
+	defer am.Unlock()
+
+	as, err := am.getAddressSpace(asId)
+	if err != nil {
+		return nil, err
+	}
+
+	ap, err := as.getAddressPool(poolId)
+	if err != nil {
+		return nil, err
+	}
+
+	return ap.getInfo(), nil
+}
+
 // RequestAddress reserves a new address from the address pool.
 func (am *addressManager) RequestAddress(asId, poolId, address string, options map[string]string) (string, error) {
 	am.Lock()
diff --git a/ipam/pool.go b/ipam/pool.go
index da07141ba4..026d7d8129 100644
--- a/ipam/pool.go
+++ b/ipam/pool.go
@@ -19,6 +19,13 @@ const (
 	globalScope = "global"
 )
 
+var (
+	// Azure VNET well-known host IDs.
+	defaultGatewayHostId = net.ParseIP("::1")
+	dnsPrimaryHostId     = net.ParseIP("::2")
+	dnsSecondaryHostId   = net.ParseIP("::3")
+)
+
 // Represents the key to an address pool.
 type addressPoolId struct {
 	AsId        string
@@ -47,6 +54,14 @@ type addressPool struct {
 	epoch     int
 }
 
+// AddressPoolInfo contains information about an address pool.
+type AddressPoolInfo struct {
+	Subnet     net.IPNet
+	Gateway    net.IP
+	DnsServers []net.IP
+	IsIPv6     bool
+}
+
 // Represents an IP address in a pool.
 type addressRecord struct {
 	Addr  net.IP
@@ -322,7 +337,26 @@ func (as *addressSpace) releasePool(poolId string) error {
 // AddressPool
 //
 
-// Returns if an addess pool is currently in use.
+// Returns if an address pool is currently in use.
+func (ap *addressPool) getInfo() *AddressPoolInfo {
+	// Generate default gateway address from subnet.
+	gateway := generateAddress(&ap.Subnet, defaultGatewayHostId)
+
+	// Generate DNS server addresses from subnet.
+	dnsPrimary := generateAddress(&ap.Subnet, dnsPrimaryHostId)
+	dnsSecondary := generateAddress(&ap.Subnet, dnsSecondaryHostId)
+
+	info := &AddressPoolInfo{
+		Subnet:     ap.Subnet,
+		Gateway:    gateway,
+		DnsServers: []net.IP{dnsPrimary, dnsSecondary},
+		IsIPv6:     ap.IsIPv6,
+	}
+
+	return info
+}
+
+// Returns if an address pool is currently in use.
 func (ap *addressPool) isInUse() bool {
 	return ap.RefCount > 0
 }
diff --git a/ipam/utils.go b/ipam/utils.go
new file mode 100644
index 0000000000..45fd98e55a
--- /dev/null
+++ b/ipam/utils.go
@@ -0,0 +1,21 @@
+// Copyright Microsoft Corp.
+// All rights reserved.
+
+package ipam
+
+import (
+	"net"
+)
+
+// generateAddress generates an IP address from the given network and host ID.
+func generateAddress(subnet *net.IPNet, hostId net.IP) net.IP {
+	// Use IPv6 addresses so it works both for IPv4 and IPv6.
+	address := net.ParseIP("::")
+	networkId := subnet.IP.To16()
+
+	for i := 0; i < len(address); i++ {
+		address[i] = networkId[i] | hostId[i]
+	}
+
+	return address
+}

From 23258937c70d8bb4a2115c77e1f95e1c988cf7f9 Mon Sep 17 00:00:00 2001
From: Onur Filiz <onur.filiz@microsoft.com>
Date: Fri, 9 Dec 2016 18:05:33 -0800
Subject: [PATCH 3/9] Moved all CNI IPAM logic to cni.ipam package

---
 cni/cni.go             |  60 +++++++++++++++++--
 cni/ipam/ipam.go       | 127 ++++++++++++++++++++++++++++++-----------
 cni/network/network.go | 125 ++++++++++++++++------------------------
 ipam/utils.go          |  11 ++++
 4 files changed, 210 insertions(+), 113 deletions(-)

diff --git a/cni/cni.go b/cni/cni.go
index 790c6f8819..0531a5c4a8 100644
--- a/cni/cni.go
+++ b/cni/cni.go
@@ -5,20 +5,70 @@ package cni
 
 import (
 	"encoding/json"
+
+	cniSkel "github.com/containernetworking/cni/pkg/skel"
+	cniTypes "github.com/containernetworking/cni/pkg/types"
+)
+
+const (
+	Internal = "internal"
+
+	CmdAdd = "ADD"
+	CmdDel = "DEL"
 )
 
+// CNI contract.
+type CniPlugin interface {
+	Add(args *cniSkel.CmdArgs) error
+	Delete(args *cniSkel.CmdArgs) error
+}
+
+// CallPlugin calls the given CNI plugin through the internal interface.
+func CallPlugin(plugin CniPlugin, cmd string, args *cniSkel.CmdArgs, nwCfg *NetworkConfig) (*cniTypes.Result, error) {
+	var err error
+
+	savedType := nwCfg.Ipam.Type
+	nwCfg.Ipam.Type = Internal
+	args.StdinData = nwCfg.Serialize()
+
+	// Call the plugin's internal interface.
+	if cmd == CmdAdd {
+		err = plugin.Add(args)
+	} else {
+		err = plugin.Delete(args)
+	}
+
+	nwCfg.Ipam.Type = savedType
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Read back the result.
+	var result cniTypes.Result
+	err = json.Unmarshal(args.StdinData, &result)
+	if err != nil {
+		return nil, err
+	}
+
+	return &result, nil
+}
+
 // NetworkConfig represents the Azure CNI plugin's network configuration.
 type NetworkConfig struct {
 	CniVersion string `json:"cniVersion"`
 	Name       string `json:"name"`
 	Type       string `json:"type"`
 	Bridge     string `json:"bridge,omitempty"`
-	IfName     string `json:"ifName,omitempty"`
+	LogLevel   string `json:"logLevel,omitempty"`
+	LogTarget  string `json:"logTarget,omitempty"`
 	Ipam       struct {
-		Type      string `json:"type"`
-		AddrSpace string `json:"addressSpace,omitempty"`
-		Subnet    string `json:"subnet,omitempty"`
-		Address   string `json:"ipAddress,omitempty"`
+		Type          string `json:"type"`
+		Environment   string `json:"environment,omitempty"`
+		AddrSpace     string `json:"addressSpace,omitempty"`
+		Subnet        string `json:"subnet,omitempty"`
+		Address       string `json:"ipAddress,omitempty"`
+		QueryInterval string `json:"queryInterval,omitempty"`
 	}
 }
 
diff --git a/cni/ipam/ipam.go b/cni/ipam/ipam.go
index 37d439586e..c29a41096b 100644
--- a/cni/ipam/ipam.go
+++ b/cni/ipam/ipam.go
@@ -4,6 +4,7 @@
 package ipam
 
 import (
+	"encoding/json"
 	"net"
 
 	"github.com/Azure/azure-container-networking/cni"
@@ -23,6 +24,15 @@ const (
 	defaultAddressSpaceId = "LocalDefaultAddressSpace"
 )
 
+var (
+	// Azure VNET pre-allocated host IDs.
+	ipv4DefaultGatewayHostId = net.ParseIP("::1")
+	ipv4DnsPrimaryHostId     = net.ParseIP("::2")
+	ipv4DnsSecondaryHostId   = net.ParseIP("::3")
+
+	ipv4DefaultRouteDstPrefix = net.IPNet{net.IPv4zero, net.IPv4Mask(0, 0, 0, 0)}
+)
+
 // IpamPlugin represents a CNI IPAM plugin.
 type ipamPlugin struct {
 	*common.Plugin
@@ -43,12 +53,15 @@ func NewPlugin(config *common.PluginConfig) (*ipamPlugin, error) {
 		return nil, err
 	}
 
-	config.IpamApi = am
-
-	return &ipamPlugin{
+	// Create IPAM plugin.
+	ipamPlg := &ipamPlugin{
 		Plugin: plugin,
 		am:     am,
-	}, nil
+	}
+
+	config.IpamApi = ipamPlg
+
+	return ipamPlg, nil
 }
 
 // Starts the plugin.
@@ -56,18 +69,18 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error {
 	// Initialize base plugin.
 	err := plugin.Initialize(config)
 	if err != nil {
-		log.Printf("[ipam] Failed to initialize base plugin, err:%v.", err)
+		log.Printf("[cni-ipam] Failed to initialize base plugin, err:%v.", err)
 		return err
 	}
 
 	// Initialize address manager.
 	err = plugin.am.Initialize(config, plugin.Options)
 	if err != nil {
-		log.Printf("[ipam] Failed to initialize address manager, err:%v.", err)
+		log.Printf("[cni-ipam] Failed to initialize address manager, err:%v.", err)
 		return err
 	}
 
-	log.Printf("[ipam] Plugin started.")
+	log.Printf("[cni-ipam] Plugin started.")
 
 	return nil
 }
@@ -76,7 +89,7 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error {
 func (plugin *ipamPlugin) Stop() {
 	plugin.am.Uninitialize()
 	plugin.Uninitialize()
-	log.Printf("[ipam] Plugin stopped.")
+	log.Printf("[cni-ipam] Plugin stopped.")
 }
 
 //
@@ -86,77 +99,125 @@ func (plugin *ipamPlugin) Stop() {
 
 // Add handles CNI add commands.
 func (plugin *ipamPlugin) Add(args *cniSkel.CmdArgs) error {
-	log.Printf("[ipam] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
+	log.Printf("[cni-ipam] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
 		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)
 
 	// Parse network configuration from stdin.
 	nwCfg, err := cni.ParseNetworkConfig(args.StdinData)
 	if err != nil {
-		log.Printf("[ipam] Failed to parse network configuration: %v.", err)
+		log.Printf("[cni-ipam] Failed to parse network configuration: %v.", err)
 		return nil
 	}
 
-	log.Printf("[ipam] Read network configuration %+v.", nwCfg)
+	log.Printf("[cni-ipam] Read network configuration %+v.", nwCfg)
 
 	// Assume default address space if not specified.
 	if nwCfg.Ipam.AddrSpace == "" {
 		nwCfg.Ipam.AddrSpace = defaultAddressSpaceId
 	}
 
+	var poolId string
+	var subnet string
+	var ipv4Address *net.IPNet
+	var result *cniTypes.Result
+	var apInfo *ipam.AddressPoolInfo
+
 	// Check if an address pool is specified.
 	if nwCfg.Ipam.Subnet == "" {
 		// Allocate an address pool.
-		poolId, subnet, err := plugin.am.RequestPool(nwCfg.Ipam.AddrSpace, "", "", nil, false)
+		poolId, subnet, err = plugin.am.RequestPool(nwCfg.Ipam.AddrSpace, "", "", nil, false)
 		if err != nil {
-			log.Printf("[ipam] Failed to allocate pool, err:%v.", err)
+			log.Printf("[cni-ipam] Failed to allocate pool, err:%v.", err)
 			return nil
 		}
 
 		nwCfg.Ipam.Subnet = subnet
-		log.Printf("[ipam] Allocated address poolId %v with subnet %v.", poolId, subnet)
+		log.Printf("[cni-ipam] Allocated address poolId %v with subnet %v.", poolId, subnet)
 	}
 
 	// Allocate an address for the endpoint.
 	address, err := plugin.am.RequestAddress(nwCfg.Ipam.AddrSpace, nwCfg.Ipam.Subnet, nwCfg.Ipam.Address, nil)
 	if err != nil {
-		log.Printf("[ipam] Failed to allocate address, err:%v.", err)
-		return nil
+		log.Printf("[cni-ipam] Failed to allocate address, err:%v.", err)
+		goto Rollback
 	}
 
-	log.Printf("[ipam] Allocated address %v.", address)
+	log.Printf("[cni-ipam] Allocated address %v.", address)
 
-	// Output the result.
-	ip, cidr, err := net.ParseCIDR(address)
-	cidr.IP = ip
+	// Parse IP address.
+	ipv4Address, err = ipam.ConvertAddressToIPNet(address)
 	if err != nil {
-		log.Printf("[ipam] Failed to parse address, err:%v.", err)
-		return nil
+		goto Rollback
+	}
+
+	// Query pool information for gateways and DNS servers.
+	apInfo, err = plugin.am.GetPoolInfo(nwCfg.Ipam.AddrSpace, nwCfg.Ipam.Subnet)
+	if err != nil {
+		goto Rollback
 	}
 
-	result := &cniTypes.Result{
-		IP4: &cniTypes.IPConfig{IP: *cidr},
+	// Populate IP configuration.
+	result = &cniTypes.Result{
+		IP4: &cniTypes.IPConfig{
+			IP:      *ipv4Address,
+			Gateway: apInfo.Gateway,
+			Routes: []cniTypes.Route{
+				cniTypes.Route{
+					Dst: ipv4DefaultRouteDstPrefix,
+					GW:  apInfo.Gateway,
+				},
+			},
+		},
 	}
 
-	result.Print()
+	// Populate DNS servers.
+	for _, ip := range apInfo.DnsServers {
+		result.DNS.Nameservers = append(result.DNS.Nameservers, ip.String())
+	}
+
+	// Output the result.
+	if nwCfg.Ipam.Type == cni.Internal {
+		// Called via the internal interface. Pass output back in args.
+		args.StdinData, _ = json.Marshal(result)
+	} else {
+		// Called via the executable interface. Print output to stdout.
+		result.Print()
+	}
+
+	log.Printf("[cni-ipam] ADD succeeded with output %+v.", result)
+
+	return nil
+
+Rollback:
+	// Roll back allocations made during this call.
+	log.Printf("[cni-ipam] ADD failed, err:%v.", err)
 
-	log.Printf("[ipam] ADD succeeded with output %+v.", result)
+	if address != "" {
+		log.Printf("[cni-ipam] Releasing address %v.", address)
+		err = plugin.am.ReleaseAddress(nwCfg.Ipam.AddrSpace, nwCfg.Ipam.Subnet, address)
+	}
+
+	if poolId != "" {
+		log.Printf("[cni-ipam] Releasing pool %v.", poolId)
+		err = plugin.am.ReleasePool(nwCfg.Ipam.AddrSpace, poolId)
+	}
 
 	return err
 }
 
 // Delete handles CNI delete commands.
 func (plugin *ipamPlugin) Delete(args *cniSkel.CmdArgs) error {
-	log.Printf("[ipam] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
+	log.Printf("[cni-ipam] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
 		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)
 
 	// Parse network configuration from stdin.
 	nwCfg, err := cni.ParseNetworkConfig(args.StdinData)
 	if err != nil {
-		log.Printf("[ipam] Failed to parse network configuration: %v.", err)
+		log.Printf("[cni-ipam] Failed to parse network configuration: %v.", err)
 		return nil
 	}
 
-	log.Printf("[ipam] Read network configuration %+v.", nwCfg)
+	log.Printf("[cni-ipam] Read network configuration %+v.", nwCfg)
 
 	// Assume default address space if not specified.
 	if nwCfg.Ipam.AddrSpace == "" {
@@ -168,19 +229,19 @@ func (plugin *ipamPlugin) Delete(args *cniSkel.CmdArgs) error {
 		// Release the address.
 		err := plugin.am.ReleaseAddress(nwCfg.Ipam.AddrSpace, nwCfg.Ipam.Subnet, nwCfg.Ipam.Address)
 		if err != nil {
-			log.Printf("[cni] Failed to release address, err:%v.", err)
+			log.Printf("[cni-ipam] Failed to release address, err:%v.", err)
 			return nil
 		}
 	} else {
 		// Release the pool.
 		err := plugin.am.ReleasePool(nwCfg.Ipam.AddrSpace, nwCfg.Ipam.Subnet)
 		if err != nil {
-			log.Printf("[cni] Failed to release pool, err:%v.", err)
+			log.Printf("[cni-ipam] Failed to release pool, err:%v.", err)
 			return nil
 		}
 	}
 
-	log.Printf("[ipam] DEL succeeded.")
+	log.Printf("[cni-ipam] DEL succeeded.")
 
-	return err
+	return nil
 }
diff --git a/cni/network/network.go b/cni/network/network.go
index c51f931cc6..50d939647a 100644
--- a/cni/network/network.go
+++ b/cni/network/network.go
@@ -4,11 +4,8 @@
 package network
 
 import (
-	"net"
-
 	"github.com/Azure/azure-container-networking/cni"
 	"github.com/Azure/azure-container-networking/common"
-	"github.com/Azure/azure-container-networking/ipam"
 	"github.com/Azure/azure-container-networking/log"
 	"github.com/Azure/azure-container-networking/network"
 
@@ -19,16 +16,13 @@ import (
 const (
 	// Plugin name.
 	name = "net"
-
-	// The default address space ID used when an explicit ID is not specified.
-	defaultAddressSpaceId = "LocalDefaultAddressSpace"
 )
 
 // NetPlugin object and its interface
 type netPlugin struct {
 	*common.Plugin
-	nm network.NetworkManager
-	am ipam.AddressManager
+	nm         network.NetworkManager
+	ipamPlugin cni.CniPlugin
 }
 
 // NewPlugin creates a new netPlugin object.
@@ -58,20 +52,20 @@ func (plugin *netPlugin) Start(config *common.PluginConfig) error {
 	// Initialize base plugin.
 	err := plugin.Initialize(config)
 	if err != nil {
-		log.Printf("[net] Failed to initialize base plugin, err:%v.", err)
+		log.Printf("[cni-net] Failed to initialize base plugin, err:%v.", err)
 		return err
 	}
 
 	// Initialize network manager.
 	err = plugin.nm.Initialize(config)
 	if err != nil {
-		log.Printf("[net] Failed to initialize network manager, err:%v.", err)
+		log.Printf("[cni-net] Failed to initialize network manager, err:%v.", err)
 		return err
 	}
 
-	plugin.am, _ = config.IpamApi.(ipam.AddressManager)
+	plugin.ipamPlugin = config.IpamApi.(cni.CniPlugin)
 
-	log.Printf("[net] Plugin started.")
+	log.Printf("[cni-net] Plugin started.")
 
 	return nil
 }
@@ -80,7 +74,7 @@ func (plugin *netPlugin) Start(config *common.PluginConfig) error {
 func (plugin *netPlugin) Stop() {
 	plugin.nm.Uninitialize()
 	plugin.Uninitialize()
-	log.Printf("[net] Plugin stopped.")
+	log.Printf("[cni-net] Plugin stopped.")
 }
 
 //
@@ -90,26 +84,20 @@ func (plugin *netPlugin) Stop() {
 
 // Add handles CNI add commands.
 func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
-	log.Printf("[cni] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
+	log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
 		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)
 
 	// Parse network configuration from stdin.
 	nwCfg, err := cni.ParseNetworkConfig(args.StdinData)
 	if err != nil {
-		log.Printf("[cni] Failed to parse network configuration, err:%v.", err)
+		log.Printf("[cni-net] Failed to parse network configuration, err:%v.", err)
 		return nil
 	}
 
-	log.Printf("[cni] Read network configuration %+v.", nwCfg)
-
-	// Assume default address space if not specified.
-	if nwCfg.Ipam.AddrSpace == "" {
-		nwCfg.Ipam.AddrSpace = defaultAddressSpaceId
-	}
+	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
 
 	// Initialize values from network config.
-	var poolId string
-	var subnet string
+	var result *cniTypes.Result
 	networkId := nwCfg.Name
 	endpointId := args.ContainerID
 
@@ -117,100 +105,85 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 	nwInfo, err := plugin.nm.GetNetworkInfo(networkId)
 	if err != nil {
 		// Network does not exist.
-		log.Printf("[cni] Creating network.")
+		log.Printf("[cni-net] Creating network.")
 
-		// Allocate an address pool for the network.
-		poolId, subnet, err = plugin.am.RequestPool(nwCfg.Ipam.AddrSpace, "", "", nil, false)
+		// Call into IPAM plugin to allocate an address pool for the network.
+		result, err = cni.CallPlugin(plugin.ipamPlugin, cni.CmdAdd, args, nwCfg)
 		if err != nil {
-			log.Printf("[cni] Failed to allocate pool, err:%v.", err)
+			log.Printf("[cni-net] Failed to allocate pool, err:%v.", err)
 			return nil
 		}
 
-		log.Printf("[cni] Allocated address pool %v with subnet %v.", poolId, subnet)
+		// Derive the subnet from allocated IP address.
+		subnet := result.IP4.IP
+		subnet.IP = subnet.IP.Mask(subnet.Mask)
+
+		log.Printf("[cni-net] IPAM plugin returned subnet %v and address %v.", subnet, result.IP4.IP.String())
 
 		// Create the network.
 		nwInfo := network.NetworkInfo{
 			Id:         networkId,
-			Subnets:    []string{subnet},
+			Subnets:    []string{subnet.String()},
 			BridgeName: nwCfg.Bridge,
 		}
 
 		err = plugin.nm.CreateNetwork(&nwInfo)
 		if err != nil {
-			log.Printf("[cni] Failed to create network, err:%v.", err)
+			log.Printf("[cni-net] Failed to create network, err:%v.", err)
 			return nil
 		}
 
-		log.Printf("[cni] Created network %v with subnet %v.", networkId, subnet)
+		log.Printf("[cni-net] Created network %v with subnet %v.", networkId, subnet)
 	} else {
 		// Network already exists.
-		log.Printf("[cni] Reusing network and pool.")
+		log.Printf("[cni-net] Found network %v with subnet %v.", networkId, nwInfo.Subnets[0])
 
-		// Query address pool.
-		poolId = nwInfo.Subnets[0]
-		subnet = nwInfo.Subnets[0]
-	}
-
-	// Allocate an address for the endpoint.
-	address, err := plugin.am.RequestAddress(nwCfg.Ipam.AddrSpace, poolId, "", nil)
-	if err != nil {
-		log.Printf("[cni] Failed to request address, err:%v.", err)
-		return nil
-	}
+		// Call into IPAM plugin to allocate an address for the endpoint.
+		nwCfg.Ipam.Subnet = nwInfo.Subnets[0]
+		result, err = cni.CallPlugin(plugin.ipamPlugin, cni.CmdAdd, args, nwCfg)
+		if err != nil {
+			log.Printf("[cni-net] Failed to allocate address, err:%v.", err)
+			return nil
+		}
 
-	ip, ipv4Address, err := net.ParseCIDR(address)
-	ipv4Address.IP = ip
-	if err != nil {
-		log.Printf("[cni] Failed to parse address %v, err:%v.", address, err)
-		return nil
+		log.Printf("[cni-net] IPAM plugin returned address %v.", result.IP4.IP.String())
 	}
 
-	log.Printf("[cni] Allocated address: %v", address)
-
 	// Create the endpoint.
-	epInfo := network.EndpointInfo{
+	epInfo := &network.EndpointInfo{
 		Id:          endpointId,
 		IfName:      args.IfName,
-		IPv4Address: *ipv4Address,
+		IPv4Address: result.IP4.IP,
 		NetNsPath:   args.Netns,
 	}
 
-	err = plugin.nm.CreateEndpoint(networkId, &epInfo)
+	err = plugin.nm.CreateEndpoint(networkId, epInfo)
 	if err != nil {
-		log.Printf("[cni] Failed to create endpoint, err:%v.", err)
+		log.Printf("[cni-net] Failed to create endpoint, err:%v.", err)
 		return nil
 	}
 
-	// Output the result.
-	result := &cniTypes.Result{
-		IP4: &cniTypes.IPConfig{IP: *ipv4Address},
-	}
-
+	// Output the result to stdout.
 	result.Print()
 
-	log.Printf("[cni] ADD succeeded with output %+v.", result)
+	log.Printf("[cni-net] ADD succeeded with output %+v.", result)
 
 	return nil
 }
 
 // Delete handles CNI delete commands.
 func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
-	log.Printf("[cni] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
+	log.Printf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
 		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)
 
 	// Parse network configuration from stdin.
 	nwCfg, err := cni.ParseNetworkConfig(args.StdinData)
 	if err != nil {
-		log.Printf("[cni] Failed to parse network configuration, err:%v.", err)
+		log.Printf("[cni-net] Failed to parse network configuration, err:%v.", err)
 		return nil
 	}
 
-	log.Printf("[cni] Read network configuration %+v.", nwCfg)
-
-	// Assume default address space if not specified.
-	if nwCfg.Ipam.AddrSpace == "" {
-		nwCfg.Ipam.AddrSpace = defaultAddressSpaceId
-	}
+	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
 
 	// Initialize values from network config.
 	networkId := nwCfg.Name
@@ -219,32 +192,34 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 	// Query the network.
 	nwInfo, err := plugin.nm.GetNetworkInfo(networkId)
 	if err != nil {
-		log.Printf("[cni] Failed to query network, err:%v.", err)
+		log.Printf("[cni-net] Failed to query network, err:%v.", err)
 		return nil
 	}
 
 	// Query the endpoint.
 	epInfo, err := plugin.nm.GetEndpointInfo(networkId, endpointId)
 	if err != nil {
-		log.Printf("[cni] Failed to query endpoint, err:%v.", err)
+		log.Printf("[cni-net] Failed to query endpoint, err:%v.", err)
 		return nil
 	}
 
 	// Delete the endpoint.
 	err = plugin.nm.DeleteEndpoint(networkId, endpointId)
 	if err != nil {
-		log.Printf("[cni] Failed to delete endpoint, err:%v.", err)
+		log.Printf("[cni-net] Failed to delete endpoint, err:%v.", err)
 		return nil
 	}
 
-	// Release the address.
-	err = plugin.am.ReleaseAddress(nwCfg.Ipam.AddrSpace, nwInfo.Subnets[0], epInfo.IPv4Address.IP.String())
+	// Call into IPAM plugin to release the endpoint's address.
+	nwCfg.Ipam.Subnet = nwInfo.Subnets[0]
+	nwCfg.Ipam.Address = epInfo.IPv4Address.IP.String()
+	_, err = cni.CallPlugin(plugin.ipamPlugin, cni.CmdDel, args, nwCfg)
 	if err != nil {
-		log.Printf("[cni] Failed to release address, err:%v.", err)
+		log.Printf("[cni-net] Failed to release address, err:%v.", err)
 		return nil
 	}
 
-	log.Printf("[cni] DEL succeeded.")
+	log.Printf("[cni-net] DEL succeeded.")
 
 	return nil
 }
diff --git a/ipam/utils.go b/ipam/utils.go
index 45fd98e55a..824664b546 100644
--- a/ipam/utils.go
+++ b/ipam/utils.go
@@ -19,3 +19,14 @@ func generateAddress(subnet *net.IPNet, hostId net.IP) net.IP {
 
 	return address
 }
+
+// ConvertAddressToIPNet returns the given IP address as an IPNet object.
+func ConvertAddressToIPNet(address string) (*net.IPNet, error) {
+	ip, ipnet, err := net.ParseCIDR(address)
+	if err != nil {
+		return nil, err
+	}
+
+	ipnet.IP = ip
+	return ipnet, nil
+}

From 3567ac1e5ab8d82f076f7e44801cc5eafe947650 Mon Sep 17 00:00:00 2001
From: Onur Filiz <onur.filiz@microsoft.com>
Date: Sun, 11 Dec 2016 18:40:36 -0800
Subject: [PATCH 4/9] Moved GetEndpointInfo logic to Endpoint class

---
 network/endpoint.go | 14 ++++++++++++++
 network/manager.go  |  7 +------
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/network/endpoint.go b/network/endpoint.go
index 8741f776ae..d00329f01c 100644
--- a/network/endpoint.go
+++ b/network/endpoint.go
@@ -246,6 +246,20 @@ func (nw *network) getEndpoint(endpointId string) (*endpoint, error) {
 	return ep, nil
 }
 
+//
+// Endpoint
+//
+
+// GetInfo returns information about the endpoint.
+func (ep *endpoint) getInfo() *EndpointInfo {
+	info := &EndpointInfo{
+		Id:          ep.Id,
+		IPv4Address: ep.IPv4Address,
+	}
+
+	return info
+}
+
 // Attach attaches an endpoint to a sandbox.
 func (ep *endpoint) attach(sandboxKey string, options map[string]interface{}) error {
 	if ep.SandboxKey != "" {
diff --git a/network/manager.go b/network/manager.go
index 5412a308c8..a0c62e307b 100644
--- a/network/manager.go
+++ b/network/manager.go
@@ -249,12 +249,7 @@ func (nm *networkManager) GetEndpointInfo(networkId string, endpointId string) (
 		return nil, err
 	}
 
-	epInfo := &EndpointInfo{
-		Id:          endpointId,
-		IPv4Address: ep.IPv4Address,
-	}
-
-	return epInfo, nil
+	return ep.getInfo(), nil
 }
 
 // AttachEndpoint attaches an endpoint to a sandbox.

From 9b63cd36358acbaab3bfb50492598a7212d79754 Mon Sep 17 00:00:00 2001
From: Onur Filiz <onur.filiz@microsoft.com>
Date: Mon, 12 Dec 2016 15:58:15 -0800
Subject: [PATCH 5/9] Exposed GetIpAddressFamily and updated comments for godoc

---
 netlink/netlink.go | 42 +++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/netlink/netlink.go b/netlink/netlink.go
index aba17a1ba2..7d98c3059c 100644
--- a/netlink/netlink.go
+++ b/netlink/netlink.go
@@ -10,12 +10,12 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-// Initializes netlink module.
+// Init initializes netlink module.
 func init() {
 	initEncoder()
 }
 
-// Sends a netlink echo request message.
+// Echo sends a netlink echo request message.
 func Echo(text string) error {
 	s, err := getSocket()
 	if err != nil {
@@ -32,7 +32,7 @@ func Echo(text string) error {
 	return s.sendAndWaitForAck(req)
 }
 
-// Adds a new network link of a specified type.
+// AddLink adds a new network interface of a specified type.
 func AddLink(name string, linkType string) error {
 	if name == "" || linkType == "" {
 		return fmt.Errorf("Invalid link name or type")
@@ -58,7 +58,7 @@ func AddLink(name string, linkType string) error {
 	return s.sendAndWaitForAck(req)
 }
 
-// Deletes a network link.
+// DeleteLink deletes a network interface.
 func DeleteLink(name string) error {
 	if name == "" {
 		return fmt.Errorf("Invalid link name")
@@ -83,7 +83,7 @@ func DeleteLink(name string) error {
 	return s.sendAndWaitForAck(req)
 }
 
-// Sets the name of a network interface.
+// SetLinkName sets the name of a network interface.
 func SetLinkName(name string, newName string) error {
 	s, err := getSocket()
 	if err != nil {
@@ -110,7 +110,7 @@ func SetLinkName(name string, newName string) error {
 	return s.sendAndWaitForAck(req)
 }
 
-// Sets the operational state of a network interface.
+// SetLinkState sets the operational state of a network interface.
 func SetLinkState(name string, up bool) error {
 	s, err := getSocket()
 	if err != nil {
@@ -141,7 +141,7 @@ func SetLinkState(name string, up bool) error {
 	return s.sendAndWaitForAck(req)
 }
 
-// Sets the master (upper) device of a network interface.
+// SetLinkMaster sets the master (upper) device of a network interface.
 func SetLinkMaster(name string, master string) error {
 	s, err := getSocket()
 	if err != nil {
@@ -177,7 +177,7 @@ func SetLinkMaster(name string, master string) error {
 	return s.sendAndWaitForAck(req)
 }
 
-// Sets the network namespace of a network interface.
+// SetLinkNetNs sets the network namespace of a network interface.
 func SetLinkNetNs(name string, fd uintptr) error {
 	s, err := getSocket()
 	if err != nil {
@@ -204,7 +204,7 @@ func SetLinkNetNs(name string, fd uintptr) error {
 	return s.sendAndWaitForAck(req)
 }
 
-// Sets the link layer hardware address of a network interface.
+// SetLinkAddress sets the link layer hardware address of a network interface.
 func SetLinkAddress(ifName string, hwAddress net.HardwareAddr) error {
 	s, err := getSocket()
 	if err != nil {
@@ -230,7 +230,7 @@ func SetLinkAddress(ifName string, hwAddress net.HardwareAddr) error {
 	return s.sendAndWaitForAck(req)
 }
 
-// Adds a new veth pair.
+// AddVethPair adds a new veth pair.
 func AddVethPair(name1 string, name2 string) error {
 	s, err := getSocket()
 	if err != nil {
@@ -262,8 +262,8 @@ func AddVethPair(name1 string, name2 string) error {
 	return s.sendAndWaitForAck(req)
 }
 
-// Returns the address family of an IP address.
-func getIpAddressFamily(ip net.IP) int {
+// GetIpAddressFamily returns the address family of an IP address.
+func GetIpAddressFamily(ip net.IP) int {
 	if len(ip) <= net.IPv4len {
 		return unix.AF_INET
 	}
@@ -273,7 +273,7 @@ func getIpAddressFamily(ip net.IP) int {
 	return unix.AF_INET6
 }
 
-// Sends an IP address set request.
+// setIpAddress sends an IP address set request.
 func setIpAddress(ifName string, ipAddress net.IP, ipNet *net.IPNet, add bool) error {
 	var msgType, flags int
 
@@ -297,7 +297,7 @@ func setIpAddress(ifName string, ipAddress net.IP, ipNet *net.IPNet, add bool) e
 
 	req := newRequest(msgType, flags)
 
-	family := getIpAddressFamily(ipAddress)
+	family := GetIpAddressFamily(ipAddress)
 
 	ifAddr := newIfAddrMsg(family)
 	ifAddr.Index = uint32(iface.Index)
@@ -318,12 +318,12 @@ func setIpAddress(ifName string, ipAddress net.IP, ipNet *net.IPNet, add bool) e
 	return s.sendAndWaitForAck(req)
 }
 
-// Adds an IP address to an interface.
+// AddIpAddress adds an IP address to a network interface.
 func AddIpAddress(ifName string, ipAddress net.IP, ipNet *net.IPNet) error {
 	return setIpAddress(ifName, ipAddress, ipNet, true)
 }
 
-// Deletes an IP address from an interface.
+// DeleteIpAddress deletes an IP address from a network interface.
 func DeleteIpAddress(ifName string, ipAddress net.IP, ipNet *net.IPNet) error {
 	return setIpAddress(ifName, ipAddress, ipNet, false)
 }
@@ -345,7 +345,7 @@ type Route struct {
 	ILinkIndex int
 }
 
-// Decodes a netlink message into a Route struct.
+// deserializeRoute decodes a netlink message into a Route struct.
 func deserializeRoute(msg *message) (*Route, error) {
 	// Parse route message.
 	rtmsg := deserializeRtMsg(msg.data)
@@ -388,7 +388,7 @@ func deserializeRoute(msg *message) (*Route, error) {
 	return &route, nil
 }
 
-// Returns a list of IP routes matching the given filter.
+// GetIpRoute returns a list of IP routes matching the given filter.
 func GetIpRoute(filter *Route) ([]*Route, error) {
 	s, err := getSocket()
 	if err != nil {
@@ -460,7 +460,7 @@ func GetIpRoute(filter *Route) ([]*Route, error) {
 	return routes, nil
 }
 
-// Sends an IP route set request.
+// setIpRoute sends an IP route set request.
 func setIpRoute(route *Route, add bool) error {
 	var msgType, flags int
 
@@ -528,12 +528,12 @@ func setIpRoute(route *Route, add bool) error {
 	return s.sendAndWaitForAck(req)
 }
 
-// Adds an IP route to the route table.
+// AddIpRoute adds an IP route to the route table.
 func AddIpRoute(route *Route) error {
 	return setIpRoute(route, true)
 }
 
-// Deletes an IP route from the route table.
+// DeleteIpRoute deletes an IP route from the route table.
 func DeleteIpRoute(route *Route) error {
 	return setIpRoute(route, false)
 }

From 0e13965ced4997de5f297b77a16e760dd8bfca08 Mon Sep 17 00:00:00 2001
From: Onur Filiz <onur.filiz@microsoft.com>
Date: Mon, 12 Dec 2016 16:03:10 -0800
Subject: [PATCH 6/9] Added support for multiple IPv4 addresses and routes

---
 cni/network/network.go | 30 ++++++++++++------
 cnm/network/network.go |  5 ++-
 network/endpoint.go    | 70 ++++++++++++++++++++++++++++--------------
 3 files changed, 69 insertions(+), 36 deletions(-)

diff --git a/cni/network/network.go b/cni/network/network.go
index 50d939647a..c7fd94bc39 100644
--- a/cni/network/network.go
+++ b/cni/network/network.go
@@ -151,10 +151,18 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 
 	// Create the endpoint.
 	epInfo := &network.EndpointInfo{
-		Id:          endpointId,
-		IfName:      args.IfName,
-		IPv4Address: result.IP4.IP,
-		NetNsPath:   args.Netns,
+		Id:        endpointId,
+		IfName:    args.IfName,
+		NetNsPath: args.Netns,
+	}
+
+	// Populate addresses and routes.
+	if result.IP4 != nil {
+		epInfo.IPAddresses = append(epInfo.IPAddresses, result.IP4.IP)
+
+		for _, route := range result.IP4.Routes {
+			epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW})
+		}
 	}
 
 	err = plugin.nm.CreateEndpoint(networkId, epInfo)
@@ -210,13 +218,15 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 		return nil
 	}
 
-	// Call into IPAM plugin to release the endpoint's address.
+	// Call into IPAM plugin to release the endpoint's addresses.
 	nwCfg.Ipam.Subnet = nwInfo.Subnets[0]
-	nwCfg.Ipam.Address = epInfo.IPv4Address.IP.String()
-	_, err = cni.CallPlugin(plugin.ipamPlugin, cni.CmdDel, args, nwCfg)
-	if err != nil {
-		log.Printf("[cni-net] Failed to release address, err:%v.", err)
-		return nil
+	for _, address := range epInfo.IPAddresses {
+		nwCfg.Ipam.Address = address.IP.String()
+		_, err = cni.CallPlugin(plugin.ipamPlugin, cni.CmdDel, args, nwCfg)
+		if err != nil {
+			log.Printf("[cni-net] Failed to release address, err:%v.", err)
+			return nil
+		}
 	}
 
 	log.Printf("[cni-net] DEL succeeded.")
diff --git a/cnm/network/network.go b/cnm/network/network.go
index 31ecfc2ace..6f3ea78b69 100644
--- a/cnm/network/network.go
+++ b/cnm/network/network.go
@@ -203,7 +203,7 @@ func (plugin *netPlugin) createEndpoint(w http.ResponseWriter, r *http.Request)
 
 	epInfo := network.EndpointInfo{
 		Id:          req.EndpointID,
-		IPv4Address: *ipv4Address,
+		IPAddresses: []net.IPNet{*ipv4Address},
 	}
 
 	err = plugin.nm.CreateEndpoint(req.NetworkID, &epInfo)
@@ -273,8 +273,7 @@ func (plugin *netPlugin) join(w http.ResponseWriter, r *http.Request) {
 
 	resp := joinResponse{
 		InterfaceName: ifname,
-		Gateway:       ep.IPv4Gateway.String(),
-		GatewayIPv6:   ep.IPv6Gateway.String(),
+		Gateway:       ep.Gateways[0].String(),
 	}
 
 	err = plugin.Listener.Encode(w, &resp)
diff --git a/network/endpoint.go b/network/endpoint.go
index d00329f01c..1fd320388b 100644
--- a/network/endpoint.go
+++ b/network/endpoint.go
@@ -27,10 +27,8 @@ type endpoint struct {
 	IfName      string
 	HostIfName  string
 	MacAddress  net.HardwareAddr
-	IPv4Address net.IPNet
-	IPv6Address net.IPNet
-	IPv4Gateway net.IP
-	IPv6Gateway net.IP
+	IPAddresses []net.IPNet
+	Gateways    []net.IP
 }
 
 // EndpointInfo contains read-only information about an endpoint.
@@ -38,7 +36,14 @@ type EndpointInfo struct {
 	Id          string
 	IfName      string
 	NetNsPath   string
-	IPv4Address net.IPNet
+	IPAddresses []net.IPNet
+	Routes      []RouteInfo
+}
+
+// RouteInfo contains information about an IP route.
+type RouteInfo struct {
+	Dst net.IPNet
+	Gw  net.IP
 }
 
 // NewEndpoint creates a new endpoint in the network.
@@ -54,8 +59,6 @@ func (nw *network) newEndpoint(epInfo *EndpointInfo) (*endpoint, error) {
 		return nil, errEndpointExists
 	}
 
-	ipAddr := epInfo.IPv4Address
-
 	// Create a veth pair.
 	hostIfName := fmt.Sprintf("%s%s", hostInterfacePrefix, epInfo.Id[:7])
 	contIfName := fmt.Sprintf("%s%s-2", hostInterfacePrefix, epInfo.Id[:7])
@@ -97,9 +100,11 @@ func (nw *network) newEndpoint(epInfo *EndpointInfo) (*endpoint, error) {
 
 	// Setup MAC address translation rules for container interface.
 	log.Printf("[net] Setting up MAC address translation rules for endpoint %v.", contIfName)
-	err = ebtables.SetupDnatBasedOnIPV4Address(ipAddr.IP.String(), containerIf.HardwareAddr.String())
-	if err != nil {
-		goto cleanup
+	for _, ipAddr := range epInfo.IPAddresses {
+		err = ebtables.SetupDnatBasedOnIPV4Address(ipAddr.IP.String(), containerIf.HardwareAddr.String())
+		if err != nil {
+			goto cleanup
+		}
 	}
 
 	// If a network namespace for the container interface is specified...
@@ -153,10 +158,29 @@ func (nw *network) newEndpoint(epInfo *EndpointInfo) (*endpoint, error) {
 	}
 
 	// Assign IP address to container network interface.
-	log.Printf("[net] Adding IP address %v to link %v.", ipAddr.String(), contIfName)
-	err = netlink.AddIpAddress(contIfName, ipAddr.IP, &ipAddr)
-	if err != nil {
-		goto cleanup
+	for _, ipAddr := range epInfo.IPAddresses {
+		log.Printf("[net] Adding IP address %v to link %v.", ipAddr.String(), contIfName)
+		err = netlink.AddIpAddress(contIfName, ipAddr.IP, &ipAddr)
+		if err != nil {
+			goto cleanup
+		}
+	}
+
+	// Add IP routes to container network interface.
+	for _, route := range epInfo.Routes {
+		log.Printf("[net] Adding IP route %+v to link %v.", route, contIfName)
+
+		nlRoute := &netlink.Route{
+			Family:    netlink.GetIpAddressFamily(route.Gw),
+			Dst:       &route.Dst,
+			Gw:        route.Gw,
+			LinkIndex: containerIf.Index,
+		}
+
+		err = netlink.AddIpRoute(nlRoute)
+		if err != nil {
+			goto cleanup
+		}
 	}
 
 	// If inside the container network namespace...
@@ -175,10 +199,8 @@ func (nw *network) newEndpoint(epInfo *EndpointInfo) (*endpoint, error) {
 		IfName:      contIfName,
 		HostIfName:  hostIfName,
 		MacAddress:  containerIf.HardwareAddr,
-		IPv4Address: ipAddr,
-		IPv6Address: net.IPNet{},
-		IPv4Gateway: nw.extIf.IPv4Gateway,
-		IPv6Gateway: nw.extIf.IPv6Gateway,
+		IPAddresses: epInfo.IPAddresses,
+		Gateways:    []net.IP{nw.extIf.IPv4Gateway},
 	}
 
 	nw.Endpoints[epInfo.Id] = ep
@@ -216,10 +238,12 @@ func (nw *network) deleteEndpoint(endpointId string) error {
 	}
 
 	// Delete MAC address translation rule.
-	log.Printf("[net] Deleting MAC address translation rule for endpoint %v.", endpointId)
-	err = ebtables.RemoveDnatBasedOnIPV4Address(ep.IPv4Address.IP.String(), ep.MacAddress.String())
-	if err != nil {
-		goto cleanup
+	log.Printf("[net] Deleting MAC address translation rules for endpoint %v.", endpointId)
+	for _, ipAddr := range ep.IPAddresses {
+		err = ebtables.RemoveDnatBasedOnIPV4Address(ipAddr.IP.String(), ep.MacAddress.String())
+		if err != nil {
+			goto cleanup
+		}
 	}
 
 	// Remove the endpoint object.
@@ -254,7 +278,7 @@ func (nw *network) getEndpoint(endpointId string) (*endpoint, error) {
 func (ep *endpoint) getInfo() *EndpointInfo {
 	info := &EndpointInfo{
 		Id:          ep.Id,
-		IPv4Address: ep.IPv4Address,
+		IPAddresses: ep.IPAddresses,
 	}
 
 	return info

From 8573b51facd6a56ef0174057712931ad42bf5047 Mon Sep 17 00:00:00 2001
From: Onur Filiz <onur.filiz@microsoft.com>
Date: Wed, 14 Dec 2016 15:42:02 -0800
Subject: [PATCH 7/9] Added Docker managed plugin config file

---
 cnm/plugin/config.json | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 cnm/plugin/config.json

diff --git a/cnm/plugin/config.json b/cnm/plugin/config.json
new file mode 100644
index 0000000000..bb00c07e34
--- /dev/null
+++ b/cnm/plugin/config.json
@@ -0,0 +1,15 @@
+{
+	"description": "Azure network plugin",
+	"documentation": "https://github.com/Azure/azure-container-networking/",
+	"entrypoint": ["/usr/bin/azure-cnm-plugin"],
+	"interface": {
+		"types": ["docker.networkdriver/1.0", "docker.ipamdriver/1.0"],
+		"socket": "azurenet.sock"
+	},
+	"network": {
+		"type": "host"
+	},
+	"linux": {
+		"capabilities": ["CAP_SYS_ADMIN", "CAP_NET_ADMIN"]
+	}
+}

From 25495a69f51b977ba0beee8a3c1ee50dac1944c9 Mon Sep 17 00:00:00 2001
From: Onur Filiz <onur.filiz@microsoft.com>
Date: Thu, 15 Dec 2016 13:34:29 -0800
Subject: [PATCH 8/9] Fixed help option alias

---
 common/config.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common/config.go b/common/config.go
index feac16a2fc..1f8da090a6 100644
--- a/common/config.go
+++ b/common/config.go
@@ -38,5 +38,5 @@ const (
 
 	// Help.
 	OptHelp      = "help"
-	OptHelpAlias = "?"
+	OptHelpAlias = "h"
 )

From 6373b2b46727b47a5643dd422ea39779b88512ae Mon Sep 17 00:00:00 2001
From: Onur Filiz <onur.filiz@microsoft.com>
Date: Thu, 15 Dec 2016 15:10:06 -0800
Subject: [PATCH 9/9] Added support for sharing the same listener between CNM
 net and IPAM plugins

---
 cnm/ipam/ipam.go       |  1 +
 cnm/network/network.go |  1 +
 cnm/plugin.go          | 48 ++++++++++++++++++++++++------------------
 common/listener.go     | 38 ++++++++++++++++++++-------------
 common/plugin.go       | 13 ++++++------
 5 files changed, 60 insertions(+), 41 deletions(-)

diff --git a/cnm/ipam/ipam.go b/cnm/ipam/ipam.go
index 1a6188cad8..ffba023e51 100644
--- a/cnm/ipam/ipam.go
+++ b/cnm/ipam/ipam.go
@@ -71,6 +71,7 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error {
 
 	// Add protocol handlers.
 	listener := plugin.Listener
+	listener.AddEndpoint(plugin.EndpointType)
 	listener.AddHandler(getCapabilitiesPath, plugin.getCapabilities)
 	listener.AddHandler(getAddressSpacesPath, plugin.getDefaultAddressSpaces)
 	listener.AddHandler(requestPoolPath, plugin.requestPool)
diff --git a/cnm/network/network.go b/cnm/network/network.go
index 6f3ea78b69..863d3e40e1 100644
--- a/cnm/network/network.go
+++ b/cnm/network/network.go
@@ -76,6 +76,7 @@ func (plugin *netPlugin) Start(config *common.PluginConfig) error {
 
 	// Add protocol handlers.
 	listener := plugin.Listener
+	listener.AddEndpoint(plugin.EndpointType)
 	listener.AddHandler(getCapabilitiesPath, plugin.getCapabilities)
 	listener.AddHandler(createNetworkPath, plugin.createNetwork)
 	listener.AddHandler(deleteNetworkPath, plugin.deleteNetwork)
diff --git a/cnm/plugin.go b/cnm/plugin.go
index cc6cd51cb0..9873c727b1 100644
--- a/cnm/plugin.go
+++ b/cnm/plugin.go
@@ -38,28 +38,36 @@ func (plugin *Plugin) Initialize(config *common.PluginConfig) error {
 	// Initialize the base plugin.
 	plugin.Plugin.Initialize(config)
 
-	// Create the plugin path.
-	os.MkdirAll(pluginPath, 0660)
-
-	// Create the listener.
-	var localAddr string
-	if plugin.Name != "test" {
-		localAddr = path.Join(pluginPath, config.Name+plugin.Name)
+	if config.Listener == nil {
+		// Create the plugin path.
+		os.MkdirAll(pluginPath, 0660)
+
+		// Create the listener.
+		var localAddr string
+		if plugin.Name != "test" {
+			localAddr = path.Join(pluginPath, config.Name+plugin.Name)
+		}
+
+		listener, err := common.NewListener("unix", localAddr)
+		if err != nil {
+			return err
+		}
+
+		// Add generic protocol handlers.
+		listener.AddHandler(activatePath, plugin.activate)
+
+		// Start the listener.
+		err = listener.Start(config.ErrChan)
+		if err != nil {
+			return err
+		}
+
+		config.Listener = listener
 	}
 
-	listener, err := common.NewListener("unix", localAddr)
-	if err != nil {
-		return err
-	}
-
-	// Add generic protocol handlers.
-	listener.AddHandler(activatePath, plugin.activate)
-
-	// Start the listener.
-	err = listener.Start(config.ErrChan)
-	plugin.Listener = listener
+	plugin.Listener = config.Listener
 
-	return err
+	return nil
 }
 
 // Uninitializes the plugin.
@@ -78,7 +86,7 @@ func (plugin *Plugin) activate(w http.ResponseWriter, r *http.Request) {
 
 	log.Request(plugin.Name, &req, nil)
 
-	resp := activateResponse{[]string{plugin.EndpointType}}
+	resp := activateResponse{Implements: plugin.Listener.GetEndpoints()}
 	err := plugin.Listener.Encode(w, &resp)
 
 	log.Response(plugin.Name, &resp, err)
diff --git a/common/listener.go b/common/listener.go
index 27a68e2668..cf6c7b7a13 100644
--- a/common/listener.go
+++ b/common/listener.go
@@ -13,15 +13,17 @@ import (
 	"github.com/Azure/azure-container-networking/log"
 )
 
-// Listener object
+// Listener represents an HTTP listener.
 type Listener struct {
 	protocol     string
 	localAddress string
+	endpoints    []string
+	active       bool
 	l            net.Listener
 	mux          *http.ServeMux
 }
 
-// Creates a new Listener.
+// NewListener creates a new Listener.
 func NewListener(protocol string, localAddress string) (*Listener, error) {
 	listener := Listener{
 		protocol:     protocol,
@@ -59,16 +61,17 @@ func (listener *Listener) Start(errChan chan error) error {
 		errChan <- http.Serve(listener.l, listener.mux)
 	}()
 
+	listener.active = true
 	return nil
 }
 
-// Stops listening for requests from libnetwork.
+// Stop stops listening for requests.
 func (listener *Listener) Stop() {
-
-	// Succeed early if no socket was requested.
-	if listener.localAddress == "" {
+	// Ignore if not active.
+	if !listener.active {
 		return
 	}
+	listener.active = false
 
 	// Stop servicing requests.
 	listener.l.Close()
@@ -81,17 +84,27 @@ func (listener *Listener) Stop() {
 	log.Printf("[Listener] Stopped listening on %s", listener.localAddress)
 }
 
-// Returns the HTTP mux for the listener.
+// GetMux returns the HTTP mux for the listener.
 func (listener *Listener) GetMux() *http.ServeMux {
 	return listener.mux
 }
 
-// Registers a protocol handler.
+// GetEndpoints returns the list of registered protocol endpoints.
+func (listener *Listener) GetEndpoints() []string {
+	return listener.endpoints
+}
+
+// AddEndpoint registers a protocol endpoint.
+func (listener *Listener) AddEndpoint(endpoint string) {
+	listener.endpoints = append(listener.endpoints, endpoint)
+}
+
+// AddHandler registers a protocol handler.
 func (listener *Listener) AddHandler(path string, handler func(http.ResponseWriter, *http.Request)) {
 	listener.mux.HandleFunc(path, handler)
 }
 
-// Decodes JSON payload.
+// Decode receives and decodes JSON payload to a request.
 func (listener *Listener) Decode(w http.ResponseWriter, r *http.Request, request interface{}) error {
 	var err error
 
@@ -108,7 +121,7 @@ func (listener *Listener) Decode(w http.ResponseWriter, r *http.Request, request
 	return err
 }
 
-// Encodes JSON payload.
+// Encode encodes and sends a response as JSON payload.
 func (listener *Listener) Encode(w http.ResponseWriter, response interface{}) error {
 	err := json.NewEncoder(w).Encode(response)
 	if err != nil {
@@ -117,8 +130,3 @@ func (listener *Listener) Encode(w http.ResponseWriter, response interface{}) er
 	}
 	return err
 }
-
-// Sends an error response.
-func (listener *Listener) SendError(w http.ResponseWriter, errMessage string) {
-	json.NewEncoder(w).Encode(map[string]string{"Err": errMessage})
-}
diff --git a/common/plugin.go b/common/plugin.go
index 437a25c513..140f0b03ea 100644
--- a/common/plugin.go
+++ b/common/plugin.go
@@ -26,12 +26,13 @@ type PluginApi interface {
 
 // Plugin common configuration.
 type PluginConfig struct {
-	Name    string
-	Version string
-	NetApi  interface{}
-	IpamApi interface{}
-	ErrChan chan error
-	Store   store.KeyValueStore
+	Name     string
+	Version  string
+	NetApi   interface{}
+	IpamApi  interface{}
+	Listener *Listener
+	ErrChan  chan error
+	Store    store.KeyValueStore
 }
 
 // NewPlugin creates a new Plugin object.