From 45aaff4a486413e2b7f43f1ca68b96f1d10f8ac6 Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Wed, 19 Feb 2020 17:06:59 -0800
Subject: [PATCH 01/12] added cniv6 plugin

---
 cni/cniv6/cniipv6.go     | 128 +++++++++++++++++++++++++
 cni/cniv6/plugin/main.go | 199 +++++++++++++++++++++++++++++++++++++++
 cni/network/network.go   |  34 ++++++-
 cni/plugin.go            |   1 +
 telemetry/constants.go   |   9 +-
 5 files changed, 362 insertions(+), 9 deletions(-)
 create mode 100644 cni/cniv6/cniipv6.go
 create mode 100644 cni/cniv6/plugin/main.go

diff --git a/cni/cniv6/cniipv6.go b/cni/cniv6/cniipv6.go
new file mode 100644
index 0000000000..0a30b1b89d
--- /dev/null
+++ b/cni/cniv6/cniipv6.go
@@ -0,0 +1,128 @@
+// Copyright 2017 Microsoft. All rights reserved.
+// MIT License
+
+package cniv6
+
+import (
+	"github.com/Azure/azure-container-networking/cni"
+	"github.com/Azure/azure-container-networking/common"
+	"github.com/Azure/azure-container-networking/ipam"
+	"github.com/Azure/azure-container-networking/log"
+	"github.com/Azure/azure-container-networking/platform"
+	"github.com/Azure/azure-container-networking/telemetry"
+
+	cniSkel "github.com/containernetworking/cni/pkg/skel"
+	cniTypesCurr "github.com/containernetworking/cni/pkg/types/current"
+)
+
+// cniV6Plugin represents the CNI IPV6 IPAM plugin.
+type cniV6Plugin struct {
+	*cni.Plugin
+	am     ipam.AddressManager
+	report *telemetry.CNIReport
+	tb     *telemetry.TelemetryBuffer
+}
+
+// NewPlugin creates a new ipamPlugin object.
+func NewPlugin(name string, config *common.PluginConfig) (*cniV6Plugin, error) {
+	// Setup base plugin.
+	plugin, err := cni.NewPlugin(name, config.Version)
+	if err != nil {
+		return nil, err
+	}
+
+	// Setup ipv6 address manager.
+	am, err := ipam.NewAddressManager()
+	if err != nil {
+		return nil, err
+	}
+
+	// Create IPAM plugin.
+	v6Plg := &cniV6Plugin{
+		Plugin: plugin,
+		am:     am,
+	}
+
+	return v6Plg, nil
+}
+
+// Starts the plugin.
+func (plugin *cniV6Plugin) Start(config *common.PluginConfig) error {
+	// Initialize base plugin.
+	err := plugin.Initialize(config)
+	if err != nil {
+		log.Printf("[cni-v6] Failed to initialize base plugin, err:%v.", err)
+		return err
+	}
+
+	// Log platform information.
+	log.Printf("[cni-v6] Plugin %v version %v.", plugin.Name, plugin.Version)
+	log.Printf("[cni-v6] Running on %v", platform.GetOSInfo())
+
+	//TODO: Set plugin.Options.Environment to IPV6NatSource
+
+	// Initialize address manager.
+	err = plugin.am.Initialize(config, plugin.Options)
+	if err != nil {
+		log.Printf("[cni-v6] Failed to initialize address manager, err:%v.", err)
+		return err
+	}
+
+	log.Printf("[cni-v6] Plugin started.")
+
+	return nil
+}
+
+// Stops the plugin.
+func (plugin *cniV6Plugin) Stop() {
+	plugin.am.Uninitialize()
+	plugin.Uninitialize()
+	log.Printf("[cni-v6] Plugin stopped.")
+}
+
+func (plugin *cniV6Plugin) SetCNIReport(report *telemetry.CNIReport, tb *telemetry.TelemetryBuffer) {
+	plugin.report = report
+	plugin.tb = tb
+}
+
+//
+// CNI implementation
+// https://github.com/containernetworking/cni/blob/master/SPEC.md
+//
+
+// Add handles CNI add commands.
+func (plugin *cniV6Plugin) Add(args *cniSkel.CmdArgs) error {
+	var (
+		result *cniTypesCurr.Result
+		err    error
+	)
+
+	log.Printf("[cni-v6] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.",
+		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData)
+
+	defer func() { log.Printf("[cni-v6] ADD command completed with result:%+v err:%v.", result, err) }()
+
+	return nil
+}
+
+// Delete handles CNI delete commands.
+func (plugin *cniV6Plugin) Delete(args *cniSkel.CmdArgs) error {
+	var err error
+
+	log.Printf("[cni-v6] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.",
+		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData)
+
+	defer func() { log.Printf("[cni-v6] DEL command completed with err:%v.", err) }()
+
+	return nil
+}
+
+// Get handles CNIV6 Get commands.
+func (plugin *cniV6Plugin) Get(args *cniSkel.CmdArgs) error {
+	return nil
+}
+
+// Update handles CNIV6 update command.
+func (plugin *cniV6Plugin) Update(args *cniSkel.CmdArgs) error {
+	return nil
+}
diff --git a/cni/cniv6/plugin/main.go b/cni/cniv6/plugin/main.go
new file mode 100644
index 0000000000..b02ea0596c
--- /dev/null
+++ b/cni/cniv6/plugin/main.go
@@ -0,0 +1,199 @@
+// Copyright 2017 Microsoft. All rights reserved.
+// MIT License
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"time"
+
+	"github.com/Azure/azure-container-networking/aitelemetry"
+	"github.com/Azure/azure-container-networking/cni"
+	"github.com/Azure/azure-container-networking/cni/cniv6"
+	"github.com/Azure/azure-container-networking/cni/network"
+	"github.com/Azure/azure-container-networking/common"
+	acn "github.com/Azure/azure-container-networking/common"
+	"github.com/Azure/azure-container-networking/log"
+	"github.com/Azure/azure-container-networking/platform"
+	"github.com/Azure/azure-container-networking/telemetry"
+)
+
+const (
+	hostNetAgentURL                 = "http://168.63.129.16/machine/plugins?comp=netagent&type=cnireport"
+	ipamQueryURL                    = "http://168.63.129.16/machine/plugins?comp=nmagent&type=getinterfaceinfov1"
+	pluginName                      = "CNIv6"
+	telemetryNumRetries             = 5
+	telemetryWaitTimeInMilliseconds = 200
+	name                            = "azure-vnet"
+	logFile                         = name + "v6"
+)
+
+// Version is populated by make during build.
+var version string
+
+// Command line arguments for CNI plugin.
+var args = acn.ArgumentList{
+	{
+		Name:         acn.OptVersion,
+		Shorthand:    acn.OptVersionAlias,
+		Description:  "Print version information",
+		Type:         "bool",
+		DefaultValue: false,
+	},
+}
+
+// Prints version information.
+func printVersion() {
+	fmt.Printf("Azure CNI Version %v\n", version)
+}
+
+// send error report to hostnetagent if CNI encounters any error.
+func reportPluginError(reportManager *telemetry.ReportManager, tb *telemetry.TelemetryBuffer, err error) {
+	log.Printf("Report plugin error")
+	reflect.ValueOf(reportManager.Report).Elem().FieldByName("ErrorMessage").SetString(err.Error())
+
+	if err := reportManager.SendReport(tb); err != nil {
+		log.Errorf("SendReport failed due to %v", err)
+	}
+}
+
+// Main is the entry point for CNI network plugin.
+func main() {
+
+	startTime := time.Now()
+
+	// Initialize and parse command line arguments.
+	acn.ParseArgs(&args, printVersion)
+	vers := acn.GetArg(acn.OptVersion).(bool)
+
+	if vers {
+		printVersion()
+		os.Exit(0)
+	}
+
+	var (
+		config       common.PluginConfig
+		err          error
+		cnimetric    telemetry.AIMetric
+		logDirectory string // This sets empty string i.e. current location
+	)
+
+	log.SetName(logFile)
+	log.SetLevel(log.LevelInfo)
+	if err = log.SetTargetLogDirectory(log.TargetLogfile, logDirectory); err != nil {
+		fmt.Printf("Failed to setup cni logging: %v\n", err)
+		return
+	}
+
+	defer log.Close()
+
+	config.Version = version
+	reportManager := &telemetry.ReportManager{
+		HostNetAgentURL: hostNetAgentURL,
+		ContentType:     telemetry.ContentType,
+		Report: &telemetry.CNIReport{
+			Context:          "AzureCNIV6",
+			SystemDetails:    telemetry.SystemInfo{},
+			InterfaceDetails: telemetry.InterfaceInfo{},
+			BridgeDetails:    telemetry.BridgeInfo{},
+		},
+	}
+
+	cniReport := reportManager.Report.(*telemetry.CNIReport)
+
+	upTime, err := platform.GetLastRebootTime()
+	if err == nil {
+		cniReport.VMUptime = upTime.Format("2006-01-02 15:04:05")
+	}
+
+	cniReport.GetReport(pluginName, version, ipamQueryURL)
+
+	cniV6Plugin, err := cniv6.NewPlugin(name, &config)
+	if err != nil {
+		fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err)
+		os.Exit(1)
+	}
+
+	// CNI Acquires lock
+	if err = cniV6Plugin.Plugin.InitializeKeyValueStore(&config); err != nil {
+		log.Errorf("Failed to initialize key-value store of network plugin, err:%v.\n", err)
+		tb := telemetry.NewTelemetryBuffer("")
+		if tberr := tb.Connect(); tberr == nil {
+			reportPluginError(reportManager, tb, err)
+			tb.Close()
+		}
+
+		if isSafe, _ := cniV6Plugin.Plugin.IsSafeToRemoveLock(name); isSafe {
+			log.Printf("[CNI] Removing lock file as process holding lock exited")
+			if errUninit := cniV6Plugin.Plugin.UninitializeKeyValueStore(true); errUninit != nil {
+				log.Errorf("Failed to uninitialize key-value store of network plugin, err:%v.\n", errUninit)
+			}
+		}
+
+		return
+	}
+
+	defer func() {
+		if errUninit := cniV6Plugin.Plugin.UninitializeKeyValueStore(false); errUninit != nil {
+			log.Errorf("Failed to uninitialize key-value store of network plugin, err:%v.\n", errUninit)
+		}
+
+		if recover() != nil {
+			return
+		}
+	}()
+
+	// Start telemetry process if not already started. This should be done inside lock, otherwise multiple process
+	// end up creating/killing telemetry process results in undesired state.
+	tb := telemetry.NewTelemetryBuffer("")
+	tb.ConnectToTelemetryService(telemetryNumRetries, telemetryWaitTimeInMilliseconds)
+	defer tb.Close()
+
+	cniV6Plugin.SetCNIReport(cniReport, tb)
+
+	t := time.Now()
+	cniReport.Timestamp = t.Format("2006-01-02 15:04:05")
+
+	if err = cniV6Plugin.Start(&config); err != nil {
+		log.Errorf("Failed to start network plugin, err:%v.\n", err)
+		reportPluginError(reportManager, tb, err)
+		panic("network plugin start fatal error")
+	}
+
+	if err = cniV6Plugin.Execute(cni.PluginApi(cniV6Plugin)); err != nil {
+		log.Errorf("Failed to execute network plugin, err:%v.\n", err)
+	}
+
+	cniV6Plugin.Stop()
+
+	// release cni lock
+	if errUninit := cniV6Plugin.Plugin.UninitializeKeyValueStore(false); errUninit != nil {
+		log.Errorf("Failed to uninitialize key-value store of network plugin, err:%v.\n", errUninit)
+	}
+
+	executionTimeMs := time.Since(startTime).Milliseconds()
+	cnimetric.Metric = aitelemetry.Metric{
+		Name:             telemetry.CNIV6ExecutimeMetricStr,
+		Value:            float64(executionTimeMs),
+		CustomDimensions: make(map[string]string),
+	}
+	network.SetCustomDimensions(&cnimetric, nil, err)
+	telemetry.SendCNIMetric(&cnimetric, tb)
+
+	if err != nil {
+		reportPluginError(reportManager, tb, err)
+		panic("network plugin execute fatal error")
+	}
+
+	// Report CNI successfully finished execution.
+	reflect.ValueOf(reportManager.Report).Elem().FieldByName("CniSucceeded").SetBool(true)
+	reflect.ValueOf(reportManager.Report).Elem().FieldByName("OperationDuration").SetInt(executionTimeMs)
+
+	if err = reportManager.SendReport(tb); err != nil {
+		log.Errorf("SendReport failed due to %v", err)
+	} else {
+		log.Printf("Sending report succeeded")
+	}
+}
diff --git a/cni/network/network.go b/cni/network/network.go
index 0b01dbd7e3..3a26fd278b 100644
--- a/cni/network/network.go
+++ b/cni/network/network.go
@@ -466,11 +466,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 					Gateway: gateway,
 				},
 			},
-			BridgeName:                    nwCfg.Bridge,
-			EnableSnatOnHost:              nwCfg.EnableSnatOnHost,
-			DNS:                           nwDNSInfo,
-			Policies:                      policies,
-			NetNs:                         args.Netns,
+			BridgeName:       nwCfg.Bridge,
+			EnableSnatOnHost: nwCfg.EnableSnatOnHost,
+			DNS:              nwDNSInfo,
+			Policies:         policies,
+			NetNs:            args.Netns,
 			DisableHairpinOnHostInterface: nwCfg.DisableHairpinOnHostInterface,
 		}
 
@@ -1025,3 +1025,27 @@ func determineSnat() (bool, bool, error) {
 
 	return snatConfig.EnableSnatForDns, snatConfig.EnableSnatOnHost, nil
 }
+
+func (plugin *netPlugin) AddV6(args *cniSkel.CmdArgs) error {
+	log.Printf("[cni-net] Processing ADDV6 command with args {Netns:%v Args:%v Path:%v}.",
+		args.Netns, args.Args, args.Path)
+	return nil
+}
+
+func (plugin *netPlugin) DeleteV6(args *cniSkel.CmdArgs) error {
+	log.Printf("[cni-net] Processing DELV6 command with args {Netns:%v Args:%v Path:%v}.",
+		args.Netns, args.Args, args.Path)
+	return nil
+}
+
+func (plugin *netPlugin) UpdateV6(args *cniSkel.CmdArgs) error {
+	log.Printf("[cni-net] Processing UPDATEV6 command with args {Netns:%v Args:%v Path:%v}.",
+		args.Netns, args.Args, args.Path)
+	return nil
+}
+
+func (plugin *netPlugin) GetV6(args *cniSkel.CmdArgs) error {
+	log.Printf("[cni-net] Processing DELETEV6 command with args {Netns:%v Args:%v Path:%v}.",
+		args.Netns, args.Args, args.Path)
+	return nil
+}
diff --git a/cni/plugin.go b/cni/plugin.go
index a1ed3fe338..59ce91a39e 100644
--- a/cni/plugin.go
+++ b/cni/plugin.go
@@ -79,6 +79,7 @@ func (plugin *Plugin) Execute(api PluginApi) (err error) {
 	pluginInfo := cniVers.PluginSupports(supportedVersions...)
 
 	// Parse args and call the appropriate cmd handler.
+
 	cniErr := cniSkel.PluginMainWithError(api.Add, api.Get, api.Delete, pluginInfo, plugin.version)
 	if cniErr != nil {
 		cniErr.Print()
diff --git a/telemetry/constants.go b/telemetry/constants.go
index 9fadc585d6..0d2e2215b2 100644
--- a/telemetry/constants.go
+++ b/telemetry/constants.go
@@ -5,10 +5,11 @@ package telemetry
 const (
 
 	// Metric Names
-	CNIExecutimeMetricStr  = "CNIExecutionTimeMs"
-	CNIAddTimeMetricStr    = "CNIAddTimeMs"
-	CNIDelTimeMetricStr    = "CNIDelTimeMs"
-	CNIUpdateTimeMetricStr = "CNIUpdateTimeMs"
+	CNIExecutimeMetricStr   = "CNIExecutionTimeMs"
+	CNIV6ExecutimeMetricStr = "CNIV6ExecutionTimeMs"
+	CNIAddTimeMetricStr     = "CNIAddTimeMs"
+	CNIDelTimeMetricStr     = "CNIDelTimeMs"
+	CNIUpdateTimeMetricStr  = "CNIUpdateTimeMs"
 
 	// Dimension Names
 	ContextStr        = "Context"

From 0a048303b8cf5fdd8d75acba7f9f9b44cba409e2 Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Wed, 19 Feb 2020 17:49:18 -0800
Subject: [PATCH 02/12] cni ipv6 base framework

---
 cni/cniv6/cniipv6.go | 62 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 59 insertions(+), 3 deletions(-)

diff --git a/cni/cniv6/cniipv6.go b/cni/cniv6/cniipv6.go
index 0a30b1b89d..53fe5f4dfd 100644
--- a/cni/cniv6/cniipv6.go
+++ b/cni/cniv6/cniipv6.go
@@ -94,25 +94,81 @@ func (plugin *cniV6Plugin) SetCNIReport(report *telemetry.CNIReport, tb *telemet
 func (plugin *cniV6Plugin) Add(args *cniSkel.CmdArgs) error {
 	var (
 		result *cniTypesCurr.Result
+		nwCfg  *cni.NetworkConfig
 		err    error
 	)
 
 	log.Printf("[cni-v6] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.",
 		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData)
 
-	defer func() { log.Printf("[cni-v6] ADD command completed with result:%+v err:%v.", result, err) }()
+	// Parse network configuration from stdin.
+	nwCfg, err = cni.ParseNetworkConfig(args.StdinData)
+	if err != nil {
+		err = plugin.Errorf("Failed to parse network configuration: %v.", err)
+		return err
+	}
+
+	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
+
+	defer func() {
+		if result == nil {
+			result = &cniTypesCurr.Result{}
+		}
+
+		res, vererr := result.GetAsVersion(nwCfg.CNIVersion)
+		if vererr != nil {
+			log.Printf("GetAsVersion failed with error %v", vererr)
+			plugin.Error(vererr)
+		}
+
+		if err == nil && res != nil {
+			// Output the result to stdout.
+			res.Print()
+		}
+
+		log.Printf("[cni-v6] ADD command completed with result:%+v err:%v.", result, err)
+	}()
 
 	return nil
 }
 
 // Delete handles CNI delete commands.
 func (plugin *cniV6Plugin) Delete(args *cniSkel.CmdArgs) error {
-	var err error
+	var (
+		result *cniTypesCurr.Result
+		nwCfg  *cni.NetworkConfig
+		err    error
+	)
 
 	log.Printf("[cni-v6] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.",
 		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData)
+	// Parse network configuration from stdin.
+	nwCfg, err = cni.ParseNetworkConfig(args.StdinData)
+	if err != nil {
+		err = plugin.Errorf("Failed to parse network configuration: %v.", err)
+		return err
+	}
+
+	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
+
+	defer func() {
+		if result == nil {
+			result = &cniTypesCurr.Result{}
+		}
+
+		res, vererr := result.GetAsVersion(nwCfg.CNIVersion)
+		if vererr != nil {
+			log.Printf("GetAsVersion failed with error %v", vererr)
+			plugin.Error(vererr)
+		}
+
+		if err == nil && res != nil {
+			// Output the result to stdout.
+			res.Print()
+		}
 
-	defer func() { log.Printf("[cni-v6] DEL command completed with err:%v.", err) }()
+		log.Printf("[cni-v6] DEL command completed with err:%v.", err)
+	}()
 
 	return nil
 }

From 7f91741dc2969986e327734a4a268098861acef7 Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Wed, 19 Feb 2020 17:53:44 -0800
Subject: [PATCH 03/12] removed unwanted changes

---
 cni/network/network.go | 24 ------------------------
 cni/plugin.go          |  1 -
 2 files changed, 25 deletions(-)

diff --git a/cni/network/network.go b/cni/network/network.go
index 2585e3649a..49e06b6797 100644
--- a/cni/network/network.go
+++ b/cni/network/network.go
@@ -1026,27 +1026,3 @@ func determineSnat() (bool, bool, error) {
 
 	return snatConfig.EnableSnatForDns, snatConfig.EnableSnatOnHost, nil
 }
-
-func (plugin *netPlugin) AddV6(args *cniSkel.CmdArgs) error {
-	log.Printf("[cni-net] Processing ADDV6 command with args {Netns:%v Args:%v Path:%v}.",
-		args.Netns, args.Args, args.Path)
-	return nil
-}
-
-func (plugin *netPlugin) DeleteV6(args *cniSkel.CmdArgs) error {
-	log.Printf("[cni-net] Processing DELV6 command with args {Netns:%v Args:%v Path:%v}.",
-		args.Netns, args.Args, args.Path)
-	return nil
-}
-
-func (plugin *netPlugin) UpdateV6(args *cniSkel.CmdArgs) error {
-	log.Printf("[cni-net] Processing UPDATEV6 command with args {Netns:%v Args:%v Path:%v}.",
-		args.Netns, args.Args, args.Path)
-	return nil
-}
-
-func (plugin *netPlugin) GetV6(args *cniSkel.CmdArgs) error {
-	log.Printf("[cni-net] Processing DELETEV6 command with args {Netns:%v Args:%v Path:%v}.",
-		args.Netns, args.Args, args.Path)
-	return nil
-}
diff --git a/cni/plugin.go b/cni/plugin.go
index 59ce91a39e..a1ed3fe338 100644
--- a/cni/plugin.go
+++ b/cni/plugin.go
@@ -79,7 +79,6 @@ func (plugin *Plugin) Execute(api PluginApi) (err error) {
 	pluginInfo := cniVers.PluginSupports(supportedVersions...)
 
 	// Parse args and call the appropriate cmd handler.
-
 	cniErr := cniSkel.PluginMainWithError(api.Add, api.Get, api.Delete, pluginInfo, plugin.version)
 	if cniErr != nil {
 		cniErr.Print()

From 083577f2052c43c0b01a4afbb02c3c01f6516878 Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Fri, 21 Feb 2020 14:42:07 -0800
Subject: [PATCH 04/12] added ipv6 change

---
 cni/cniv6/cniipv6.go | 117 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 116 insertions(+), 1 deletion(-)

diff --git a/cni/cniv6/cniipv6.go b/cni/cniv6/cniipv6.go
index 53fe5f4dfd..b5d9a5a8ab 100644
--- a/cni/cniv6/cniipv6.go
+++ b/cni/cniv6/cniipv6.go
@@ -4,13 +4,15 @@
 package cniv6
 
 import (
+	"os"
+
 	"github.com/Azure/azure-container-networking/cni"
 	"github.com/Azure/azure-container-networking/common"
 	"github.com/Azure/azure-container-networking/ipam"
 	"github.com/Azure/azure-container-networking/log"
+	"github.com/Azure/azure-container-networking/network"
 	"github.com/Azure/azure-container-networking/platform"
 	"github.com/Azure/azure-container-networking/telemetry"
-
 	cniSkel "github.com/containernetworking/cni/pkg/skel"
 	cniTypesCurr "github.com/containernetworking/cni/pkg/types/current"
 )
@@ -95,6 +97,7 @@ func (plugin *cniV6Plugin) Add(args *cniSkel.CmdArgs) error {
 	var (
 		result *cniTypesCurr.Result
 		nwCfg  *cni.NetworkConfig
+		ns     *network.Namespace
 		err    error
 	)
 
@@ -115,6 +118,22 @@ func (plugin *cniV6Plugin) Add(args *cniSkel.CmdArgs) error {
 			result = &cniTypesCurr.Result{}
 		}
 
+		// _, ipv6net, _ := net.ParseCIDR("fc00::2/64")
+		// ipv6net := net.IPNet{IP: ipaddr, Mask: net.CIDRMask(24, 32)}
+		// ip := &cniTypesCurr.IPConfig{
+		// 	Version: "6",
+		// 	Address: ipv6net,
+		// 	Gateway: net.ParseIP("fc00::1"),
+		// }
+
+		// result.IPs = append(result.IPs, ip)
+
+		iface := &cniTypesCurr.Interface{
+			Name: args.IfName,
+		}
+
+		result.Interfaces = append(result.Interfaces, iface)
+
 		res, vererr := result.GetAsVersion(nwCfg.CNIVersion)
 		if vererr != nil {
 			log.Printf("GetAsVersion failed with error %v", vererr)
@@ -129,6 +148,30 @@ func (plugin *cniV6Plugin) Add(args *cniSkel.CmdArgs) error {
 		log.Printf("[cni-v6] ADD command completed with result:%+v err:%v.", result, err)
 	}()
 
+	log.Printf("[cni-v6] Entering container namespace")
+	ns, err = network.OpenNamespace(args.Netns)
+	if err != nil {
+		return err
+	}
+	defer ns.Close()
+
+	// Enter the container network namespace.
+	log.Printf("[net] Entering netns %v.", args.Netns)
+	if err = ns.Enter(); err != nil {
+		return err
+	}
+
+	// Return to host network namespace.
+	defer func() {
+		log.Printf("[net] Exiting netns %v.", args.Netns)
+		if err := ns.Exit(); err != nil {
+			log.Printf("[net] Failed to exit netns, err:%v.", err)
+		}
+	}()
+
+	platform.ExecuteCommand("sysctl -w net.ipv6.conf.all.disable_ipv6=0")
+	platform.ExecuteCommand("ip -6 addr add fc00::2/64 dev eth0")
+
 	return nil
 }
 
@@ -182,3 +225,75 @@ func (plugin *cniV6Plugin) Get(args *cniSkel.CmdArgs) error {
 func (plugin *cniV6Plugin) Update(args *cniSkel.CmdArgs) error {
 	return nil
 }
+
+// Namespace represents a network namespace.
+type Namespace struct {
+	file   *os.File
+	prevNs *Namespace
+}
+
+// // OpenNamespace creates a new namespace object for the given netns path.
+// func OpenNamespace(nsPath string) (*Namespace, error) {
+// 	fd, err := os.Open(nsPath)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+
+// 	return &Namespace{file: fd}, nil
+// }
+
+// func (ns *Namespace) Close() error {
+// 	if ns.file == nil {
+// 		return nil
+// 	}
+
+// 	err := ns.file.Close()
+// 	if err != nil {
+// 		return fmt.Errorf("Failed to close namespace %v, err:%v", ns.file.Name(), err)
+// 	}
+
+// 	ns.file = nil
+
+// 	return nil
+// }
+
+// // Enter puts the caller thread inside the namespace.
+// func (ns *Namespace) Enter() error {
+// 	var err error
+
+// 	ns.prevNs, err = GetCurrentThreadNamespace()
+// 	if err != nil {
+// 		return err
+// 	}
+
+// 	runtime.LockOSThread()
+
+// 	err = ns.set()
+// 	if err != nil {
+// 		runtime.UnlockOSThread()
+// 		return err
+// 	}
+
+// 	// Recycle the netlink socket for the new network namespace.
+// 	netlink.ResetSocket()
+
+// 	return nil
+// }
+
+// // Exit puts the caller thread to its previous namespace.
+// func (ns *Namespace) Exit() error {
+// 	err := ns.prevNs.set()
+// 	if err != nil {
+// 		return err
+// 	}
+
+// 	ns.prevNs.Close()
+// 	ns.prevNs = nil
+
+// 	runtime.UnlockOSThread()
+
+// 	// Recycle the netlink socket for the new network namespace.
+// 	netlink.ResetSocket()
+
+// 	return nil
+// }

From 7b192abe90a434b0de89bef9aaac39a929bad2fa Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Mon, 30 Mar 2020 17:17:23 -0700
Subject: [PATCH 05/12] added cni ipv6 changes

---
 Makefile                               |  12 +-
 cni/cniv6/cniipv6.go                   | 299 -------------------------
 cni/cniv6/plugin/main.go               | 199 ----------------
 cni/ipam/pluginv6/main.go              |  80 +++++++
 cni/netconfig.go                       |   2 +
 cni/network/network.go                 | 236 ++++++++++++++-----
 cni/network/network_linux.go           |   2 +-
 cni/network/network_windows.go         |   2 +-
 ebtables/ebtables.go                   |  30 ++-
 iptables/iptables.go                   |  47 ++--
 network/bridge_endpointclient_linux.go |  37 +--
 network/bridge_networkclient_linux.go  |  34 ++-
 network/endpoint.go                    |   1 +
 network/epcommon/endpoint_common.go    |  36 ++-
 network/manager.go                     |  12 +-
 network/network.go                     |   7 +
 network/network_linux.go               |  53 ++++-
 network/ovssnat/ovssnat.go             |  32 +--
 18 files changed, 488 insertions(+), 633 deletions(-)
 delete mode 100644 cni/cniv6/cniipv6.go
 delete mode 100644 cni/cniv6/plugin/main.go
 create mode 100644 cni/ipam/pluginv6/main.go

diff --git a/Makefile b/Makefile
index 183977dcdc..92355c69e2 100644
--- a/Makefile
+++ b/Makefile
@@ -68,6 +68,7 @@ GOARCH ?= amd64
 CNM_DIR = cnm/plugin
 CNI_NET_DIR = cni/network/plugin
 CNI_IPAM_DIR = cni/ipam/plugin
+CNI_IPAMV6_DIR = cni/ipam/pluginv6
 CNI_TELEMETRY_DIR = cni/telemetry/service
 TELEMETRY_CONF_DIR = telemetry
 CNS_DIR = cns/service
@@ -134,7 +135,8 @@ ENSURE_OUTPUT_DIR_EXISTS := $(shell mkdir -p $(OUTPUT_DIR))
 azure-cnm-plugin: $(CNM_BUILD_DIR)/azure-vnet-plugin$(EXE_EXT) cnm-archive
 azure-vnet: $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT)
 azure-vnet-ipam: $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT)
-azure-cni-plugin: azure-vnet azure-vnet-ipam azure-vnet-telemetry cni-archive
+azure-vnet-ipamv6: $(CNI_BUILD_DIR)/azure-vnet-ipamv6$(EXE_EXT)
+azure-cni-plugin: azure-vnet azure-vnet-ipam azure-vnet-ipamv6 azure-vnet-telemetry cni-archive
 azure-cns: $(CNS_BUILD_DIR)/azure-cns$(EXE_EXT) cns-archive
 azure-vnet-telemetry: $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT)
 
@@ -173,6 +175,10 @@ $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT): $(CNIFILES)
 $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT): $(CNIFILES)
 	go build -v -o $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNI_IPAM_DIR)/*.go
 
+# Build the Azure CNI IPAMV6 plugin.
+$(CNI_BUILD_DIR)/azure-vnet-ipamv6$(EXE_EXT): $(CNIFILES)
+	go build -v -o $(CNI_BUILD_DIR)/azure-vnet-ipamv6$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -s -w" $(CNI_IPAMV6_DIR)/*.go
+
 # Build the Azure CNI telemetry plugin.
 $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT): $(CNIFILES)
 	go build -v -o $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) -ldflags "-X main.version=$(VERSION) -X $(ACN_PACKAGE_PATH)/telemetry.aiMetadata=$(CNI_AI_ID) -s -w" $(CNI_TELEMETRY_DIR)/*.go
@@ -299,8 +305,8 @@ publish-azure-cns-image:
 cni-archive:
 	cp cni/azure-$(GOOS).conflist $(CNI_BUILD_DIR)/10-azure.conflist
 	cp telemetry/azure-vnet-telemetry.config $(CNI_BUILD_DIR)/azure-vnet-telemetry.config
-	chmod 0755 $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT)
-	cd $(CNI_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config
+	chmod 0755 $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipamv6$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT)
+	cd $(CNI_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-ipamv6$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config
 	chown $(BUILD_USER):$(BUILD_USER) $(CNI_BUILD_DIR)/$(CNI_ARCHIVE_NAME)
 	mkdir -p $(CNI_MULTITENANCY_BUILD_DIR)
 	cp cni/azure-$(GOOS)-multitenancy.conflist $(CNI_MULTITENANCY_BUILD_DIR)/10-azure.conflist
diff --git a/cni/cniv6/cniipv6.go b/cni/cniv6/cniipv6.go
deleted file mode 100644
index b5d9a5a8ab..0000000000
--- a/cni/cniv6/cniipv6.go
+++ /dev/null
@@ -1,299 +0,0 @@
-// Copyright 2017 Microsoft. All rights reserved.
-// MIT License
-
-package cniv6
-
-import (
-	"os"
-
-	"github.com/Azure/azure-container-networking/cni"
-	"github.com/Azure/azure-container-networking/common"
-	"github.com/Azure/azure-container-networking/ipam"
-	"github.com/Azure/azure-container-networking/log"
-	"github.com/Azure/azure-container-networking/network"
-	"github.com/Azure/azure-container-networking/platform"
-	"github.com/Azure/azure-container-networking/telemetry"
-	cniSkel "github.com/containernetworking/cni/pkg/skel"
-	cniTypesCurr "github.com/containernetworking/cni/pkg/types/current"
-)
-
-// cniV6Plugin represents the CNI IPV6 IPAM plugin.
-type cniV6Plugin struct {
-	*cni.Plugin
-	am     ipam.AddressManager
-	report *telemetry.CNIReport
-	tb     *telemetry.TelemetryBuffer
-}
-
-// NewPlugin creates a new ipamPlugin object.
-func NewPlugin(name string, config *common.PluginConfig) (*cniV6Plugin, error) {
-	// Setup base plugin.
-	plugin, err := cni.NewPlugin(name, config.Version)
-	if err != nil {
-		return nil, err
-	}
-
-	// Setup ipv6 address manager.
-	am, err := ipam.NewAddressManager()
-	if err != nil {
-		return nil, err
-	}
-
-	// Create IPAM plugin.
-	v6Plg := &cniV6Plugin{
-		Plugin: plugin,
-		am:     am,
-	}
-
-	return v6Plg, nil
-}
-
-// Starts the plugin.
-func (plugin *cniV6Plugin) Start(config *common.PluginConfig) error {
-	// Initialize base plugin.
-	err := plugin.Initialize(config)
-	if err != nil {
-		log.Printf("[cni-v6] Failed to initialize base plugin, err:%v.", err)
-		return err
-	}
-
-	// Log platform information.
-	log.Printf("[cni-v6] Plugin %v version %v.", plugin.Name, plugin.Version)
-	log.Printf("[cni-v6] Running on %v", platform.GetOSInfo())
-
-	//TODO: Set plugin.Options.Environment to IPV6NatSource
-
-	// Initialize address manager.
-	err = plugin.am.Initialize(config, plugin.Options)
-	if err != nil {
-		log.Printf("[cni-v6] Failed to initialize address manager, err:%v.", err)
-		return err
-	}
-
-	log.Printf("[cni-v6] Plugin started.")
-
-	return nil
-}
-
-// Stops the plugin.
-func (plugin *cniV6Plugin) Stop() {
-	plugin.am.Uninitialize()
-	plugin.Uninitialize()
-	log.Printf("[cni-v6] Plugin stopped.")
-}
-
-func (plugin *cniV6Plugin) SetCNIReport(report *telemetry.CNIReport, tb *telemetry.TelemetryBuffer) {
-	plugin.report = report
-	plugin.tb = tb
-}
-
-//
-// CNI implementation
-// https://github.com/containernetworking/cni/blob/master/SPEC.md
-//
-
-// Add handles CNI add commands.
-func (plugin *cniV6Plugin) Add(args *cniSkel.CmdArgs) error {
-	var (
-		result *cniTypesCurr.Result
-		nwCfg  *cni.NetworkConfig
-		ns     *network.Namespace
-		err    error
-	)
-
-	log.Printf("[cni-v6] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.",
-		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData)
-
-	// Parse network configuration from stdin.
-	nwCfg, err = cni.ParseNetworkConfig(args.StdinData)
-	if err != nil {
-		err = plugin.Errorf("Failed to parse network configuration: %v.", err)
-		return err
-	}
-
-	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
-
-	defer func() {
-		if result == nil {
-			result = &cniTypesCurr.Result{}
-		}
-
-		// _, ipv6net, _ := net.ParseCIDR("fc00::2/64")
-		// ipv6net := net.IPNet{IP: ipaddr, Mask: net.CIDRMask(24, 32)}
-		// ip := &cniTypesCurr.IPConfig{
-		// 	Version: "6",
-		// 	Address: ipv6net,
-		// 	Gateway: net.ParseIP("fc00::1"),
-		// }
-
-		// result.IPs = append(result.IPs, ip)
-
-		iface := &cniTypesCurr.Interface{
-			Name: args.IfName,
-		}
-
-		result.Interfaces = append(result.Interfaces, iface)
-
-		res, vererr := result.GetAsVersion(nwCfg.CNIVersion)
-		if vererr != nil {
-			log.Printf("GetAsVersion failed with error %v", vererr)
-			plugin.Error(vererr)
-		}
-
-		if err == nil && res != nil {
-			// Output the result to stdout.
-			res.Print()
-		}
-
-		log.Printf("[cni-v6] ADD command completed with result:%+v err:%v.", result, err)
-	}()
-
-	log.Printf("[cni-v6] Entering container namespace")
-	ns, err = network.OpenNamespace(args.Netns)
-	if err != nil {
-		return err
-	}
-	defer ns.Close()
-
-	// Enter the container network namespace.
-	log.Printf("[net] Entering netns %v.", args.Netns)
-	if err = ns.Enter(); err != nil {
-		return err
-	}
-
-	// Return to host network namespace.
-	defer func() {
-		log.Printf("[net] Exiting netns %v.", args.Netns)
-		if err := ns.Exit(); err != nil {
-			log.Printf("[net] Failed to exit netns, err:%v.", err)
-		}
-	}()
-
-	platform.ExecuteCommand("sysctl -w net.ipv6.conf.all.disable_ipv6=0")
-	platform.ExecuteCommand("ip -6 addr add fc00::2/64 dev eth0")
-
-	return nil
-}
-
-// Delete handles CNI delete commands.
-func (plugin *cniV6Plugin) Delete(args *cniSkel.CmdArgs) error {
-	var (
-		result *cniTypesCurr.Result
-		nwCfg  *cni.NetworkConfig
-		err    error
-	)
-
-	log.Printf("[cni-v6] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v StdinData:%s}.",
-		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData)
-	// Parse network configuration from stdin.
-	nwCfg, err = cni.ParseNetworkConfig(args.StdinData)
-	if err != nil {
-		err = plugin.Errorf("Failed to parse network configuration: %v.", err)
-		return err
-	}
-
-	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
-
-	defer func() {
-		if result == nil {
-			result = &cniTypesCurr.Result{}
-		}
-
-		res, vererr := result.GetAsVersion(nwCfg.CNIVersion)
-		if vererr != nil {
-			log.Printf("GetAsVersion failed with error %v", vererr)
-			plugin.Error(vererr)
-		}
-
-		if err == nil && res != nil {
-			// Output the result to stdout.
-			res.Print()
-		}
-
-		log.Printf("[cni-v6] DEL command completed with err:%v.", err)
-	}()
-
-	return nil
-}
-
-// Get handles CNIV6 Get commands.
-func (plugin *cniV6Plugin) Get(args *cniSkel.CmdArgs) error {
-	return nil
-}
-
-// Update handles CNIV6 update command.
-func (plugin *cniV6Plugin) Update(args *cniSkel.CmdArgs) error {
-	return nil
-}
-
-// Namespace represents a network namespace.
-type Namespace struct {
-	file   *os.File
-	prevNs *Namespace
-}
-
-// // OpenNamespace creates a new namespace object for the given netns path.
-// func OpenNamespace(nsPath string) (*Namespace, error) {
-// 	fd, err := os.Open(nsPath)
-// 	if err != nil {
-// 		return nil, err
-// 	}
-
-// 	return &Namespace{file: fd}, nil
-// }
-
-// func (ns *Namespace) Close() error {
-// 	if ns.file == nil {
-// 		return nil
-// 	}
-
-// 	err := ns.file.Close()
-// 	if err != nil {
-// 		return fmt.Errorf("Failed to close namespace %v, err:%v", ns.file.Name(), err)
-// 	}
-
-// 	ns.file = nil
-
-// 	return nil
-// }
-
-// // Enter puts the caller thread inside the namespace.
-// func (ns *Namespace) Enter() error {
-// 	var err error
-
-// 	ns.prevNs, err = GetCurrentThreadNamespace()
-// 	if err != nil {
-// 		return err
-// 	}
-
-// 	runtime.LockOSThread()
-
-// 	err = ns.set()
-// 	if err != nil {
-// 		runtime.UnlockOSThread()
-// 		return err
-// 	}
-
-// 	// Recycle the netlink socket for the new network namespace.
-// 	netlink.ResetSocket()
-
-// 	return nil
-// }
-
-// // Exit puts the caller thread to its previous namespace.
-// func (ns *Namespace) Exit() error {
-// 	err := ns.prevNs.set()
-// 	if err != nil {
-// 		return err
-// 	}
-
-// 	ns.prevNs.Close()
-// 	ns.prevNs = nil
-
-// 	runtime.UnlockOSThread()
-
-// 	// Recycle the netlink socket for the new network namespace.
-// 	netlink.ResetSocket()
-
-// 	return nil
-// }
diff --git a/cni/cniv6/plugin/main.go b/cni/cniv6/plugin/main.go
deleted file mode 100644
index b02ea0596c..0000000000
--- a/cni/cniv6/plugin/main.go
+++ /dev/null
@@ -1,199 +0,0 @@
-// Copyright 2017 Microsoft. All rights reserved.
-// MIT License
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"reflect"
-	"time"
-
-	"github.com/Azure/azure-container-networking/aitelemetry"
-	"github.com/Azure/azure-container-networking/cni"
-	"github.com/Azure/azure-container-networking/cni/cniv6"
-	"github.com/Azure/azure-container-networking/cni/network"
-	"github.com/Azure/azure-container-networking/common"
-	acn "github.com/Azure/azure-container-networking/common"
-	"github.com/Azure/azure-container-networking/log"
-	"github.com/Azure/azure-container-networking/platform"
-	"github.com/Azure/azure-container-networking/telemetry"
-)
-
-const (
-	hostNetAgentURL                 = "http://168.63.129.16/machine/plugins?comp=netagent&type=cnireport"
-	ipamQueryURL                    = "http://168.63.129.16/machine/plugins?comp=nmagent&type=getinterfaceinfov1"
-	pluginName                      = "CNIv6"
-	telemetryNumRetries             = 5
-	telemetryWaitTimeInMilliseconds = 200
-	name                            = "azure-vnet"
-	logFile                         = name + "v6"
-)
-
-// Version is populated by make during build.
-var version string
-
-// Command line arguments for CNI plugin.
-var args = acn.ArgumentList{
-	{
-		Name:         acn.OptVersion,
-		Shorthand:    acn.OptVersionAlias,
-		Description:  "Print version information",
-		Type:         "bool",
-		DefaultValue: false,
-	},
-}
-
-// Prints version information.
-func printVersion() {
-	fmt.Printf("Azure CNI Version %v\n", version)
-}
-
-// send error report to hostnetagent if CNI encounters any error.
-func reportPluginError(reportManager *telemetry.ReportManager, tb *telemetry.TelemetryBuffer, err error) {
-	log.Printf("Report plugin error")
-	reflect.ValueOf(reportManager.Report).Elem().FieldByName("ErrorMessage").SetString(err.Error())
-
-	if err := reportManager.SendReport(tb); err != nil {
-		log.Errorf("SendReport failed due to %v", err)
-	}
-}
-
-// Main is the entry point for CNI network plugin.
-func main() {
-
-	startTime := time.Now()
-
-	// Initialize and parse command line arguments.
-	acn.ParseArgs(&args, printVersion)
-	vers := acn.GetArg(acn.OptVersion).(bool)
-
-	if vers {
-		printVersion()
-		os.Exit(0)
-	}
-
-	var (
-		config       common.PluginConfig
-		err          error
-		cnimetric    telemetry.AIMetric
-		logDirectory string // This sets empty string i.e. current location
-	)
-
-	log.SetName(logFile)
-	log.SetLevel(log.LevelInfo)
-	if err = log.SetTargetLogDirectory(log.TargetLogfile, logDirectory); err != nil {
-		fmt.Printf("Failed to setup cni logging: %v\n", err)
-		return
-	}
-
-	defer log.Close()
-
-	config.Version = version
-	reportManager := &telemetry.ReportManager{
-		HostNetAgentURL: hostNetAgentURL,
-		ContentType:     telemetry.ContentType,
-		Report: &telemetry.CNIReport{
-			Context:          "AzureCNIV6",
-			SystemDetails:    telemetry.SystemInfo{},
-			InterfaceDetails: telemetry.InterfaceInfo{},
-			BridgeDetails:    telemetry.BridgeInfo{},
-		},
-	}
-
-	cniReport := reportManager.Report.(*telemetry.CNIReport)
-
-	upTime, err := platform.GetLastRebootTime()
-	if err == nil {
-		cniReport.VMUptime = upTime.Format("2006-01-02 15:04:05")
-	}
-
-	cniReport.GetReport(pluginName, version, ipamQueryURL)
-
-	cniV6Plugin, err := cniv6.NewPlugin(name, &config)
-	if err != nil {
-		fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err)
-		os.Exit(1)
-	}
-
-	// CNI Acquires lock
-	if err = cniV6Plugin.Plugin.InitializeKeyValueStore(&config); err != nil {
-		log.Errorf("Failed to initialize key-value store of network plugin, err:%v.\n", err)
-		tb := telemetry.NewTelemetryBuffer("")
-		if tberr := tb.Connect(); tberr == nil {
-			reportPluginError(reportManager, tb, err)
-			tb.Close()
-		}
-
-		if isSafe, _ := cniV6Plugin.Plugin.IsSafeToRemoveLock(name); isSafe {
-			log.Printf("[CNI] Removing lock file as process holding lock exited")
-			if errUninit := cniV6Plugin.Plugin.UninitializeKeyValueStore(true); errUninit != nil {
-				log.Errorf("Failed to uninitialize key-value store of network plugin, err:%v.\n", errUninit)
-			}
-		}
-
-		return
-	}
-
-	defer func() {
-		if errUninit := cniV6Plugin.Plugin.UninitializeKeyValueStore(false); errUninit != nil {
-			log.Errorf("Failed to uninitialize key-value store of network plugin, err:%v.\n", errUninit)
-		}
-
-		if recover() != nil {
-			return
-		}
-	}()
-
-	// Start telemetry process if not already started. This should be done inside lock, otherwise multiple process
-	// end up creating/killing telemetry process results in undesired state.
-	tb := telemetry.NewTelemetryBuffer("")
-	tb.ConnectToTelemetryService(telemetryNumRetries, telemetryWaitTimeInMilliseconds)
-	defer tb.Close()
-
-	cniV6Plugin.SetCNIReport(cniReport, tb)
-
-	t := time.Now()
-	cniReport.Timestamp = t.Format("2006-01-02 15:04:05")
-
-	if err = cniV6Plugin.Start(&config); err != nil {
-		log.Errorf("Failed to start network plugin, err:%v.\n", err)
-		reportPluginError(reportManager, tb, err)
-		panic("network plugin start fatal error")
-	}
-
-	if err = cniV6Plugin.Execute(cni.PluginApi(cniV6Plugin)); err != nil {
-		log.Errorf("Failed to execute network plugin, err:%v.\n", err)
-	}
-
-	cniV6Plugin.Stop()
-
-	// release cni lock
-	if errUninit := cniV6Plugin.Plugin.UninitializeKeyValueStore(false); errUninit != nil {
-		log.Errorf("Failed to uninitialize key-value store of network plugin, err:%v.\n", errUninit)
-	}
-
-	executionTimeMs := time.Since(startTime).Milliseconds()
-	cnimetric.Metric = aitelemetry.Metric{
-		Name:             telemetry.CNIV6ExecutimeMetricStr,
-		Value:            float64(executionTimeMs),
-		CustomDimensions: make(map[string]string),
-	}
-	network.SetCustomDimensions(&cnimetric, nil, err)
-	telemetry.SendCNIMetric(&cnimetric, tb)
-
-	if err != nil {
-		reportPluginError(reportManager, tb, err)
-		panic("network plugin execute fatal error")
-	}
-
-	// Report CNI successfully finished execution.
-	reflect.ValueOf(reportManager.Report).Elem().FieldByName("CniSucceeded").SetBool(true)
-	reflect.ValueOf(reportManager.Report).Elem().FieldByName("OperationDuration").SetInt(executionTimeMs)
-
-	if err = reportManager.SendReport(tb); err != nil {
-		log.Errorf("SendReport failed due to %v", err)
-	} else {
-		log.Printf("Sending report succeeded")
-	}
-}
diff --git a/cni/ipam/pluginv6/main.go b/cni/ipam/pluginv6/main.go
new file mode 100644
index 0000000000..cb93640c48
--- /dev/null
+++ b/cni/ipam/pluginv6/main.go
@@ -0,0 +1,80 @@
+// Copyright 2017 Microsoft. All rights reserved.
+// MIT License
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/Azure/azure-container-networking/cni"
+	"github.com/Azure/azure-container-networking/cni/ipam"
+	"github.com/Azure/azure-container-networking/common"
+	"github.com/Azure/azure-container-networking/log"
+)
+
+const (
+	name = "azure-vnet-ipamv6"
+)
+
+// Version is populated by make during build.
+var version string
+
+// Main is the entry point for CNI IPAM plugin.
+func main() {
+	var config common.PluginConfig
+	config.Version = version
+	logDirectory := "" // Sets the current location as log directory
+
+	log.SetName(name)
+	log.SetLevel(log.LevelInfo)
+	if err := log.SetTargetLogDirectory(log.TargetLogfile, logDirectory); err != nil {
+		fmt.Printf("Failed to setup cni logging: %v\n", err)
+		return
+	}
+
+	defer log.Close()
+
+	ipamPlugin, err := ipam.NewPlugin(name, &config)
+	if err != nil {
+		fmt.Printf("Failed to create IPAM plugin, err:%v.\n", err)
+		os.Exit(1)
+	}
+
+	if err := ipamPlugin.Plugin.InitializeKeyValueStore(&config); err != nil {
+		fmt.Printf("Failed to initialize key-value store of ipam plugin, err:%v.\n", err)
+
+		if isSafe, _ := ipamPlugin.Plugin.IsSafeToRemoveLock(ipamPlugin.Plugin.Name); isSafe {
+			log.Printf("[IPAM] Removing lock file as process holding lock exited")
+			if errUninit := ipamPlugin.Plugin.UninitializeKeyValueStore(true); errUninit != nil {
+				log.Errorf("Failed to uninitialize key-value store of network plugin, err:%v.\n", errUninit)
+			}
+		}
+
+		os.Exit(1)
+	}
+
+	defer func() {
+		if errUninit := ipamPlugin.Plugin.UninitializeKeyValueStore(false); errUninit != nil {
+			fmt.Printf("Failed to uninitialize key-value store of ipam plugin, err:%v.\n", err)
+		}
+
+		if recover() != nil {
+			os.Exit(1)
+		}
+	}()
+
+	err = ipamPlugin.Start(&config)
+	if err != nil {
+		fmt.Printf("Failed to start IPAM plugin, err:%v.\n", err)
+		panic("ipam plugin fatal error")
+	}
+
+	err = ipamPlugin.Execute(cni.PluginApi(ipamPlugin))
+
+	ipamPlugin.Stop()
+
+	if err != nil {
+		panic("ipam plugin fatal error")
+	}
+}
diff --git a/cni/netconfig.go b/cni/netconfig.go
index 93f272fa34..74da2469a9 100644
--- a/cni/netconfig.go
+++ b/cni/netconfig.go
@@ -51,6 +51,8 @@ type NetworkConfig struct {
 	LogLevel                      string   `json:"logLevel,omitempty"`
 	LogTarget                     string   `json:"logTarget,omitempty"`
 	InfraVnetAddressSpace         string   `json:"infraVnetAddressSpace,omitempty"`
+	IPV6Mode                      string   `json:"ipv6Mode,omitempty"`
+	ServiceCidrs                  string   `json:"serviceCidrs,omitempty"`
 	PodNamespaceForDualNetwork    []string `json:"podNamespaceForDualNetwork,omitempty"`
 	MultiTenancy                  bool     `json:"multiTenancy,omitempty"`
 	EnableSnatOnHost              bool     `json:"enableSnatOnHost,omitempty"`
diff --git a/cni/network/network.go b/cni/network/network.go
index 49e06b6797..61e2b8fcfa 100644
--- a/cni/network/network.go
+++ b/cni/network/network.go
@@ -33,6 +33,7 @@ const (
 	opModeTransparent   = "transparent"
 	// Supported IP version. Currently support only IPv4
 	ipVersion = "4"
+	ipamV6    = "azure-vnet-ipamv6"
 )
 
 // CNI Operation Types
@@ -226,6 +227,83 @@ func (plugin *netPlugin) setCNIReportDetails(nwCfg *cni.NetworkConfig, opType st
 	plugin.report.InterfaceDetails.SecondaryCAUsedCount = plugin.nm.GetNumberOfEndpoints("", nwCfg.Name)
 }
 
+func addNatIPV6SubnetInfo(nwCfg *cni.NetworkConfig,
+	resultV6 *cniTypesCurr.Result,
+	nwInfo *network.NetworkInfo) {
+	if nwCfg.IPV6Mode == network.IPV6Nat {
+		ipv6Subnet := resultV6.IPs[0].Address
+		ipv6Subnet.IP = ipv6Subnet.IP.Mask(ipv6Subnet.Mask)
+		ipv6SubnetInfo := network.SubnetInfo{
+			Family:  platform.AfINET6,
+			Prefix:  ipv6Subnet,
+			Gateway: resultV6.IPs[0].Gateway,
+		}
+		log.Printf("[net] ipv6 subnet info:%+v", ipv6SubnetInfo)
+		nwInfo.Subnets = append(nwInfo.Subnets, ipv6SubnetInfo)
+	}
+}
+
+func (plugin *netPlugin) invokeIpamDel(
+	result *cniTypesCurr.Result,
+	ipamType string,
+	nwCfg *cni.NetworkConfig,
+	isDeletePoolOnError bool) {
+
+	if result != nil {
+		nwCfg.Ipam.Subnet = result.IPs[0].Address.String()
+		nwCfg.Ipam.Address = result.IPs[0].Address.IP.String()
+		plugin.DelegateDel(ipamType, nwCfg)
+
+		// Release pool
+		if isDeletePoolOnError {
+			nwCfg.Ipam.Address = ""
+			plugin.DelegateDel(ipamType, nwCfg)
+		}
+	}
+}
+
+func (plugin *netPlugin) invokeIpamAdd(
+	nwCfg *cni.NetworkConfig,
+	nwInfo network.NetworkInfo,
+	isDeletePoolOnError bool) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) {
+
+	var (
+		result   *cniTypesCurr.Result
+		resultV6 *cniTypesCurr.Result
+		err      error
+	)
+
+	if len(nwInfo.Subnets) > 0 {
+		nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String()
+	}
+
+	// Call into IPAM plugin to allocate an address pool for the network.
+	result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg)
+	if err != nil {
+		err = plugin.Errorf("Failed to allocate pool: %v", err)
+		return result, resultV6, err
+	}
+
+	defer func() {
+		if err != nil {
+			plugin.invokeIpamDel(result, nwCfg.Ipam.Type, nwCfg, isDeletePoolOnError)
+		}
+	}()
+
+	if nwCfg.IPV6Mode != "" {
+		if len(nwInfo.Subnets) > 1 {
+			nwCfg.Ipam.Subnet = nwInfo.Subnets[1].Prefix.String()
+		}
+
+		resultV6, err = plugin.DelegateAdd(ipamV6, nwCfg)
+		if err != nil {
+			err = plugin.Errorf("Failed to allocate v6 pool: %v", err)
+		}
+	}
+
+	return result, resultV6, err
+}
+
 //
 // CNI implementation
 // https://github.com/containernetworking/cni/blob/master/SPEC.md
@@ -235,6 +313,7 @@ func (plugin *netPlugin) setCNIReportDetails(nwCfg *cni.NetworkConfig, opType st
 func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 	var (
 		result           *cniTypesCurr.Result
+		resultV6         *cniTypesCurr.Result
 		azIpamResult     *cniTypesCurr.Result
 		err              error
 		vethName         string
@@ -291,8 +370,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		iface = &cniTypesCurr.Interface{
 			Name: args.IfName,
 		}
-
 		result.Interfaces = append(result.Interfaces, iface)
+
 		addSnatInterface(nwCfg, result)
 		// Convert result to the requested CNI version.
 		res, vererr := result.GetAsVersion(nwCfg.CNIVersion)
@@ -398,34 +477,34 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		log.Printf("[cni-net] Creating network %v.", networkId)
 
 		if !nwCfg.MultiTenancy {
-			// Call into IPAM plugin to allocate an address pool for the network.
-			result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg)
+			result, resultV6, err = plugin.invokeIpamAdd(nwCfg, nwInfo, true)
 			if err != nil {
-				err = plugin.Errorf("Failed to allocate pool: %v", err)
 				return err
 			}
 
-			// Derive the subnet prefix from allocated IP address.
+			defer func() {
+				if err != nil {
+					plugin.invokeIpamDel(result, nwCfg.Ipam.Type, nwCfg, true)
+					plugin.invokeIpamDel(resultV6, ipamV6, nwCfg, true)
+				}
+			}()
+
+			// resultV6 = &cniTypesCurr.Result{}
+			// ip6Net := net.IPNet{
+			// 	IP:   net.ParseIP("fc00::1"),
+			// 	Mask: net.CIDRMask(64, 128),
+			// }
+			// gw6 := net.ParseIP("fc00::0")
+			// ip6config := &cniTypesCurr.IPConfig{
+			// 	Version: "6",
+			// 	Address: ip6Net,
+			// 	Gateway: gw6,
+			// }
+			// resultV6.IPs = append(resultV6.IPs, ip6config)
 			subnetPrefix = result.IPs[0].Address
-			iface := &cniTypesCurr.Interface{Name: args.IfName}
-			result.Interfaces = append(result.Interfaces, iface)
 		}
 
-		ipconfig := result.IPs[0]
-		gateway := ipconfig.Gateway
-
-		// On failure, call into IPAM plugin to release the address and address pool.
-		defer func() {
-			if err != nil {
-				nwCfg.Ipam.Subnet = subnetPrefix.String()
-				nwCfg.Ipam.Address = ipconfig.Address.IP.String()
-				plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg)
-
-				nwCfg.Ipam.Address = ""
-				plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg)
-			}
-		}()
-
+		gateway := result.IPs[0].Gateway
 		subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask)
 		// Find the master interface.
 		masterIfName := plugin.findMasterInterface(nwCfg, &subnetPrefix)
@@ -473,11 +552,15 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 			Policies:         policies,
 			NetNs:            args.Netns,
 			DisableHairpinOnHostInterface: nwCfg.DisableHairpinOnHostInterface,
+			IPV6Mode:                      nwCfg.IPV6Mode,
+			ServiceCidrs:                  nwCfg.ServiceCidrs,
 		}
 
 		nwInfo.Options = make(map[string]interface{})
 		setNetworkOptions(cnsNetworkConfig, &nwInfo)
 
+		addNatIPV6SubnetInfo(nwCfg, resultV6, &nwInfo)
+
 		err = plugin.nm.CreateNetwork(&nwInfo)
 		if err != nil {
 			err = plugin.Errorf("Failed to create network: %v", err)
@@ -488,28 +571,32 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 	} else {
 		if !nwCfg.MultiTenancy {
 			// Network already exists.
-			subnetPrefix := nwInfo.Subnets[0].Prefix.String()
-			log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix)
-			nwCfg.Ipam.Subnet = subnetPrefix
+			log.Printf("[cni-net] Found network %v with subnet %v.", networkId, nwInfo.Subnets[0].Prefix.String())
 
-			// Call into IPAM plugin to allocate an address for the endpoint.
-			result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg)
+			result, resultV6, err = plugin.invokeIpamAdd(nwCfg, nwInfo, false)
 			if err != nil {
-				err = plugin.Errorf("Failed to allocate address: %v", err)
 				return err
 			}
 
-			ipconfig := result.IPs[0]
-			iface := &cniTypesCurr.Interface{Name: args.IfName}
-			result.Interfaces = append(result.Interfaces, iface)
-
-			// On failure, call into IPAM plugin to release the address.
 			defer func() {
 				if err != nil {
-					nwCfg.Ipam.Address = ipconfig.Address.IP.String()
-					plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg)
+					plugin.invokeIpamDel(result, nwCfg.Ipam.Type, nwCfg, false)
+					plugin.invokeIpamDel(resultV6, ipamV6, nwCfg, false)
 				}
 			}()
+
+			// resultV6 = &cniTypesCurr.Result{}
+			// ip6Net := net.IPNet{
+			// 	IP:   net.ParseIP("fc00::2"),
+			// 	Mask: net.CIDRMask(64, 128),
+			// }
+			// gw6 := net.ParseIP("fc00::0")
+			// ip6config := &cniTypesCurr.IPConfig{
+			// 	Version: "6",
+			// 	Address: ip6Net,
+			// 	Gateway: gw6,
+			// }
+			// resultV6.IPs = append(resultV6.IPs, ip6config)
 		}
 	}
 
@@ -534,6 +621,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		PODName:            k8sPodName,
 		PODNameSpace:       k8sNamespace,
 		SkipHotAttachEp:    false, // Hot attach at the time of endpoint creation
+		IPV6Mode:           nwCfg.IPV6Mode,
 	}
 
 	epPolicies := getPoliciesFromRuntimeCfg(nwCfg)
@@ -546,11 +634,19 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address)
 	}
 
+	for _, ipconfig := range resultV6.IPs {
+		epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address)
+	}
+
 	// Populate routes.
 	for _, route := range result.Routes {
 		epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW})
 	}
 
+	for _, route := range resultV6.Routes {
+		epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW})
+	}
+
 	if azIpamResult != nil && azIpamResult.IPs != nil {
 		epInfo.InfraVnetIP = azIpamResult.IPs[0].Address
 	}
@@ -693,9 +789,10 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 		k8sPodName   string
 		k8sNamespace string
 		networkId    string
-		nwInfo       *network.NetworkInfo
+		nwInfo       network.NetworkInfo
 		epInfo       *network.EndpointInfo
 		cniMetric    telemetry.AIMetric
+		msg          string
 	)
 
 	startTime := time.Now()
@@ -703,7 +800,9 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 	log.Printf("[cni-net] Processing DEL command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v, StdinData:%s}.",
 		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path, args.StdinData)
 
-	defer func() { log.Printf("[cni-net] DEL command completed with err:%v.", err) }()
+	defer func() {
+		log.Printf("[cni-net] DEL command completed with err:%v.", err)
+	}()
 
 	// Parse network configuration from stdin.
 	if nwCfg, err = cni.ParseNetworkConfig(args.StdinData); err != nil {
@@ -713,14 +812,13 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 
 	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
 
-	iptables.DisableIPTableLock = nwCfg.DisableIPTableLock
-	plugin.setCNIReportDetails(nwCfg, CNI_DEL, "")
-
 	// Parse Pod arguments.
 	if k8sPodName, k8sNamespace, err = plugin.getPodInfo(args.Args); err != nil {
 		log.Printf("[cni-net] Failed to get POD info due to error: %v", err)
 	}
 
+	iptables.DisableIPTableLock = nwCfg.DisableIPTableLock
+
 	if nwCfg.MultiTenancy {
 		// Initialize CNSClient
 		cnsclient.InitCnsClient(nwCfg.CNSUrl)
@@ -749,6 +847,24 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 		return err
 	}
 
+	defer func() {
+		msg = fmt.Sprintf("CNI DEL succeeded : Released ip %+v podname %v namespace %v", nwCfg.Ipam.Address, k8sPodName, k8sNamespace)
+		if err != nil {
+			msg = fmt.Sprintf("CNI DEL failed : ip %+v podname %v namespace %v", nwCfg.Ipam.Address, k8sPodName, k8sNamespace)
+		}
+
+		plugin.setCNIReportDetails(nwCfg, CNI_DEL, msg)
+
+		operationTimeMs := time.Since(startTime).Milliseconds()
+		cniMetric.Metric = aitelemetry.Metric{
+			Name:             telemetry.CNIDelTimeMetricStr,
+			Value:            float64(operationTimeMs),
+			CustomDimensions: make(map[string]string),
+		}
+		SetCustomDimensions(&cniMetric, nwCfg, err)
+		telemetry.SendCNIMetric(&cniMetric, plugin.tb)
+	}()
+
 	// Delete the endpoint.
 	if err = plugin.nm.DeleteEndpoint(networkId, endpointId); err != nil {
 		err = plugin.Errorf("Failed to delete endpoint: %v", err)
@@ -757,13 +873,27 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 
 	if !nwCfg.MultiTenancy {
 		// Call into IPAM plugin to release the endpoint's addresses.
-		nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String()
 		for _, address := range epInfo.IPAddresses {
 			nwCfg.Ipam.Address = address.IP.String()
-			err = plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg)
-			if err != nil {
-				err = plugin.Errorf("Failed to release address: %v", err)
-				return err
+			if address.IP.To4() != nil {
+				nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String()
+				log.Printf("Releasing ipv4 address :%s pool: %s",
+					nwCfg.Ipam.Address, nwCfg.Ipam.Subnet)
+				if err = plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg); err != nil {
+					log.Printf("Failed to release ipv4 address: %v", err)
+					err = plugin.Errorf("Failed to release ipv4 address: %v", err)
+				}
+			} else {
+				if len(nwInfo.Subnets) > 1 {
+					nwCfg.Ipam.Subnet = nwInfo.Subnets[1].Prefix.String()
+				}
+
+				log.Printf("Releasing ipv6 address :%s pool: %s",
+					nwCfg.Ipam.Address, nwCfg.Ipam.Subnet)
+				if err = plugin.DelegateDel(ipamV6, nwCfg); err != nil {
+					log.Printf("Failed to release ipv6 address: %v", err)
+					err = plugin.Errorf("Failed to release ipv6 address: %v", err)
+				}
 			}
 		}
 	} else if epInfo.EnableInfraVnet {
@@ -771,24 +901,12 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 		nwCfg.Ipam.Address = epInfo.InfraVnetIP.IP.String()
 		err = plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg)
 		if err != nil {
+			log.Printf("Failed to release address: %v", err)
 			err = plugin.Errorf("Failed to release address: %v", err)
-			return err
 		}
 	}
 
-	msg := fmt.Sprintf("CNI DEL succeeded : Released ip %+v podname %v namespace %v", nwCfg.Ipam.Address, k8sPodName, k8sNamespace)
-	plugin.setCNIReportDetails(nwCfg, CNI_DEL, msg)
-
-	operationTimeMs := time.Since(startTime).Milliseconds()
-	cniMetric.Metric = aitelemetry.Metric{
-		Name:             telemetry.CNIDelTimeMetricStr,
-		Value:            float64(operationTimeMs),
-		CustomDimensions: make(map[string]string),
-	}
-	SetCustomDimensions(&cniMetric, nwCfg, nil)
-	telemetry.SendCNIMetric(&cniMetric, plugin.tb)
-
-	return nil
+	return err
 }
 
 // Update handles CNI update commands.
diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go
index 263a30ae31..e4a7f72757 100644
--- a/cni/network/network_linux.go
+++ b/cni/network/network_linux.go
@@ -25,7 +25,7 @@ const (
 )
 
 // handleConsecutiveAdd is a dummy function for Linux platform.
-func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) {
+func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) {
 	return nil, nil
 }
 
diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go
index d839fa0175..07016264e8 100644
--- a/cni/network/network_windows.go
+++ b/cni/network/network_windows.go
@@ -33,7 +33,7 @@ var (
  * We can delete this if statement once they fix it.
  * Issue link: https://github.com/kubernetes/kubernetes/issues/57253
  */
-func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) {
+func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) {
 	// Return in case of HNSv2 as consecutive add call doesn't need to be handled
 	if useHnsV2, err := network.UseHnsV2(args.Netns); useHnsV2 {
 		return nil, err
diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go
index ce1e80c934..d30cd5871e 100644
--- a/ebtables/ebtables.go
+++ b/ebtables/ebtables.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-container-networking/log"
+	"github.com/Azure/azure-container-networking/platform"
 )
 
 const (
@@ -82,11 +83,34 @@ func SetVepaMode(bridgeName string, downstreamIfNamePrefix string, upstreamMacAd
 
 // SetDnatForIPAddress sets a MAC DNAT rule for an IP address.
 func SetDnatForIPAddress(interfaceName string, ipAddress net.IP, macAddress net.HardwareAddr, action string) error {
+	protocol := "IPv4"
+	dst := "--ip-dst"
+	if ipAddress.To4() == nil {
+		protocol = "IPv6"
+		dst = "--ip6-dst"
+	}
+
 	command := fmt.Sprintf(
-		"ebtables -t nat %s PREROUTING -p IPv4 -i %s --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT",
-		action, interfaceName, ipAddress.String(), macAddress.String())
+		"ebtables -t nat %s PREROUTING -p %s -i %s %s %s -j dnat --to-dst %s --dnat-target ACCEPT",
+		action, protocol, interfaceName, dst, ipAddress.String(), macAddress.String())
 
-	return executeShellCommand(command)
+	out, err := platform.ExecuteCommand(command)
+	log.Printf("dnat out:%s", out)
+	return err
+}
+
+func SetBrouteRule(ipNet net.IPNet, action string) error {
+	protocol := "IPv4"
+	dst := "--ip-dst"
+	if ipNet.IP.To4() == nil {
+		protocol = "IPv6"
+		dst = "--ip6-dst"
+	}
+	cmd := fmt.Sprintf("ebtables -t broute %s BROUTING -p %s %s %s -j redirect --redirect-target ACCEPT",
+		action, protocol, dst, ipNet.String())
+	log.Printf("ebcmd:%s", cmd)
+	_, err := platform.ExecuteCommand(cmd)
+	return err
 }
 
 func executeShellCommand(command string) error {
diff --git a/iptables/iptables.go b/iptables/iptables.go
index a3d8d73373..6f070b7c93 100644
--- a/iptables/iptables.go
+++ b/iptables/iptables.go
@@ -52,21 +52,32 @@ const (
 
 const (
 	iptables    = "iptables"
+	ip6tables   = "ip6tables"
 	lockTimeout = 60
 )
 
+const (
+	V4 = "4"
+	V6 = "6"
+)
+
 var (
 	DisableIPTableLock bool
 )
 
 // Run iptables command
-func runCmd(params string) error {
+func runCmd(version, params string) error {
 	var cmd string
 
+	iptCmd := iptables
+	if version == V6 {
+		iptCmd = ip6tables
+	}
+
 	if DisableIPTableLock {
-		cmd = fmt.Sprintf("%s %s", iptables, params)
+		cmd = fmt.Sprintf("%s %s", iptCmd, params)
 	} else {
-		cmd = fmt.Sprintf("%s -w %d %s", iptables, lockTimeout, params)
+		cmd = fmt.Sprintf("%s -w %d %s", iptCmd, lockTimeout, params)
 	}
 
 	if _, err := platform.ExecuteCommand(cmd); err != nil {
@@ -77,9 +88,9 @@ func runCmd(params string) error {
 }
 
 // check if iptable chain alreay exists
-func ChainExists(tableName, chainName string) bool {
+func ChainExists(version, tableName, chainName string) bool {
 	params := fmt.Sprintf("-t %s -L %s", tableName, chainName)
-	if err := runCmd(params); err != nil {
+	if err := runCmd(version, params); err != nil {
 		return false
 	}
 
@@ -87,12 +98,12 @@ func ChainExists(tableName, chainName string) bool {
 }
 
 // create new iptable chain under specified table name
-func CreateChain(tableName, chainName string) error {
+func CreateChain(version, tableName, chainName string) error {
 	var err error
 
-	if !ChainExists(tableName, chainName) {
+	if !ChainExists(version, tableName, chainName) {
 		params := fmt.Sprintf("-t %s -N %s", tableName, chainName)
-		err = runCmd(params)
+		err = runCmd(version, params)
 	} else {
 		log.Printf("%s Chain exists in table %s", chainName, tableName)
 	}
@@ -101,38 +112,38 @@ func CreateChain(tableName, chainName string) error {
 }
 
 // check if iptable rule alreay exists
-func RuleExists(tableName, chainName, match, target string) bool {
+func RuleExists(version, tableName, chainName, match, target string) bool {
 	params := fmt.Sprintf("-t %s -C %s %s -j %s", tableName, chainName, match, target)
-	if err := runCmd(params); err != nil {
+	if err := runCmd(version, params); err != nil {
 		return false
 	}
 	return true
 }
 
 // Insert iptable rule at beginning of iptable chain
-func InsertIptableRule(tableName, chainName, match, target string) error {
-	if RuleExists(tableName, chainName, match, target) {
+func InsertIptableRule(version, tableName, chainName, match, target string) error {
+	if RuleExists(version, tableName, chainName, match, target) {
 		log.Printf("Rule already exists")
 		return nil
 	}
 
 	params := fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target)
-	return runCmd(params)
+	return runCmd(version, params)
 }
 
 // Append iptable rule at end of iptable chain
-func AppendIptableRule(tableName, chainName, match, target string) error {
-	if RuleExists(tableName, chainName, match, target) {
+func AppendIptableRule(version, tableName, chainName, match, target string) error {
+	if RuleExists(version, tableName, chainName, match, target) {
 		log.Printf("Rule already exists")
 		return nil
 	}
 
 	params := fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target)
-	return runCmd(params)
+	return runCmd(version, params)
 }
 
 // Delete matched iptable rule
-func DeleteIptableRule(tableName, chainName, match, target string) error {
+func DeleteIptableRule(version, tableName, chainName, match, target string) error {
 	params := fmt.Sprintf("-t %s -D %s %s -j %s", tableName, chainName, match, target)
-	return runCmd(params)
+	return runCmd(version, params)
 }
diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go
index 4527fb94a7..a75dbd7429 100644
--- a/network/bridge_endpointclient_linux.go
+++ b/network/bridge_endpointclient_linux.go
@@ -61,10 +61,12 @@ func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo)
 	}
 
 	for _, ipAddr := range epInfo.IPAddresses {
-		// Add ARP reply rule.
-		log.Printf("[net] Adding ARP reply rule for IP address %v", ipAddr.String())
-		if err = ebtables.SetArpReply(ipAddr.IP, client.getArpReplyAddress(client.containerMac), ebtables.Append); err != nil {
-			return err
+		if ipAddr.IP.To4() != nil {
+			// Add ARP reply rule.
+			log.Printf("[net] Adding ARP reply rule for IP address %v", ipAddr.String())
+			if err = ebtables.SetArpReply(ipAddr.IP, client.getArpReplyAddress(client.containerMac), ebtables.Append); err != nil {
+				return err
+			}
 		}
 
 		// Add MAC address translation rule.
@@ -73,7 +75,7 @@ func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo)
 			return err
 		}
 
-		if client.mode != opModeTunnel {
+		if client.mode != opModeTunnel && ipAddr.IP.To4() != nil {
 			log.Printf("[net] Adding static arp for IP address %v and MAC %v in VM", ipAddr.String(), client.containerMac.String())
 			if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.bridgeName, ipAddr.IP, client.containerMac); err != nil {
 				log.Printf("Failed setting arp in vm: %v", err)
@@ -93,23 +95,25 @@ func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo)
 func (client *LinuxBridgeEndpointClient) DeleteEndpointRules(ep *endpoint) {
 	// Delete rules for IP addresses on the container interface.
 	for _, ipAddr := range ep.IPAddresses {
-		// Delete ARP reply rule.
-		log.Printf("[net] Deleting ARP reply rule for IP address %v on %v.", ipAddr.String(), ep.Id)
-		err := ebtables.SetArpReply(ipAddr.IP, client.getArpReplyAddress(ep.MacAddress), ebtables.Delete)
-		if err != nil {
-			log.Printf("[net] Failed to delete ARP reply rule for IP address %v: %v.", ipAddr.String(), err)
+		if ipAddr.IP.To4() != nil {
+			// Delete ARP reply rule.
+			log.Printf("[net] Deleting ARP reply rule for IP address %v on %v.", ipAddr.String(), ep.Id)
+			err := ebtables.SetArpReply(ipAddr.IP, client.getArpReplyAddress(ep.MacAddress), ebtables.Delete)
+			if err != nil {
+				log.Printf("[net] Failed to delete ARP reply rule for IP address %v: %v.", ipAddr.String(), err)
+			}
 		}
 
 		// Delete MAC address translation rule.
 		log.Printf("[net] Deleting MAC DNAT rule for IP address %v on %v.", ipAddr.String(), ep.Id)
-		err = ebtables.SetDnatForIPAddress(client.hostPrimaryIfName, ipAddr.IP, ep.MacAddress, ebtables.Delete)
+		err := ebtables.SetDnatForIPAddress(client.hostPrimaryIfName, ipAddr.IP, ep.MacAddress, ebtables.Delete)
 		if err != nil {
 			log.Printf("[net] Failed to delete MAC DNAT rule for IP address %v: %v.", ipAddr.String(), err)
 		}
 
-		if client.mode != opModeTunnel {
+		if client.mode != opModeTunnel && ipAddr.IP.To4() != nil {
 			log.Printf("[net] Removing static arp for IP address %v and MAC %v from VM", ipAddr.String(), ep.MacAddress.String())
-			netlink.AddOrRemoveStaticArp(netlink.REMOVE, client.bridgeName, ipAddr.IP, ep.MacAddress)
+			err := netlink.AddOrRemoveStaticArp(netlink.REMOVE, client.bridgeName, ipAddr.IP, ep.MacAddress)
 			if err != nil {
 				log.Printf("Failed removing arp from vm: %v", err)
 			}
@@ -153,6 +157,13 @@ func (client *LinuxBridgeEndpointClient) SetupContainerInterfaces(epInfo *Endpoi
 }
 
 func (client *LinuxBridgeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error {
+	if epInfo.IPV6Mode != "" {
+		// Enable ipv6 setting in container
+		if err := epcommon.UpdateIPV6Setting(0); err != nil {
+			return err
+		}
+	}
+
 	if err := epcommon.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil {
 		return err
 	}
diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go
index 24ebb0b79d..40805748f4 100644
--- a/network/bridge_networkclient_linux.go
+++ b/network/bridge_networkclient_linux.go
@@ -2,6 +2,7 @@ package network
 
 import (
 	"net"
+	"strings"
 
 	"github.com/Azure/azure-container-networking/ebtables"
 	"github.com/Azure/azure-container-networking/log"
@@ -11,13 +12,13 @@ import (
 type LinuxBridgeClient struct {
 	bridgeName        string
 	hostInterfaceName string
-	mode              string
+	nwInfo            NetworkInfo
 }
 
-func NewLinuxBridgeClient(bridgeName string, hostInterfaceName string, mode string) *LinuxBridgeClient {
+func NewLinuxBridgeClient(bridgeName string, hostInterfaceName string, nwInfo NetworkInfo) *LinuxBridgeClient {
 	client := &LinuxBridgeClient{
 		bridgeName:        bridgeName,
-		mode:              mode,
+		nwInfo:            nwInfo,
 		hostInterfaceName: hostInterfaceName,
 	}
 
@@ -80,8 +81,12 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error {
 		return err
 	}
 
+	if err := client.addBrouteServiceCidrs(); err != nil {
+		return err
+	}
+
 	// Enable VEPA for host policy enforcement if necessary.
-	if client.mode == opModeTunnel {
+	if client.nwInfo.Mode == opModeTunnel {
 		log.Printf("[net] Enabling VEPA mode for %v.", client.hostInterfaceName)
 		if err := ebtables.SetVepaMode(client.bridgeName, commonInterfacePrefix, virtualMacAddress, ebtables.Append); err != nil {
 			return err
@@ -105,3 +110,24 @@ func (client *LinuxBridgeClient) SetBridgeMasterToHostInterface() error {
 func (client *LinuxBridgeClient) SetHairpinOnHostInterface(enable bool) error {
 	return netlink.SetLinkHairpin(client.hostInterfaceName, enable)
 }
+
+func (client *LinuxBridgeClient) addBrouteServiceCidrs() error {
+	if client.nwInfo.ServiceCidrs != "" {
+		serviceCidrs := strings.Split(client.nwInfo.ServiceCidrs, ",")
+		for _, ipCidrStr := range serviceCidrs {
+			log.Printf("[net] Adding brouting rule for service cidr %s.", ipCidrStr)
+
+			ip, ipNet, _ := net.ParseCIDR(ipCidrStr)
+			svcAddr := net.IPNet{
+				IP:   ip,
+				Mask: ipNet.Mask,
+			}
+
+			if err := ebtables.SetBrouteRule(svcAddr, ebtables.Append); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
diff --git a/network/endpoint.go b/network/endpoint.go
index 772ffce3f4..a19865b740 100644
--- a/network/endpoint.go
+++ b/network/endpoint.go
@@ -71,6 +71,7 @@ type EndpointInfo struct {
 	Data                     map[string]interface{}
 	InfraVnetAddressSpace    string
 	SkipHotAttachEp          bool
+	IPV6Mode                 string
 }
 
 // RouteInfo contains information about an IP route.
diff --git a/network/epcommon/endpoint_common.go b/network/epcommon/endpoint_common.go
index 5d7c9d5065..74f8f69622 100644
--- a/network/epcommon/endpoint_common.go
+++ b/network/epcommon/endpoint_common.go
@@ -29,6 +29,7 @@ RFC for Link Local Addresses: https://tools.ietf.org/html/rfc3927
 
 const (
 	enableIPForwardCmd = "sysctl -w net.ipv4.ip_forward=1"
+	enableIPV6Cmd      = "sysctl -w net.ipv6.conf.all.disable_ipv6=%d"
 )
 
 func getPrivateIPSpace() []string {
@@ -115,11 +116,11 @@ func addOrDeleteFilterRule(bridgeName string, action string, ipAddress string, c
 
 	switch action {
 	case iptables.Insert:
-		err = iptables.InsertIptableRule(iptables.Filter, chainName, matchCondition, target)
+		err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target)
 	case iptables.Append:
-		err = iptables.AppendIptableRule(iptables.Filter, chainName, matchCondition, target)
+		err = iptables.AppendIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target)
 	case iptables.Delete:
-		err = iptables.DeleteIptableRule(iptables.Filter, chainName, matchCondition, target)
+		err = iptables.DeleteIptableRule(iptables.V4, iptables.Filter, chainName, matchCondition, target)
 	}
 
 	return err
@@ -173,9 +174,7 @@ func BlockIPAddresses(bridgeName string, action string) error {
 	return nil
 }
 
-/**
- This fucntion enables ip forwarding in VM and allow forwarding packets from the interface
-**/
+// This fucntion enables ip forwarding in VM and allow forwarding packets from the interface
 func EnableIPForwarding(ifName string) error {
 	// Enable ip forwading on linux vm.
 	// sysctl -w net.ipv4.ip_forward=1
@@ -187,10 +186,33 @@ func EnableIPForwarding(ifName string) error {
 	}
 
 	// Append a rule in forward chain to allow forwarding from bridge
-	if err := iptables.AppendIptableRule(iptables.Filter, iptables.Forward, "", iptables.Accept); err != nil {
+	if err := iptables.AppendIptableRule(iptables.V4, iptables.Filter, iptables.Forward, "", iptables.Accept); err != nil {
 		log.Printf("[net] Appending forward chain rule: allow traffic coming from snatbridge failed with: %v", err)
 		return err
 	}
 
 	return nil
 }
+
+// This functions enables/disables ipv6 setting based on enable parameter passed.
+func UpdateIPV6Setting(disable int) error {
+	// sysctl -w net.ipv6.conf.all.disable_ipv6=0/1
+	cmd := fmt.Sprintf(enableIPV6Cmd, disable)
+	_, err := platform.ExecuteCommand(cmd)
+	if err != nil {
+		log.Printf("[net] Update IPV6 Setting failed with: %v", err)
+	}
+
+	return err
+}
+
+// This fucntion adds rule which snat to ip passed filtered by match string.
+func AddSnatRule(match string, ip net.IP) error {
+	version := iptables.V4
+	if ip.To4() == nil {
+		version = iptables.V6
+	}
+
+	target := fmt.Sprintf("SNAT --to %s", ip.String())
+	return iptables.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target)
+}
diff --git a/network/manager.go b/network/manager.go
index 98bcbf5ace..567cfc4d4b 100644
--- a/network/manager.go
+++ b/network/manager.go
@@ -57,7 +57,7 @@ type NetworkManager interface {
 
 	CreateNetwork(nwInfo *NetworkInfo) error
 	DeleteNetwork(networkId string) error
-	GetNetworkInfo(networkId string) (*NetworkInfo, error)
+	GetNetworkInfo(networkId string) (NetworkInfo, error)
 
 	CreateEndpoint(networkId string, epInfo *EndpointInfo) error
 	DeleteEndpoint(networkId string, endpointId string) error
@@ -169,7 +169,7 @@ func (nm *networkManager) restore() error {
 
 				extIf.BridgeName = ""
 
-				_, err = nm.newNetworkImpl(nwInfo, extIf)
+				_, err = nm.newNetworkImpl(&nwInfo, extIf)
 				if err != nil {
 					log.Printf("[net] Restoring network failed for nwInfo %v extif %v. This should not happen %v", nwInfo, extIf, err)
 					return err
@@ -269,16 +269,16 @@ func (nm *networkManager) DeleteNetwork(networkId string) error {
 }
 
 // GetNetworkInfo returns information about the given network.
-func (nm *networkManager) GetNetworkInfo(networkId string) (*NetworkInfo, error) {
+func (nm *networkManager) GetNetworkInfo(networkId string) (NetworkInfo, error) {
 	nm.Lock()
 	defer nm.Unlock()
 
 	nw, err := nm.getNetwork(networkId)
 	if err != nil {
-		return nil, err
+		return NetworkInfo{}, err
 	}
 
-	nwInfo := &NetworkInfo{
+	nwInfo := NetworkInfo{
 		Id:               networkId,
 		Subnets:          nw.Subnets,
 		Mode:             nw.Mode,
@@ -287,7 +287,7 @@ func (nm *networkManager) GetNetworkInfo(networkId string) (*NetworkInfo, error)
 		Options:          make(map[string]interface{}),
 	}
 
-	getNetworkInfoImpl(nwInfo, nw)
+	getNetworkInfoImpl(&nwInfo, nw)
 
 	if nw.extIf != nil {
 		nwInfo.BridgeName = nw.extIf.BridgeName
diff --git a/network/network.go b/network/network.go
index bdaff589b7..db39bb1882 100644
--- a/network/network.go
+++ b/network/network.go
@@ -20,6 +20,11 @@ const (
 	opModeDefault     = opModeTunnel
 )
 
+const (
+	// ipv6 modes
+	IPV6Nat = "ipv6nat"
+)
+
 // ExternalInterface is a host network interface that bridges containers to external networks.
 type externalInterface struct {
 	Name        string
@@ -62,6 +67,8 @@ type NetworkInfo struct {
 	NetNs                         string
 	Options                       map[string]interface{}
 	DisableHairpinOnHostInterface bool
+	IPV6Mode                      string
+	ServiceCidrs                  string
 }
 
 // SubnetInfo contains subnet information for a container network.
diff --git a/network/network_linux.go b/network/network_linux.go
index 5c7e3531d6..02988e61a0 100644
--- a/network/network_linux.go
+++ b/network/network_linux.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/Azure/azure-container-networking/log"
 	"github.com/Azure/azure-container-networking/netlink"
+	"github.com/Azure/azure-container-networking/network/epcommon"
 	"github.com/Azure/azure-container-networking/platform"
 	"golang.org/x/sys/unix"
 )
@@ -90,7 +91,8 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error {
 	if nw.VlanId != 0 {
 		networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name)
 	} else {
-		networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, nw.Mode)
+		nwInfo, _ := nm.GetNetworkInfo(nw.Id)
+		networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, nwInfo)
 	}
 
 	// Disconnect the interface if this was the last network using it.
@@ -287,8 +289,11 @@ func applyDnsConfig(extIf *externalInterface, ifName string) error {
 
 // ConnectExternalInterface connects the given host interface to a bridge.
 func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error {
-	var err error
-	var networkClient NetworkClient
+	var (
+		err           error
+		networkClient NetworkClient
+	)
+
 	log.Printf("[net] Connecting interface %v.", extIf.Name)
 	defer func() { log.Printf("[net] Connecting interface %v completed with err:%v.", extIf.Name, err) }()
 
@@ -314,7 +319,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 	if opt != nil && opt[VlanIDKey] != nil {
 		networkClient = NewOVSClient(bridgeName, extIf.Name)
 	} else {
-		networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, nwInfo.Mode)
+		networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, *nwInfo)
 	}
 
 	// Check if the bridge already exists.
@@ -416,6 +421,18 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 		log.Printf("[net] Applied dns config %v on %v", extIf.DNSInfo, bridgeName)
 	}
 
+	if nwInfo.IPV6Mode != "" {
+		if err := addIpv6NatGateway(nwInfo); err != nil {
+			log.Errorf("[net] Adding IPv6 Nat Gateway failed:%v", err)
+			return err
+		}
+
+		if err := addIpv6SnatRule(extIf); err != nil {
+			log.Errorf("[net] Adding IPv6 Snat Rule failed:%v", err)
+			return err
+		}
+	}
+
 	extIf.BridgeName = bridgeName
 	log.Printf("[net] Connected interface %v to bridge %v.", extIf.Name, extIf.BridgeName)
 
@@ -450,6 +467,34 @@ func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface,
 	log.Printf("[net] Disconnected interface %v.", extIf.Name)
 }
 
+func addIpv6NatGateway(nwInfo *NetworkInfo) error {
+	if nwInfo.IPV6Mode == IPV6Nat {
+		log.Printf("[net] Adding ipv6 nat gateway on azure bridge")
+		for _, subnetInfo := range nwInfo.Subnets {
+			if subnetInfo.Family == platform.AfINET6 {
+				ipAddr := []net.IPNet{{
+					IP:   subnetInfo.Gateway,
+					Mask: subnetInfo.Prefix.Mask,
+				}}
+				return epcommon.AssignIPToInterface(nwInfo.BridgeName, ipAddr)
+			}
+		}
+	}
+
+	return nil
+}
+
+func addIpv6SnatRule(extIf *externalInterface) error {
+	log.Printf("[net] Adding ipv6 snat rule")
+	for _, ipAddr := range extIf.IPAddresses {
+		if ipAddr.IP.To4() == nil {
+			return epcommon.AddSnatRule("", ipAddr.IP)
+		}
+	}
+
+	return nil
+}
+
 func getNetworkInfoImpl(nwInfo *NetworkInfo, nw *network) {
 	if nw.VlanId != 0 {
 		vlanMap := make(map[string]interface{})
diff --git a/network/ovssnat/ovssnat.go b/network/ovssnat/ovssnat.go
index 63a8eaa7c3..8e908750c2 100644
--- a/network/ovssnat/ovssnat.go
+++ b/network/ovssnat/ovssnat.go
@@ -141,40 +141,40 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error {
 	bridgeIP, containerIP := getNCLocalAndGatewayIP(client)
 
 	// Create CNI Ouptut chain
-	if err := iptables.CreateChain(iptables.Filter, iptables.CNIOutputChain); err != nil {
+	if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil {
 		log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err)
 		return err
 	}
 
 	// Forward traffic from Ouptut chain to CNI Output chain
-	if err := iptables.InsertIptableRule(iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil {
+	if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil {
 		log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err)
 		return err
 	}
 
 	// Allow connection from Host to NC
 	matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String())
-	err := iptables.InsertIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept)
+	err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept)
 	if err != nil {
 		log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err)
 		return err
 	}
 
 	// Create cniinput chain
-	if err := iptables.CreateChain(iptables.Filter, iptables.CNIInputChain); err != nil {
+	if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil {
 		log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err)
 		return err
 	}
 
 	// Forward from Input to cniinput chain
-	if err := iptables.InsertIptableRule(iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil {
+	if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil {
 		log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err)
 		return err
 	}
 
 	// Accept packets from NC only if established connection
 	matchCondition = fmt.Sprintf(" -i %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related)
-	err = iptables.InsertIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept)
+	err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept)
 	if err != nil {
 		log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err)
 		return err
@@ -197,7 +197,7 @@ func (client *OVSSnatClient) DeleteInboundFromHostToNC() error {
 
 	// Delete allow connection from Host to NC
 	matchCondition := fmt.Sprintf("-s %s -d %s", bridgeIP.String(), containerIP.String())
-	err := iptables.DeleteIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept)
+	err := iptables.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept)
 	if err != nil {
 		log.Printf("DeleteInboundFromHostToNC: Error removing output rule %v", err)
 	}
@@ -219,40 +219,40 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error {
 	bridgeIP, containerIP := getNCLocalAndGatewayIP(client)
 
 	// Create CNI Input chain
-	if err := iptables.CreateChain(iptables.Filter, iptables.CNIInputChain); err != nil {
+	if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIInputChain); err != nil {
 		log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIInputChain, err)
 		return err
 	}
 
 	// Forward traffic from Input to cniinput chain
-	if err := iptables.InsertIptableRule(iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil {
+	if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Input, "", iptables.CNIInputChain); err != nil {
 		log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIInputChain, err)
 		return err
 	}
 
 	// Allow NC to Host connection
 	matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String())
-	err := iptables.InsertIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept)
+	err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept)
 	if err != nil {
 		log.Printf("AllowInboundFromHostToNC: Inserting output rule failed: %v", err)
 		return err
 	}
 
 	// Create CNI output chain
-	if err := iptables.CreateChain(iptables.Filter, iptables.CNIOutputChain); err != nil {
+	if err := iptables.CreateChain(iptables.V4, iptables.Filter, iptables.CNIOutputChain); err != nil {
 		log.Printf("AllowInboundFromHostToNC: Creating %v failed with error: %v", iptables.CNIOutputChain, err)
 		return err
 	}
 
 	// Forward traffic from Output to CNI Output chain
-	if err := iptables.InsertIptableRule(iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil {
+	if err := iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.Output, "", iptables.CNIOutputChain); err != nil {
 		log.Printf("AllowInboundFromHostToNC: Inserting forward rule to %v failed with error: %v", iptables.CNIOutputChain, err)
 		return err
 	}
 
 	// Accept packets from Host only if established connection
 	matchCondition = fmt.Sprintf(" -o %s -m state --state %s,%s", SnatBridgeName, iptables.Established, iptables.Related)
-	err = iptables.InsertIptableRule(iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept)
+	err = iptables.InsertIptableRule(iptables.V4, iptables.Filter, iptables.CNIOutputChain, matchCondition, iptables.Accept)
 	if err != nil {
 		log.Printf("AllowInboundFromHostToNC: Inserting input rule failed: %v", err)
 		return err
@@ -275,7 +275,7 @@ func (client *OVSSnatClient) DeleteInboundFromNCToHost() error {
 
 	// Delete allow NC to Host connection
 	matchCondition := fmt.Sprintf("-s %s -d %s", containerIP.String(), bridgeIP.String())
-	err := iptables.DeleteIptableRule(iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept)
+	err := iptables.DeleteIptableRule(iptables.V4, iptables.Filter, iptables.CNIInputChain, matchCondition, iptables.Accept)
 	if err != nil {
 		log.Printf("DeleteInboundFromNCToHost: Error removing output rule %v", err)
 	}
@@ -417,7 +417,7 @@ func DeleteSnatBridge(bridgeName string) error {
 func AddMasqueradeRule(snatBridgeIPWithPrefix string) error {
 	_, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix)
 	matchCondition := fmt.Sprintf("-s %s", ipNet.String())
-	return iptables.InsertIptableRule(iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade)
+	return iptables.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade)
 }
 
 func DeleteMasqueradeRule() error {
@@ -436,7 +436,7 @@ func DeleteMasqueradeRule() error {
 
 		if ipAddr.To4() != nil {
 			matchCondition := fmt.Sprintf("-s %s", ipNet.String())
-			return iptables.DeleteIptableRule(iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade)
+			return iptables.DeleteIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, matchCondition, iptables.Masquerade)
 		}
 	}
 

From 9f79c6a1ae1fe61c55950478bd4868462836febd Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Fri, 3 Apr 2020 13:11:57 -0700
Subject: [PATCH 06/12] drop neighbor discovery messages to other VMs

---
 Makefile                              |  3 +-
 cni/ipam/ipam.go                      | 11 ++++-
 cni/network/network.go                | 70 ++++++++++-----------------
 ebtables/ebtables.go                  | 12 +++++
 network/bridge_networkclient_linux.go |  6 +++
 5 files changed, 56 insertions(+), 46 deletions(-)

diff --git a/Makefile b/Makefile
index 7981828a24..a86dcdbdf3 100644
--- a/Makefile
+++ b/Makefile
@@ -376,7 +376,8 @@ test-all:
 		./aitelemetry/ \
         ./cnm/network/ \
         ./cni/ipam/ \
-        ./cns/ipamclient/ 
+        ./cns/ipamclient/ \
+        ./cnms/service/
 	cd npm/ && go test -v -covermode count -coverprofile=coverage-npm.out \
         ./iptm/ \
         ./ipsm/ \
diff --git a/cni/ipam/ipam.go b/cni/ipam/ipam.go
index 2c17ef92b1..49e8e5c52c 100644
--- a/cni/ipam/ipam.go
+++ b/cni/ipam/ipam.go
@@ -19,6 +19,10 @@ import (
 	cniTypesCurr "github.com/containernetworking/cni/pkg/types/current"
 )
 
+const (
+	ipamV6 = "azure-vnet-ipamv6"
+)
+
 var (
 	ipv4DefaultRouteDstPrefix = net.IPNet{net.IPv4zero, net.IPv4Mask(0, 0, 0, 0)}
 )
@@ -154,8 +158,13 @@ func (plugin *ipamPlugin) Add(args *cniSkel.CmdArgs) error {
 		options := make(map[string]string)
 		options[ipam.OptInterfaceName] = nwCfg.Master
 
+		isIpv6 := false
+		if nwCfg.Ipam.Type == ipamV6 {
+			isIpv6 = true
+		}
+
 		// Allocate an address pool.
-		poolID, subnet, err = plugin.am.RequestPool(nwCfg.Ipam.AddrSpace, "", "", options, false)
+		poolID, subnet, err = plugin.am.RequestPool(nwCfg.Ipam.AddrSpace, "", "", options, isIpv6)
 		if err != nil {
 			err = plugin.Errorf("Failed to allocate pool: %v", err)
 			return err
diff --git a/cni/network/network.go b/cni/network/network.go
index a0f1b14a98..571d45121c 100644
--- a/cni/network/network.go
+++ b/cni/network/network.go
@@ -246,24 +246,29 @@ func addNatIPV6SubnetInfo(nwCfg *cni.NetworkConfig,
 func (plugin *netPlugin) invokeIpamDel(
 	result *cniTypesCurr.Result,
 	ipamType string,
-	nwCfg *cni.NetworkConfig,
+	nwCfg cni.NetworkConfig,
 	isDeletePoolOnError bool) {
 
 	if result != nil {
+		if ipamType == ipamV6 {
+			nwCfg.Ipam.Environment = common.OptEnvironmentIPv6NodeIpam
+			nwCfg.Ipam.Type = ipamType
+		}
+
 		nwCfg.Ipam.Subnet = result.IPs[0].Address.String()
 		nwCfg.Ipam.Address = result.IPs[0].Address.IP.String()
-		plugin.DelegateDel(ipamType, nwCfg)
+		plugin.DelegateDel(ipamType, &nwCfg)
 
 		// Release pool
 		if isDeletePoolOnError {
 			nwCfg.Ipam.Address = ""
-			plugin.DelegateDel(ipamType, nwCfg)
+			plugin.DelegateDel(ipamType, &nwCfg)
 		}
 	}
 }
 
 func (plugin *netPlugin) invokeIpamAdd(
-	nwCfg *cni.NetworkConfig,
+	nwCfg cni.NetworkConfig,
 	nwInfo network.NetworkInfo,
 	isDeletePoolOnError bool) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) {
 
@@ -278,7 +283,7 @@ func (plugin *netPlugin) invokeIpamAdd(
 	}
 
 	// Call into IPAM plugin to allocate an address pool for the network.
-	result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg)
+	result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, &nwCfg)
 	if err != nil {
 		err = plugin.Errorf("Failed to allocate pool: %v", err)
 		return result, resultV6, err
@@ -291,11 +296,14 @@ func (plugin *netPlugin) invokeIpamAdd(
 	}()
 
 	if nwCfg.IPV6Mode != "" {
+		nwCfg.Ipam.Environment = common.OptEnvironmentIPv6NodeIpam
+		nwCfg.Ipam.Type = ipamV6
+
 		if len(nwInfo.Subnets) > 1 {
 			nwCfg.Ipam.Subnet = nwInfo.Subnets[1].Prefix.String()
 		}
 
-		resultV6, err = plugin.DelegateAdd(ipamV6, nwCfg)
+		resultV6, err = plugin.DelegateAdd(ipamV6, &nwCfg)
 		if err != nil {
 			err = plugin.Errorf("Failed to allocate v6 pool: %v", err)
 		}
@@ -481,30 +489,18 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		log.Printf("[cni-net] Creating network %v.", networkId)
 
 		if !nwCfg.MultiTenancy {
-			result, resultV6, err = plugin.invokeIpamAdd(nwCfg, nwInfo, true)
+			result, resultV6, err = plugin.invokeIpamAdd(*nwCfg, nwInfo, true)
 			if err != nil {
 				return err
 			}
 
 			defer func() {
 				if err != nil {
-					plugin.invokeIpamDel(result, nwCfg.Ipam.Type, nwCfg, true)
-					plugin.invokeIpamDel(resultV6, ipamV6, nwCfg, true)
+					plugin.invokeIpamDel(result, nwCfg.Ipam.Type, *nwCfg, true)
+					plugin.invokeIpamDel(resultV6, ipamV6, *nwCfg, true)
 				}
 			}()
 
-			// resultV6 = &cniTypesCurr.Result{}
-			// ip6Net := net.IPNet{
-			// 	IP:   net.ParseIP("fc00::1"),
-			// 	Mask: net.CIDRMask(64, 128),
-			// }
-			// gw6 := net.ParseIP("fc00::0")
-			// ip6config := &cniTypesCurr.IPConfig{
-			// 	Version: "6",
-			// 	Address: ip6Net,
-			// 	Gateway: gw6,
-			// }
-			// resultV6.IPs = append(resultV6.IPs, ip6config)
 			subnetPrefix = result.IPs[0].Address
 		}
 
@@ -577,30 +573,17 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 			// Network already exists.
 			log.Printf("[cni-net] Found network %v with subnet %v.", networkId, nwInfo.Subnets[0].Prefix.String())
 
-			result, resultV6, err = plugin.invokeIpamAdd(nwCfg, nwInfo, false)
+			result, resultV6, err = plugin.invokeIpamAdd(*nwCfg, nwInfo, false)
 			if err != nil {
 				return err
 			}
 
 			defer func() {
 				if err != nil {
-					plugin.invokeIpamDel(result, nwCfg.Ipam.Type, nwCfg, false)
-					plugin.invokeIpamDel(resultV6, ipamV6, nwCfg, false)
+					plugin.invokeIpamDel(result, nwCfg.Ipam.Type, *nwCfg, false)
+					plugin.invokeIpamDel(resultV6, ipamV6, *nwCfg, false)
 				}
 			}()
-
-			// resultV6 = &cniTypesCurr.Result{}
-			// ip6Net := net.IPNet{
-			// 	IP:   net.ParseIP("fc00::2"),
-			// 	Mask: net.CIDRMask(64, 128),
-			// }
-			// gw6 := net.ParseIP("fc00::0")
-			// ip6config := &cniTypesCurr.IPConfig{
-			// 	Version: "6",
-			// 	Address: ip6Net,
-			// 	Gateway: gw6,
-			// }
-			// resultV6.IPs = append(resultV6.IPs, ip6config)
 		}
 	}
 
@@ -648,10 +631,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW})
 	}
 
-	for _, route := range resultV6.Routes {
-		epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW})
-	}
-
 	if azIpamResult != nil && azIpamResult.IPs != nil {
 		epInfo.InfraVnetIP = azIpamResult.IPs[0].Address
 	}
@@ -889,13 +868,16 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 					err = plugin.Errorf("Failed to release ipv4 address: %v", err)
 				}
 			} else {
+				nwCfgIpv6 := *nwCfg
+				nwCfgIpv6.Ipam.Environment = common.OptEnvironmentIPv6NodeIpam
+				nwCfgIpv6.Ipam.Type = ipamV6
 				if len(nwInfo.Subnets) > 1 {
-					nwCfg.Ipam.Subnet = nwInfo.Subnets[1].Prefix.String()
+					nwCfgIpv6.Ipam.Subnet = nwInfo.Subnets[1].Prefix.String()
 				}
 
 				log.Printf("Releasing ipv6 address :%s pool: %s",
-					nwCfg.Ipam.Address, nwCfg.Ipam.Subnet)
-				if err = plugin.DelegateDel(ipamV6, nwCfg); err != nil {
+					nwCfgIpv6.Ipam.Address, nwCfgIpv6.Ipam.Subnet)
+				if err = plugin.DelegateDel(nwCfgIpv6.Ipam.Type, &nwCfgIpv6); err != nil {
 					log.Printf("Failed to release ipv6 address: %v", err)
 					err = plugin.Errorf("Failed to release ipv6 address: %v", err)
 				}
diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go
index 2d27b443e6..fd336013a0 100644
--- a/ebtables/ebtables.go
+++ b/ebtables/ebtables.go
@@ -22,6 +22,7 @@ const (
 	PreRouting  = "PREROUTING"
 	PostRouting = "POSTROUTING"
 	Brouting    = "BROUTING"
+	Filter      = "FILTER"
 )
 
 // SetSnatForInterface sets a MAC SNAT rule for an interface.
@@ -99,6 +100,17 @@ func SetDnatForIPAddress(interfaceName string, ipAddress net.IP, macAddress net.
 	return runEbCmd(table, action, chain, rule)
 }
 
+// Drop Icmpv6 discovery messages going out of interface
+func DropICMPv6Solicitation(interfaceName string, action string) error {
+	table := Nat
+	chain := Filter
+
+	rule := fmt.Sprintf("-p IPv6 --ip6-proto ipv6-icmp --ip6-icmp-type neighbour-solicitation -o %s -j DROP",
+		interfaceName)
+
+	return runEbCmd(table, action, chain, rule)
+}
+
 // SetEbRule sets any given eb rule
 func SetEbRule(table, action, chain, rule string) error {
 	return runEbCmd(table, action, chain, rule)
diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go
index 12db43e58e..4c9fa3d66f 100644
--- a/network/bridge_networkclient_linux.go
+++ b/network/bridge_networkclient_linux.go
@@ -85,6 +85,12 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error {
 		return err
 	}
 
+	if client.nwInfo.IPV6Mode != "" {
+		if err := ebtables.DropICMPv6Solicitation(client.hostInterfaceName, ebtables.Append); err != nil {
+			return err
+		}
+	}
+
 	// Enable VEPA for host policy enforcement if necessary.
 	if client.nwInfo.Mode == opModeTunnel {
 		log.Printf("[net] Enabling VEPA mode for %v.", client.hostInterfaceName)

From b65b636da840022500275f7dd5de9ac13b5d6352 Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Fri, 3 Apr 2020 15:16:29 -0700
Subject: [PATCH 07/12] fixed issues

---
 ebtables/ebtables.go                  |  7 ++++---
 network/bridge_networkclient_linux.go | 12 ++++++++----
 network/network_linux.go              | 28 +++++++++++++--------------
 3 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go
index fd336013a0..119de9bc6a 100644
--- a/ebtables/ebtables.go
+++ b/ebtables/ebtables.go
@@ -18,11 +18,12 @@ const (
 	// Ebtable tables.
 	Nat    = "nat"
 	Broute = "broute"
+	Filter = "filter"
 	// Ebtable chains.
 	PreRouting  = "PREROUTING"
 	PostRouting = "POSTROUTING"
 	Brouting    = "BROUTING"
-	Filter      = "FILTER"
+	Forward     = "FORWARD"
 )
 
 // SetSnatForInterface sets a MAC SNAT rule for an interface.
@@ -102,8 +103,8 @@ func SetDnatForIPAddress(interfaceName string, ipAddress net.IP, macAddress net.
 
 // Drop Icmpv6 discovery messages going out of interface
 func DropICMPv6Solicitation(interfaceName string, action string) error {
-	table := Nat
-	chain := Filter
+	table := Filter
+	chain := Forward
 
 	rule := fmt.Sprintf("-p IPv6 --ip6-proto ipv6-icmp --ip6-icmp-type neighbour-solicitation -o %s -j DROP",
 		interfaceName)
diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go
index 4c9fa3d66f..85bffb3b54 100644
--- a/network/bridge_networkclient_linux.go
+++ b/network/bridge_networkclient_linux.go
@@ -81,7 +81,7 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error {
 		return err
 	}
 
-	if err := client.addBrouteServiceCidrs(); err != nil {
+	if err := client.setBrouteServiceCidrs(ebtables.Append); err != nil {
 		return err
 	}
 
@@ -107,6 +107,10 @@ func (client *LinuxBridgeClient) DeleteL2Rules(extIf *externalInterface) {
 	ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete)
 	ebtables.SetArpReply(extIf.IPAddresses[0].IP, extIf.MacAddress, ebtables.Delete)
 	ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete)
+	client.setBrouteServiceCidrs(ebtables.Delete)
+	if client.nwInfo.IPV6Mode != "" {
+		ebtables.DropICMPv6Solicitation(extIf.Name, ebtables.Delete)
+	}
 }
 
 func (client *LinuxBridgeClient) SetBridgeMasterToHostInterface() error {
@@ -117,11 +121,11 @@ func (client *LinuxBridgeClient) SetHairpinOnHostInterface(enable bool) error {
 	return netlink.SetLinkHairpin(client.hostInterfaceName, enable)
 }
 
-func (client *LinuxBridgeClient) addBrouteServiceCidrs() error {
+func (client *LinuxBridgeClient) setBrouteServiceCidrs(action string) error {
 	if client.nwInfo.ServiceCidrs != "" {
 		serviceCidrs := strings.Split(client.nwInfo.ServiceCidrs, ",")
 		for _, ipCidrStr := range serviceCidrs {
-			log.Printf("[net] Adding brouting rule for service cidr %s.", ipCidrStr)
+			log.Printf("[net] Setting brouting rule for service cidr %s. Action", ipCidrStr)
 
 			ip, ipNet, _ := net.ParseCIDR(ipCidrStr)
 			svcAddr := net.IPNet{
@@ -129,7 +133,7 @@ func (client *LinuxBridgeClient) addBrouteServiceCidrs() error {
 				Mask: ipNet.Mask,
 			}
 
-			if err := ebtables.SetBrouteAcceptCidr(svcAddr, ebtables.Append); err != nil {
+			if err := ebtables.SetBrouteAcceptCidr(svcAddr, action); err != nil {
 				return err
 			}
 		}
diff --git a/network/network_linux.go b/network/network_linux.go
index 02988e61a0..b344b49f04 100644
--- a/network/network_linux.go
+++ b/network/network_linux.go
@@ -326,18 +326,11 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 	bridge, err := net.InterfaceByName(bridgeName)
 	if err != nil {
 		// Create the bridge.
-		if err := networkClient.CreateBridge(); err != nil {
+		if err = networkClient.CreateBridge(); err != nil {
 			log.Printf("Error while creating bridge %+v", err)
 			return err
 		}
 
-		// On failure, delete the bridge.
-		defer func() {
-			if err != nil {
-				networkClient.DeleteBridge()
-			}
-		}()
-
 		bridge, err = net.InterfaceByName(bridgeName)
 		if err != nil {
 			return err
@@ -347,6 +340,13 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 		log.Printf("[net] Found existing bridge %v.", bridgeName)
 	}
 
+	defer func() {
+		if err != nil {
+			log.Printf("[net] cleanup network")
+			nm.disconnectExternalInterface(extIf, networkClient)
+		}
+	}()
+
 	// Save host IP configuration.
 	err = nm.saveIPConfig(hostIf, extIf)
 	if err != nil {
@@ -356,7 +356,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 	isGreaterOrEqualUbuntu17 := isGreaterOrEqaulUbuntuVersion(ubuntuVersion17)
 	if isGreaterOrEqualUbuntu17 {
 		log.Printf("[net] Saving dns config from %v", extIf.Name)
-		if err := saveDnsConfig(extIf); err != nil {
+		if err = saveDnsConfig(extIf); err != nil {
 			log.Printf("[net] Failed to save dns config: %v", err)
 			return err
 		}
@@ -371,7 +371,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 
 	// Connect the external interface to the bridge.
 	log.Printf("[net] Setting link %v master %v.", hostIf.Name, bridgeName)
-	if err := networkClient.SetBridgeMasterToHostInterface(); err != nil {
+	if err = networkClient.SetBridgeMasterToHostInterface(); err != nil {
 		return err
 	}
 
@@ -398,7 +398,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 	// External interface hairpin on.
 	if !nwInfo.DisableHairpinOnHostInterface {
 		log.Printf("[net] Setting link %v hairpin on.", hostIf.Name)
-		if err := networkClient.SetHairpinOnHostInterface(true); err != nil {
+		if err = networkClient.SetHairpinOnHostInterface(true); err != nil {
 			return err
 		}
 	}
@@ -413,7 +413,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 	if isGreaterOrEqualUbuntu17 {
 		log.Printf("[net] Applying dns config on %v", bridgeName)
 
-		if err := applyDnsConfig(extIf, bridgeName); err != nil {
+		if err = applyDnsConfig(extIf, bridgeName); err != nil {
 			log.Printf("[net] Failed to apply DNS configuration: %v.", err)
 			return err
 		}
@@ -422,12 +422,12 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 	}
 
 	if nwInfo.IPV6Mode != "" {
-		if err := addIpv6NatGateway(nwInfo); err != nil {
+		if err = addIpv6NatGateway(nwInfo); err != nil {
 			log.Errorf("[net] Adding IPv6 Nat Gateway failed:%v", err)
 			return err
 		}
 
-		if err := addIpv6SnatRule(extIf); err != nil {
+		if err = addIpv6SnatRule(extIf); err != nil {
 			log.Errorf("[net] Adding IPv6 Snat Rule failed:%v", err)
 			return err
 		}

From 1cd06b1028883393f6a07b6dd94c5b37d806b22f Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Mon, 6 Apr 2020 17:32:42 -0700
Subject: [PATCH 08/12] fixed unit tests

---
 aitelemetry/telemetrywrapper_test.go | 1 +
 network/network_linux.go             | 3 +--
 telemetry/telemetry_test.go          | 1 +
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/aitelemetry/telemetrywrapper_test.go b/aitelemetry/telemetrywrapper_test.go
index 0c1f3f70e8..6b551155f2 100644
--- a/aitelemetry/telemetrywrapper_test.go
+++ b/aitelemetry/telemetrywrapper_test.go
@@ -62,6 +62,7 @@ func TestMain(m *testing.M) {
 	}
 
 	log.Close()
+	hostAgent.Stop()
 	os.Exit(exitCode)
 }
 
diff --git a/network/network_linux.go b/network/network_linux.go
index b344b49f04..78ceaef671 100644
--- a/network/network_linux.go
+++ b/network/network_linux.go
@@ -91,8 +91,7 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error {
 	if nw.VlanId != 0 {
 		networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name)
 	} else {
-		nwInfo, _ := nm.GetNetworkInfo(nw.Id)
-		networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, nwInfo)
+		networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, NetworkInfo{})
 	}
 
 	// Disconnect the interface if this was the last network using it.
diff --git a/telemetry/telemetry_test.go b/telemetry/telemetry_test.go
index 463ff2301c..56b8d5ba14 100644
--- a/telemetry/telemetry_test.go
+++ b/telemetry/telemetry_test.go
@@ -124,6 +124,7 @@ func TestMain(m *testing.M) {
 	}
 
 	tb.Cleanup(FdName)
+	ipamAgent.Stop()
 	os.Exit(exitCode)
 }
 

From 1a49cf513b60777fdfd154131ddf9dceee96779a Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Tue, 7 Apr 2020 19:00:34 -0700
Subject: [PATCH 09/12] fix nil dereference

---
 cni/network/network.go | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/cni/network/network.go b/cni/network/network.go
index 571d45121c..287e855c30 100644
--- a/cni/network/network.go
+++ b/cni/network/network.go
@@ -622,8 +622,10 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address)
 	}
 
-	for _, ipconfig := range resultV6.IPs {
-		epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address)
+	if resultV6 != nil {
+		for _, ipconfig := range resultV6.IPs {
+			epInfo.IPAddresses = append(epInfo.IPAddresses, ipconfig.Address)
+		}
 	}
 
 	// Populate routes.

From 8b9bcc164712d747a43edf0a82df3f3ce5a2334f Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Wed, 8 Apr 2020 21:49:07 -0700
Subject: [PATCH 10/12] addressed comments

---
 cni/network/network.go   | 11 ++++-------
 ebtables/ebtables.go     |  2 ++
 network/network_linux.go | 16 ++++++++++------
 telemetry/constants.go   |  9 ++++-----
 4 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/cni/network/network.go b/cni/network/network.go
index 287e855c30..5a2e0ed7af 100644
--- a/cni/network/network.go
+++ b/cni/network/network.go
@@ -803,6 +803,7 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 		log.Printf("[cni-net] Failed to get POD info due to error: %v", err)
 	}
 
+	plugin.setCNIReportDetails(nwCfg, CNI_DEL, "")
 	iptables.DisableIPTableLock = nwCfg.DisableIPTableLock
 
 	if nwCfg.MultiTenancy {
@@ -834,13 +835,6 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 	}
 
 	defer func() {
-		msg = fmt.Sprintf("CNI DEL succeeded : Released ip %+v podname %v namespace %v", nwCfg.Ipam.Address, k8sPodName, k8sNamespace)
-		if err != nil {
-			msg = fmt.Sprintf("CNI DEL failed : ip %+v podname %v namespace %v", nwCfg.Ipam.Address, k8sPodName, k8sNamespace)
-		}
-
-		plugin.setCNIReportDetails(nwCfg, CNI_DEL, msg)
-
 		operationTimeMs := time.Since(startTime).Milliseconds()
 		cniMetric.Metric = aitelemetry.Metric{
 			Name:             telemetry.CNIDelTimeMetricStr,
@@ -895,6 +889,9 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 		}
 	}
 
+	msg = fmt.Sprintf("CNI DEL succeeded : Released ip %+v podname %v namespace %v", nwCfg.Ipam.Address, k8sPodName, k8sNamespace)
+	plugin.setCNIReportDetails(nwCfg, CNI_DEL, msg)
+
 	return err
 }
 
diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go
index 119de9bc6a..2a64cf45ec 100644
--- a/ebtables/ebtables.go
+++ b/ebtables/ebtables.go
@@ -153,6 +153,8 @@ func GetEbtableRules(tableName, chainName string) ([]string, error) {
 	return rules, nil
 }
 
+// SetBrouteAcceptCidr - broute chain MAC redirect rule. Will change mac target address to bridge port
+// that receives the frame.
 func SetBrouteAcceptCidr(ipNet net.IPNet, action string) error {
 	protocol := "IPv4"
 	dst := "--ip-dst"
diff --git a/network/network_linux.go b/network/network_linux.go
index 78ceaef671..d092df4e7d 100644
--- a/network/network_linux.go
+++ b/network/network_linux.go
@@ -426,7 +426,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 			return err
 		}
 
-		if err = addIpv6SnatRule(extIf); err != nil {
+		if err = addIpv6SnatRule(extIf, nwInfo.IPV6Mode); err != nil {
 			log.Errorf("[net] Adding IPv6 Snat Rule failed:%v", err)
 			return err
 		}
@@ -466,6 +466,7 @@ func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface,
 	log.Printf("[net] Disconnected interface %v.", extIf.Name)
 }
 
+// Add ipv6 nat gateway IP on bridge
 func addIpv6NatGateway(nwInfo *NetworkInfo) error {
 	if nwInfo.IPV6Mode == IPV6Nat {
 		log.Printf("[net] Adding ipv6 nat gateway on azure bridge")
@@ -483,11 +484,14 @@ func addIpv6NatGateway(nwInfo *NetworkInfo) error {
 	return nil
 }
 
-func addIpv6SnatRule(extIf *externalInterface) error {
-	log.Printf("[net] Adding ipv6 snat rule")
-	for _, ipAddr := range extIf.IPAddresses {
-		if ipAddr.IP.To4() == nil {
-			return epcommon.AddSnatRule("", ipAddr.IP)
+// snat ipv6 traffic to secondary ipv6 ip before leaving VM
+func addIpv6SnatRule(extIf *externalInterface, ipv6Mode string) error {
+	if ipv6Mode == IPV6Nat {
+		log.Printf("[net] Adding ipv6 snat rule")
+		for _, ipAddr := range extIf.IPAddresses {
+			if ipAddr.IP.To4() == nil {
+				return epcommon.AddSnatRule("", ipAddr.IP)
+			}
 		}
 	}
 
diff --git a/telemetry/constants.go b/telemetry/constants.go
index 0d2e2215b2..9fadc585d6 100644
--- a/telemetry/constants.go
+++ b/telemetry/constants.go
@@ -5,11 +5,10 @@ package telemetry
 const (
 
 	// Metric Names
-	CNIExecutimeMetricStr   = "CNIExecutionTimeMs"
-	CNIV6ExecutimeMetricStr = "CNIV6ExecutionTimeMs"
-	CNIAddTimeMetricStr     = "CNIAddTimeMs"
-	CNIDelTimeMetricStr     = "CNIDelTimeMs"
-	CNIUpdateTimeMetricStr  = "CNIUpdateTimeMs"
+	CNIExecutimeMetricStr  = "CNIExecutionTimeMs"
+	CNIAddTimeMetricStr    = "CNIAddTimeMs"
+	CNIDelTimeMetricStr    = "CNIDelTimeMs"
+	CNIUpdateTimeMetricStr = "CNIUpdateTimeMs"
 
 	// Dimension Names
 	ContextStr        = "Context"

From 9ff3c9c6db9aeefb0ec25b1c8bf03614c16f8c36 Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Wed, 15 Apr 2020 01:55:38 -0700
Subject: [PATCH 11/12] ipv6 fixes and changes

---
 cni/netconfig.go                       |  1 +
 cni/network/network.go                 | 16 +++--
 ebtables/ebtables.go                   | 34 +++++++++--
 iptables/iptables.go                   |  1 +
 netlink/link.go                        | 13 +++-
 netlink/netlink_test.go                |  4 +-
 network/bridge_endpointclient_linux.go | 84 +++++++++++++++++++++++++-
 network/bridge_networkclient_linux.go  | 56 +++++++++++------
 network/endpoint.go                    |  3 +
 network/endpoint_linux.go              |  8 ++-
 network/epcommon/endpoint_common.go    | 16 ++++-
 network/network_linux.go               | 41 +++++++------
 network/ovssnat/ovssnat.go             |  8 +--
 13 files changed, 222 insertions(+), 63 deletions(-)

diff --git a/cni/netconfig.go b/cni/netconfig.go
index 02c6f2455f..3faeef8451 100644
--- a/cni/netconfig.go
+++ b/cni/netconfig.go
@@ -53,6 +53,7 @@ type NetworkConfig struct {
 	InfraVnetAddressSpace         string   `json:"infraVnetAddressSpace,omitempty"`
 	IPV6Mode                      string   `json:"ipv6Mode,omitempty"`
 	ServiceCidrs                  string   `json:"serviceCidrs,omitempty"`
+	VnetCidrs                     string   `json:"vnetCidrs,omitempty"`
 	PodNamespaceForDualNetwork    []string `json:"podNamespaceForDualNetwork,omitempty"`
 	IPsToRouteViaHost             []string `json:"ipsToRouteViaHost,omitempty"`
 	MultiTenancy                  bool     `json:"multiTenancy,omitempty"`
diff --git a/cni/network/network.go b/cni/network/network.go
index 5a2e0ed7af..b6b3829abd 100644
--- a/cni/network/network.go
+++ b/cni/network/network.go
@@ -255,7 +255,8 @@ func (plugin *netPlugin) invokeIpamDel(
 			nwCfg.Ipam.Type = ipamType
 		}
 
-		nwCfg.Ipam.Subnet = result.IPs[0].Address.String()
+		_, subnet, _ := net.ParseCIDR(result.IPs[0].Address.String())
+		nwCfg.Ipam.Subnet = subnet.String()
 		nwCfg.Ipam.Address = result.IPs[0].Address.IP.String()
 		plugin.DelegateDel(ipamType, &nwCfg)
 
@@ -296,14 +297,15 @@ func (plugin *netPlugin) invokeIpamAdd(
 	}()
 
 	if nwCfg.IPV6Mode != "" {
-		nwCfg.Ipam.Environment = common.OptEnvironmentIPv6NodeIpam
-		nwCfg.Ipam.Type = ipamV6
+		nwCfg6 := nwCfg
+		nwCfg6.Ipam.Environment = common.OptEnvironmentIPv6NodeIpam
+		nwCfg6.Ipam.Type = ipamV6
 
 		if len(nwInfo.Subnets) > 1 {
-			nwCfg.Ipam.Subnet = nwInfo.Subnets[1].Prefix.String()
+			nwCfg6.Ipam.Subnet = nwInfo.Subnets[1].Prefix.String()
 		}
 
-		resultV6, err = plugin.DelegateAdd(ipamV6, &nwCfg)
+		resultV6, err = plugin.DelegateAdd(ipamV6, &nwCfg6)
 		if err != nil {
 			err = plugin.Errorf("Failed to allocate v6 pool: %v", err)
 		}
@@ -540,7 +542,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 			Mode:         nwCfg.Mode,
 			MasterIfName: masterIfName,
 			Subnets: []network.SubnetInfo{
-				network.SubnetInfo{
+				{
 					Family:  platform.AfINET,
 					Prefix:  subnetPrefix,
 					Gateway: gateway,
@@ -610,6 +612,8 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		PODNameSpace:       k8sNamespace,
 		SkipHotAttachEp:    false, // Hot attach at the time of endpoint creation
 		IPV6Mode:           nwCfg.IPV6Mode,
+		VnetCidrs:          nwCfg.VnetCidrs,
+		ServiceCidrs:       nwCfg.ServiceCidrs,
 	}
 
 	epPolicies := getPoliciesFromRuntimeCfg(nwCfg)
diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go
index 2a64cf45ec..02d0c4cc27 100644
--- a/ebtables/ebtables.go
+++ b/ebtables/ebtables.go
@@ -24,6 +24,12 @@ const (
 	PostRouting = "POSTROUTING"
 	Brouting    = "BROUTING"
 	Forward     = "FORWARD"
+	// Ebtable Protocols
+	IPV4 = "IPv4"
+	IPV6 = "IPv6"
+	// Ebtable Targets
+	Accept         = "ACCEPT"
+	RedirectAccept = "redirect --redirect-target ACCEPT"
 )
 
 // SetSnatForInterface sets a MAC SNAT rule for an interface.
@@ -155,21 +161,37 @@ func GetEbtableRules(tableName, chainName string) ([]string, error) {
 
 // SetBrouteAcceptCidr - broute chain MAC redirect rule. Will change mac target address to bridge port
 // that receives the frame.
-func SetBrouteAcceptCidr(ipNet net.IPNet, action string) error {
-	protocol := "IPv4"
+func SetBrouteAcceptByCidr(ipNet *net.IPNet, protocol, action, target string) error {
 	dst := "--ip-dst"
-	if ipNet.IP.To4() == nil {
-		protocol = "IPv6"
+	if protocol == IPV6 {
 		dst = "--ip6-dst"
 	}
 
+	var rule string
 	table := Broute
 	chain := Brouting
-	rule := fmt.Sprintf("-p %s %s %s -j redirect --redirect-target ACCEPT",
-		protocol, dst, ipNet.String())
+	if ipNet != nil {
+		rule = fmt.Sprintf("-p %s %s %s -j %s",
+			protocol, dst, ipNet.String(), target)
+	} else {
+		rule = fmt.Sprintf("-p %s -j %s",
+			protocol, target)
+	}
 
 	return runEbCmd(table, action, chain, rule)
+}
 
+func SetBrouteAcceptByInterface(ifName string, protocol, action, target string) error {
+	var rule string
+	table := Broute
+	chain := Brouting
+	if protocol == "" {
+		rule = fmt.Sprintf("-i %s -j %s", ifName, target)
+	} else {
+		rule = fmt.Sprintf("-p %s -i %s -j %s", protocol, ifName, target)
+	}
+
+	return runEbCmd(table, action, chain, rule)
 }
 
 // EbTableRuleExists checks if eb rule exists in table and chain.
diff --git a/iptables/iptables.go b/iptables/iptables.go
index 6f070b7c93..98c5ee7bdb 100644
--- a/iptables/iptables.go
+++ b/iptables/iptables.go
@@ -28,6 +28,7 @@ const (
 const (
 	Filter = "filter"
 	Nat    = "nat"
+	Mangle = "mangle"
 )
 
 // target
diff --git a/netlink/link.go b/netlink/link.go
index dfbdbaaa48..a1f5e9b6c4 100644
--- a/netlink/link.go
+++ b/netlink/link.go
@@ -391,7 +391,7 @@ func SetLinkHairpin(bridgeName string, on bool) error {
 }
 
 // AddOrRemoveStaticArp sets/removes static arp entry based on mode
-func AddOrRemoveStaticArp(mode int, name string, ipaddr net.IP, mac net.HardwareAddr) error {
+func AddOrRemoveStaticArp(mode int, name string, ipaddr net.IP, mac net.HardwareAddr, isProxy bool) error {
 	s, err := getSocket()
 	if err != nil {
 		return err
@@ -413,13 +413,22 @@ func AddOrRemoveStaticArp(mode int, name string, ipaddr net.IP, mac net.Hardware
 	}
 
 	msg := neighMsg{
-		Family: uint8(unix.AF_INET),
+		Family: uint8(GetIpAddressFamily(ipaddr)),
 		Index:  uint32(iface.Index),
 		State:  uint16(state),
 	}
+
+	if isProxy {
+		msg.Flags = msg.Flags | NTF_PROXY
+	}
+
 	req.addPayload(&msg)
 
 	ipData := ipaddr.To4()
+	if ipData == nil {
+		ipData = ipaddr.To16()
+	}
+
 	dstData := newRtAttr(NDA_DST, ipData)
 	req.addPayload(dstData)
 
diff --git a/netlink/netlink_test.go b/netlink/netlink_test.go
index 622db1d279..c4630b7eff 100644
--- a/netlink/netlink_test.go
+++ b/netlink/netlink_test.go
@@ -244,12 +244,12 @@ func TestAddRemoveStaticArp(t *testing.T) {
 	ip := net.ParseIP("192.168.0.2")
 	mac, _ := net.ParseMAC("aa:b3:4d:5e:e2:4a")
 
-	err = AddOrRemoveStaticArp(ADD, ifName, ip, mac)
+	err = AddOrRemoveStaticArp(ADD, ifName, ip, mac, false)
 	if err != nil {
 		t.Errorf("ret val %v", err)
 	}
 
-	err = AddOrRemoveStaticArp(REMOVE, ifName, ip, mac)
+	err = AddOrRemoveStaticArp(REMOVE, ifName, ip, mac, false)
 	if err != nil {
 		t.Errorf("ret val %v", err)
 	}
diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go
index 3137303090..05397af117 100644
--- a/network/bridge_endpointclient_linux.go
+++ b/network/bridge_endpointclient_linux.go
@@ -10,6 +10,12 @@ import (
 	"github.com/Azure/azure-container-networking/network/epcommon"
 )
 
+const (
+	defaultV6VnetCidr = "2001:1234:5678:9abc::/64"
+	defaultV6HostGw   = "fe80::1234:5678:9abc"
+	defaultHostGwMac  = "12:34:56:78:9a:bc"
+)
+
 type LinuxBridgeEndpointClient struct {
 	bridgeName        string
 	hostPrimaryIfName string
@@ -17,6 +23,7 @@ type LinuxBridgeEndpointClient struct {
 	containerVethName string
 	hostPrimaryMac    net.HardwareAddr
 	containerMac      net.HardwareAddr
+	hostIPAddresses   []*net.IPNet
 	mode              string
 }
 
@@ -33,9 +40,14 @@ func NewLinuxBridgeEndpointClient(
 		hostVethName:      hostVethName,
 		containerVethName: containerVethName,
 		hostPrimaryMac:    extIf.MacAddress,
+		hostIPAddresses:   []*net.IPNet{},
 		mode:              mode,
 	}
 
+	for _, ipAddr := range extIf.IPAddresses {
+		client.hostIPAddresses = append(client.hostIPAddresses, ipAddr)
+	}
+
 	return client
 }
 
@@ -78,7 +90,7 @@ func (client *LinuxBridgeEndpointClient) AddEndpointRules(epInfo *EndpointInfo)
 
 		if client.mode != opModeTunnel && ipAddr.IP.To4() != nil {
 			log.Printf("[net] Adding static arp for IP address %v and MAC %v in VM", ipAddr.String(), client.containerMac.String())
-			if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.bridgeName, ipAddr.IP, client.containerMac); err != nil {
+			if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.bridgeName, ipAddr.IP, client.containerMac, false); err != nil {
 				log.Printf("Failed setting arp in vm: %v", err)
 			}
 		}
@@ -116,7 +128,7 @@ func (client *LinuxBridgeEndpointClient) DeleteEndpointRules(ep *endpoint) {
 
 		if client.mode != opModeTunnel && ipAddr.IP.To4() != nil {
 			log.Printf("[net] Removing static arp for IP address %v and MAC %v from VM", ipAddr.String(), ep.MacAddress.String())
-			err := netlink.AddOrRemoveStaticArp(netlink.REMOVE, client.bridgeName, ipAddr.IP, ep.MacAddress)
+			err := netlink.AddOrRemoveStaticArp(netlink.REMOVE, client.bridgeName, ipAddr.IP, ep.MacAddress, false)
 			if err != nil {
 				log.Printf("Failed removing arp from vm: %v", err)
 			}
@@ -175,6 +187,14 @@ func (client *LinuxBridgeEndpointClient) ConfigureContainerInterfacesAndRoutes(e
 		return err
 	}
 
+	if err := client.setupIPV6Routes(epInfo); err != nil {
+		return err
+	}
+
+	if err := client.setIPV6NeighEntry(epInfo); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -218,3 +238,63 @@ func addRuleToRouteViaHost(epInfo *EndpointInfo) error {
 
 	return nil
 }
+
+func (client *LinuxBridgeEndpointClient) setupIPV6Routes(epInfo *EndpointInfo) error {
+	if epInfo.IPV6Mode != "" {
+		if epInfo.VnetCidrs == "" {
+			epInfo.VnetCidrs = defaultV6VnetCidr
+		}
+
+		routes := []RouteInfo{}
+		_, v6IpNet, _ := net.ParseCIDR(epInfo.VnetCidrs)
+		v6Gw := net.ParseIP(defaultV6HostGw)
+		vnetRoute := RouteInfo{
+			Dst:      *v6IpNet,
+			Gw:       v6Gw,
+			Priority: 101,
+		}
+
+		var vmV6Route RouteInfo
+
+		for _, ipAddr := range client.hostIPAddresses {
+			if ipAddr.IP.To4() == nil {
+				vmV6Route = RouteInfo{
+					Dst:      *ipAddr,
+					Priority: 100,
+				}
+			}
+		}
+
+		_, defIPNet, _ := net.ParseCIDR("::/0")
+		defaultV6Route := RouteInfo{
+			Dst: *defIPNet,
+			Gw:  v6Gw,
+		}
+
+		routes = append(routes, vnetRoute)
+		routes = append(routes, vmV6Route)
+		routes = append(routes, defaultV6Route)
+
+		log.Printf("[net] Adding ipv6 routes in container %+v", routes)
+		if err := addRoutes(client.containerVethName, routes); err != nil {
+			return nil
+		}
+	}
+
+	return nil
+}
+
+func (client *LinuxBridgeEndpointClient) setIPV6NeighEntry(epInfo *EndpointInfo) error {
+	if epInfo.IPV6Mode != "" {
+		log.Printf("[net] Add neigh entry for host gw ip")
+		hardwareAddr, _ := net.ParseMAC(defaultHostGwMac)
+		hostGwIp := net.ParseIP(defaultV6HostGw)
+		if err := netlink.AddOrRemoveStaticArp(netlink.ADD, client.containerVethName,
+			hostGwIp, hardwareAddr, false); err != nil {
+			log.Printf("Failed setting neigh entry in container: %v", err)
+			return err
+		}
+	}
+
+	return nil
+}
diff --git a/network/bridge_networkclient_linux.go b/network/bridge_networkclient_linux.go
index 85bffb3b54..1c72f3b848 100644
--- a/network/bridge_networkclient_linux.go
+++ b/network/bridge_networkclient_linux.go
@@ -2,11 +2,15 @@ package network
 
 import (
 	"net"
-	"strings"
 
 	"github.com/Azure/azure-container-networking/ebtables"
 	"github.com/Azure/azure-container-networking/log"
 	"github.com/Azure/azure-container-networking/netlink"
+	"github.com/Azure/azure-container-networking/network/epcommon"
+)
+
+const (
+	multicastSolicitPrefix = "ff02::1:ff00:0/104"
 )
 
 type LinuxBridgeClient struct {
@@ -81,14 +85,28 @@ func (client *LinuxBridgeClient) AddL2Rules(extIf *externalInterface) error {
 		return err
 	}
 
-	if err := client.setBrouteServiceCidrs(ebtables.Append); err != nil {
-		return err
-	}
-
 	if client.nwInfo.IPV6Mode != "" {
+		// for ipv6 node cidr set broute accept
+		if err := ebtables.SetBrouteAcceptByCidr(&client.nwInfo.Subnets[1].Prefix, ebtables.IPV6, ebtables.Append, ebtables.Accept); err != nil {
+			return err
+		}
+
+		_, mIpNet, _ := net.ParseCIDR(multicastSolicitPrefix)
+		if err := ebtables.SetBrouteAcceptByCidr(mIpNet, ebtables.IPV6, ebtables.Append, ebtables.Accept); err != nil {
+			return err
+		}
+
 		if err := ebtables.DropICMPv6Solicitation(client.hostInterfaceName, ebtables.Append); err != nil {
 			return err
 		}
+
+		if err := client.setBrouteRedirect(ebtables.Append); err != nil {
+			return err
+		}
+
+		if err := epcommon.EnableIPV6Forwarding(); err != nil {
+			return err
+		}
 	}
 
 	// Enable VEPA for host policy enforcement if necessary.
@@ -107,8 +125,13 @@ func (client *LinuxBridgeClient) DeleteL2Rules(extIf *externalInterface) {
 	ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete)
 	ebtables.SetArpReply(extIf.IPAddresses[0].IP, extIf.MacAddress, ebtables.Delete)
 	ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete)
-	client.setBrouteServiceCidrs(ebtables.Delete)
 	if client.nwInfo.IPV6Mode != "" {
+		if len(extIf.IPAddresses) > 1 {
+			ebtables.SetBrouteAcceptByCidr(extIf.IPAddresses[1], ebtables.IPV6, ebtables.Delete, ebtables.Accept)
+		}
+		_, mIpNet, _ := net.ParseCIDR(multicastSolicitPrefix)
+		ebtables.SetBrouteAcceptByCidr(mIpNet, ebtables.IPV6, ebtables.Delete, ebtables.Accept)
+		client.setBrouteRedirect(ebtables.Delete)
 		ebtables.DropICMPv6Solicitation(extIf.Name, ebtables.Delete)
 	}
 }
@@ -121,21 +144,14 @@ func (client *LinuxBridgeClient) SetHairpinOnHostInterface(enable bool) error {
 	return netlink.SetLinkHairpin(client.hostInterfaceName, enable)
 }
 
-func (client *LinuxBridgeClient) setBrouteServiceCidrs(action string) error {
+func (client *LinuxBridgeClient) setBrouteRedirect(action string) error {
 	if client.nwInfo.ServiceCidrs != "" {
-		serviceCidrs := strings.Split(client.nwInfo.ServiceCidrs, ",")
-		for _, ipCidrStr := range serviceCidrs {
-			log.Printf("[net] Setting brouting rule for service cidr %s. Action", ipCidrStr)
-
-			ip, ipNet, _ := net.ParseCIDR(ipCidrStr)
-			svcAddr := net.IPNet{
-				IP:   ip,
-				Mask: ipNet.Mask,
-			}
-
-			if err := ebtables.SetBrouteAcceptCidr(svcAddr, action); err != nil {
-				return err
-			}
+		if err := ebtables.SetBrouteAcceptByCidr(nil, ebtables.IPV4, ebtables.Append, ebtables.RedirectAccept); err != nil {
+			return err
+		}
+
+		if err := ebtables.SetBrouteAcceptByCidr(nil, ebtables.IPV6, ebtables.Append, ebtables.RedirectAccept); err != nil {
+			return err
 		}
 	}
 
diff --git a/network/endpoint.go b/network/endpoint.go
index 9161477829..bf4d19cad1 100644
--- a/network/endpoint.go
+++ b/network/endpoint.go
@@ -73,6 +73,8 @@ type EndpointInfo struct {
 	InfraVnetAddressSpace    string
 	SkipHotAttachEp          bool
 	IPV6Mode                 string
+	VnetCidrs                string
+	ServiceCidrs             string
 }
 
 // RouteInfo contains information about an IP route.
@@ -83,6 +85,7 @@ type RouteInfo struct {
 	Protocol int
 	DevName  string
 	Scope    int
+	Priority int
 }
 
 // NewEndpoint creates a new endpoint in the network.
diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go
index 453691bffb..5ea6d9345d 100644
--- a/network/endpoint_linux.go
+++ b/network/endpoint_linux.go
@@ -258,11 +258,17 @@ func addRoutes(interfaceName string, routes []RouteInfo) error {
 			ifIndex = interfaceIf.Index
 		}
 
+		family := netlink.GetIpAddressFamily(route.Gw)
+		if route.Gw == nil {
+			family = netlink.GetIpAddressFamily(route.Dst.IP)
+		}
+
 		nlRoute := &netlink.Route{
-			Family:    netlink.GetIpAddressFamily(route.Gw),
+			Family:    family,
 			Dst:       &route.Dst,
 			Gw:        route.Gw,
 			LinkIndex: ifIndex,
+			Priority:  route.Priority,
 		}
 
 		if err := netlink.AddIpRoute(nlRoute); err != nil {
diff --git a/network/epcommon/endpoint_common.go b/network/epcommon/endpoint_common.go
index 74f8f69622..edd0cae173 100644
--- a/network/epcommon/endpoint_common.go
+++ b/network/epcommon/endpoint_common.go
@@ -28,8 +28,9 @@ RFC for Link Local Addresses: https://tools.ietf.org/html/rfc3927
 */
 
 const (
-	enableIPForwardCmd = "sysctl -w net.ipv4.ip_forward=1"
-	enableIPV6Cmd      = "sysctl -w net.ipv6.conf.all.disable_ipv6=%d"
+	enableIPForwardCmd   = "sysctl -w net.ipv4.ip_forward=1"
+	enableIPV6Cmd        = "sysctl -w net.ipv6.conf.all.disable_ipv6=%d"
+	enableIPV6ForwardCmd = "sysctl -w net.ipv6.conf.all.forwarding=1"
 )
 
 func getPrivateIPSpace() []string {
@@ -194,6 +195,17 @@ func EnableIPForwarding(ifName string) error {
 	return nil
 }
 
+func EnableIPV6Forwarding() error {
+	cmd := fmt.Sprintf(enableIPV6ForwardCmd)
+	_, err := platform.ExecuteCommand(cmd)
+	if err != nil {
+		log.Printf("[net] Enable ipv6 forwarding failed with: %v", err)
+		return err
+	}
+
+	return nil
+}
+
 // This functions enables/disables ipv6 setting based on enable parameter passed.
 func UpdateIPV6Setting(disable int) error {
 	// sysctl -w net.ipv6.conf.all.disable_ipv6=0/1
diff --git a/network/network_linux.go b/network/network_linux.go
index d092df4e7d..7c1ccf6526 100644
--- a/network/network_linux.go
+++ b/network/network_linux.go
@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/Azure/azure-container-networking/iptables"
 	"github.com/Azure/azure-container-networking/log"
 	"github.com/Azure/azure-container-networking/netlink"
 	"github.com/Azure/azure-container-networking/network/epcommon"
@@ -420,16 +421,22 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 		log.Printf("[net] Applied dns config %v on %v", extIf.DNSInfo, bridgeName)
 	}
 
-	if nwInfo.IPV6Mode != "" {
+	if nwInfo.IPV6Mode == IPV6Nat {
 		if err = addIpv6NatGateway(nwInfo); err != nil {
 			log.Errorf("[net] Adding IPv6 Nat Gateway failed:%v", err)
 			return err
 		}
 
-		if err = addIpv6SnatRule(extIf, nwInfo.IPV6Mode); err != nil {
+		if err = addIpv6SnatRule(extIf, nwInfo); err != nil {
 			log.Errorf("[net] Adding IPv6 Snat Rule failed:%v", err)
 			return err
 		}
+
+		// unmark packet if set by kube-proxy
+		if err = iptables.InsertIptableRule(iptables.V6, iptables.Mangle, iptables.Postrouting, "", "MARK --set-mark 0x0"); err != nil {
+			log.Errorf("[net] Adding Iptable mangle rule failed:%v", err)
+			return err
+		}
 	}
 
 	extIf.BridgeName = bridgeName
@@ -468,16 +475,14 @@ func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface,
 
 // Add ipv6 nat gateway IP on bridge
 func addIpv6NatGateway(nwInfo *NetworkInfo) error {
-	if nwInfo.IPV6Mode == IPV6Nat {
-		log.Printf("[net] Adding ipv6 nat gateway on azure bridge")
-		for _, subnetInfo := range nwInfo.Subnets {
-			if subnetInfo.Family == platform.AfINET6 {
-				ipAddr := []net.IPNet{{
-					IP:   subnetInfo.Gateway,
-					Mask: subnetInfo.Prefix.Mask,
-				}}
-				return epcommon.AssignIPToInterface(nwInfo.BridgeName, ipAddr)
-			}
+	log.Printf("[net] Adding ipv6 nat gateway on azure bridge")
+	for _, subnetInfo := range nwInfo.Subnets {
+		if subnetInfo.Family == platform.AfINET6 {
+			ipAddr := []net.IPNet{{
+				IP:   subnetInfo.Gateway,
+				Mask: subnetInfo.Prefix.Mask,
+			}}
+			return epcommon.AssignIPToInterface(nwInfo.BridgeName, ipAddr)
 		}
 	}
 
@@ -485,12 +490,12 @@ func addIpv6NatGateway(nwInfo *NetworkInfo) error {
 }
 
 // snat ipv6 traffic to secondary ipv6 ip before leaving VM
-func addIpv6SnatRule(extIf *externalInterface, ipv6Mode string) error {
-	if ipv6Mode == IPV6Nat {
-		log.Printf("[net] Adding ipv6 snat rule")
-		for _, ipAddr := range extIf.IPAddresses {
-			if ipAddr.IP.To4() == nil {
-				return epcommon.AddSnatRule("", ipAddr.IP)
+func addIpv6SnatRule(extIf *externalInterface, nwInfo *NetworkInfo) error {
+	log.Printf("[net] Adding ipv6 snat rule")
+	for _, ipAddr := range extIf.IPAddresses {
+		if ipAddr.IP.To4() == nil {
+			if err := epcommon.AddSnatRule("", ipAddr.IP); err != nil {
+				return err
 			}
 		}
 	}
diff --git a/network/ovssnat/ovssnat.go b/network/ovssnat/ovssnat.go
index 8e908750c2..9891189597 100644
--- a/network/ovssnat/ovssnat.go
+++ b/network/ovssnat/ovssnat.go
@@ -184,7 +184,7 @@ func (client *OVSSnatClient) AllowInboundFromHostToNC() error {
 
 	// Add static arp entry for localIP to prevent arp going out of VM
 	log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String())
-	err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr)
+	err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr, false)
 	if err != nil {
 		log.Printf("AllowInboundFromHostToNC: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err)
 	}
@@ -204,7 +204,7 @@ func (client *OVSSnatClient) DeleteInboundFromHostToNC() error {
 
 	// Remove static arp entry added for container local IP
 	log.Printf("Removing static arp entry for ip %s ", containerIP)
-	err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil)
+	err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil, false)
 	if err != nil {
 		log.Printf("AllowInboundFromHostToNC: Error removing static arp entry for ip %s: %v", containerIP, err)
 	}
@@ -262,7 +262,7 @@ func (client *OVSSnatClient) AllowInboundFromNCToHost() error {
 
 	// Add static arp entry for localIP to prevent arp going out of VM
 	log.Printf("Adding static arp entry for ip %s mac %s", containerIP, snatContainerVeth.HardwareAddr.String())
-	err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr)
+	err = netlink.AddOrRemoveStaticArp(netlink.ADD, SnatBridgeName, containerIP, snatContainerVeth.HardwareAddr, false)
 	if err != nil {
 		log.Printf("AllowInboundFromNCToHost: Error adding static arp entry for ip %s mac %s: %v", containerIP, snatContainerVeth.HardwareAddr.String(), err)
 	}
@@ -282,7 +282,7 @@ func (client *OVSSnatClient) DeleteInboundFromNCToHost() error {
 
 	// Remove static arp entry added for container local IP
 	log.Printf("Removing static arp entry for ip %s ", containerIP)
-	err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil)
+	err = netlink.AddOrRemoveStaticArp(netlink.REMOVE, SnatBridgeName, containerIP, nil, false)
 	if err != nil {
 		log.Printf("DeleteInboundFromNCToHost: Error removing static arp entry for ip %s: %v", containerIP, err)
 	}

From aaff1a84b815c4c989edeec00e0c00919c1329b3 Mon Sep 17 00:00:00 2001
From: Tamilmani Manoharan <tamanoha@microsoft.com>
Date: Wed, 15 Apr 2020 17:33:58 -0700
Subject: [PATCH 12/12] addressed comments

---
 netlink/link.go                     | 1 +
 network/epcommon/endpoint_common.go | 4 ++--
 network/network_linux.go            | 4 +++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/netlink/link.go b/netlink/link.go
index a1f5e9b6c4..3c899a42e7 100644
--- a/netlink/link.go
+++ b/netlink/link.go
@@ -418,6 +418,7 @@ func AddOrRemoveStaticArp(mode int, name string, ipaddr net.IP, mac net.Hardware
 		State:  uint16(state),
 	}
 
+	// NTF_PROXY is for setting neighbor proxy
 	if isProxy {
 		msg.Flags = msg.Flags | NTF_PROXY
 	}
diff --git a/network/epcommon/endpoint_common.go b/network/epcommon/endpoint_common.go
index edd0cae173..923f918447 100644
--- a/network/epcommon/endpoint_common.go
+++ b/network/epcommon/endpoint_common.go
@@ -29,7 +29,7 @@ RFC for Link Local Addresses: https://tools.ietf.org/html/rfc3927
 
 const (
 	enableIPForwardCmd   = "sysctl -w net.ipv4.ip_forward=1"
-	enableIPV6Cmd        = "sysctl -w net.ipv6.conf.all.disable_ipv6=%d"
+	toggleIPV6Cmd        = "sysctl -w net.ipv6.conf.all.disable_ipv6=%d"
 	enableIPV6ForwardCmd = "sysctl -w net.ipv6.conf.all.forwarding=1"
 )
 
@@ -209,7 +209,7 @@ func EnableIPV6Forwarding() error {
 // This functions enables/disables ipv6 setting based on enable parameter passed.
 func UpdateIPV6Setting(disable int) error {
 	// sysctl -w net.ipv6.conf.all.disable_ipv6=0/1
-	cmd := fmt.Sprintf(enableIPV6Cmd, disable)
+	cmd := fmt.Sprintf(toggleIPV6Cmd, disable)
 	_, err := platform.ExecuteCommand(cmd)
 	if err != nil {
 		log.Printf("[net] Update IPV6 Setting failed with: %v", err)
diff --git a/network/network_linux.go b/network/network_linux.go
index 7c1ccf6526..be0dfad817 100644
--- a/network/network_linux.go
+++ b/network/network_linux.go
@@ -422,6 +422,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 	}
 
 	if nwInfo.IPV6Mode == IPV6Nat {
+		// adds pod cidr gateway ip to bridge
 		if err = addIpv6NatGateway(nwInfo); err != nil {
 			log.Errorf("[net] Adding IPv6 Nat Gateway failed:%v", err)
 			return err
@@ -432,7 +433,8 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI
 			return err
 		}
 
-		// unmark packet if set by kube-proxy
+		// unmark packet if set by kube-proxy to skip kube-postrouting rule and processed
+		// by cni snat rule
 		if err = iptables.InsertIptableRule(iptables.V6, iptables.Mangle, iptables.Postrouting, "", "MARK --set-mark 0x0"); err != nil {
 			log.Errorf("[net] Adding Iptable mangle rule failed:%v", err)
 			return err