From d182e67b2f3490719b83fce0ea90366e172b652c Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Wed, 26 May 2021 16:38:47 +0530 Subject: [PATCH 1/8] disable cleaning up hns network and state files from windows cni --- cni/ipam/ipam.go | 3 +- cni/network/network.go | 4 +++ cni/network/network_linux.go | 1 - cni/network/network_windows.go | 4 +-- cnm/ipam/ipam.go | 3 +- ipam/manager.go | 51 +++++++++++++++------------------- 6 files changed, 32 insertions(+), 34 deletions(-) diff --git a/cni/ipam/ipam.go b/cni/ipam/ipam.go index 32fc849c5a..1020abb129 100644 --- a/cni/ipam/ipam.go +++ b/cni/ipam/ipam.go @@ -21,6 +21,7 @@ import ( const ( ipamV6 = "azure-vnet-ipamv6" + rehydrateIpamInfoOnReboot = false ) var ( @@ -72,7 +73,7 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error { log.Printf("[cni-ipam] Running on %v", platform.GetOSInfo()) // Initialize address manager. - err = plugin.am.Initialize(config, plugin.Options) + err = plugin.am.Initialize(config, rehydrateIpamInfoOnReboot, plugin.Options) if err != nil { log.Printf("[cni-ipam] Failed to initialize address manager, err:%v.", err) return err diff --git a/cni/network/network.go b/cni/network/network.go index d8c3f011d0..e66419e406 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -70,6 +70,10 @@ const ( Baremetal ExecutionMode = "baremetal" ) +const ( + rehydrateNetworkInfoOnReboot = false +) + // NetPlugin represents the CNI network plugin. type netPlugin struct { *cni.Plugin diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index a8dfda1474..a64331efa1 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -22,7 +22,6 @@ const ( const ( snatConfigFileName = "/tmp/snatConfig" - rehydrateNetworkInfoOnReboot = false ) // handleConsecutiveAdd is a dummy function for Linux platform. diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index f5d3afeeba..5de1b9d38b 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -29,9 +29,7 @@ var ( win1903Version = 18362 ) -const ( - rehydrateNetworkInfoOnReboot = true -) + /* handleConsecutiveAdd handles consecutive add calls for infrastructure containers on Windows platform. * This is a temporary work around for issue #57253 of Kubernetes. diff --git a/cnm/ipam/ipam.go b/cnm/ipam/ipam.go index 5aa7f00247..ad60acd170 100644 --- a/cnm/ipam/ipam.go +++ b/cnm/ipam/ipam.go @@ -21,6 +21,7 @@ const ( requiresRequestReplay = false returnCode = 0 returnStr = "Success" + rehydrateIpamInfoOnReboot = true ) // IpamPlugin represents a CNM (libnetwork) IPAM plugin. @@ -65,7 +66,7 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error { } // Initialize address manager. - err = plugin.am.Initialize(config, plugin.Options) + err = plugin.am.Initialize(config, rehydrateIpamInfoOnReboot, plugin.Options) if err != nil { log.Printf("[ipam] Failed to initialize address manager, err:%v.", err) return err diff --git a/ipam/manager.go b/ipam/manager.go index 3fc932abaa..09370f26bc 100644 --- a/ipam/manager.go +++ b/ipam/manager.go @@ -31,7 +31,7 @@ type addressManager struct { // AddressManager API. type AddressManager interface { - Initialize(config *common.PluginConfig, options map[string]interface{}) error + Initialize(config *common.PluginConfig, rehydrateIpamInfoOnReboot bool, options map[string]interface{}) error Uninitialize() StartSource(options map[string]interface{}) error @@ -70,13 +70,13 @@ func NewAddressManager() (AddressManager, error) { } // Initialize configures address manager. -func (am *addressManager) Initialize(config *common.PluginConfig, options map[string]interface{}) error { +func (am *addressManager) Initialize(config *common.PluginConfig, rehydrateIpamInfoOnReboot bool, options map[string]interface{}) error { am.Version = config.Version am.store = config.Store am.netApi = config.NetApi // Restore persisted state. - err := am.restore() + err := am.restore(rehydrateIpamInfoOnReboot) if err != nil { return err } @@ -93,29 +93,15 @@ func (am *addressManager) Uninitialize() { } // Restore reads address manager state from persistent store. -func (am *addressManager) restore() error { +func (am *addressManager) restore(rehydrateIpamInfoOnReboot bool) error { // Skip if a store is not provided. if am.store == nil { log.Printf("[ipam] ipam store is nil") return nil } - rebooted := false - - // Check if the VM is rebooted. - modTime, err := am.store.GetModificationTime() - if err == nil { - rebootTime, err := platform.GetLastRebootTime() - log.Printf("[ipam] reboot time %v store mod time %v", rebootTime, modTime) - - if err == nil && rebootTime.After(modTime) { - log.Printf("[ipam] Detected Reboot") - rebooted = true - } - } - // Read any persisted state. - err = am.store.Read(storeKey, am) + err := am.store.Read(storeKey, am) if err != nil { if err == store.ErrKeyNotFound { log.Printf("[ipam] store key not found") @@ -141,15 +127,24 @@ func (am *addressManager) restore() error { } // if rebooted mark the ip as not in use. - if rebooted { - log.Printf("[ipam] Rehydrating ipam state from persistent store") - for _, as := range am.AddrSpaces { - for _, ap := range as.Pools { - ap.as = as - ap.RefCount = 0 - - for _, ar := range ap.Addresses { - ar.InUse = false + if rehydrateIpamInfoOnReboot { + // Check if the VM is rebooted. + modTime, err := am.store.GetModificationTime() + if err == nil { + rebootTime, err := platform.GetLastRebootTime() + log.Printf("[ipam] reboot time %v store mod time %v", rebootTime, modTime) + + if err == nil && rebootTime.After(modTime) { + log.Printf("[ipam] Rehydrating ipam state from persistent store") + for _, as := range am.AddrSpaces { + for _, ap := range as.Pools { + ap.as = as + ap.RefCount = 0 + + for _, ar := range ap.Addresses { + ar.InUse = false + } + } } } } From 9202c7681436f527038f3a15df00fe74f4edfe1c Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 16 Jul 2021 14:12:44 -0700 Subject: [PATCH 2/8] fixed uts --- ipam/manager_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ipam/manager_test.go b/ipam/manager_test.go index 0c1f0ec251..c0f0979874 100644 --- a/ipam/manager_test.go +++ b/ipam/manager_test.go @@ -49,7 +49,7 @@ func createAddressManager(options map[string]interface{}) (AddressManager, error return nil, err } - if err := am.Initialize(&config, options); err != nil { + if err := am.Initialize(&config, false, options); err != nil { return nil, err } @@ -164,7 +164,7 @@ var ( am, err := NewAddressManager() Expect(am).NotTo(BeNil()) Expect(err).NotTo(HaveOccurred()) - err = am.Initialize(&config, options) + err = am.Initialize(&config, false,options) Expect(err).To(BeNil()) }) }) @@ -180,7 +180,7 @@ var ( am, err := NewAddressManager() Expect(am).NotTo(BeNil()) Expect(err).NotTo(HaveOccurred()) - err = am.Initialize(&config, options) + err = am.Initialize(&config, false,options) Expect(err).To(BeNil()) }) }) @@ -196,7 +196,7 @@ var ( am, err := NewAddressManager() Expect(am).NotTo(BeNil()) Expect(err).NotTo(HaveOccurred()) - err = am.Initialize(&config, options) + err = am.Initialize(&config, false, options) Expect(err).To(HaveOccurred()) }) }) @@ -209,7 +209,7 @@ var ( am, err := NewAddressManager() Expect(am).NotTo(BeNil()) Expect(err).NotTo(HaveOccurred()) - err = am.Initialize(&config, options) + err = am.Initialize(&config, false,options) Expect(err).To(HaveOccurred()) }) }) @@ -221,7 +221,7 @@ var ( am := &addressManager{ AddrSpaces: make(map[string]*addressSpace), } - err := am.restore() + err := am.restore(false) Expect(err).To(BeNil()) }) }) @@ -250,7 +250,7 @@ var ( } as.Pools["ap-test"] = ap am.AddrSpaces["as-test"] = as - err := am.restore() + err := am.restore(false) Expect(err).To(BeNil()) as = am.AddrSpaces["as-test"] ap = as.Pools["ap-test"] @@ -284,7 +284,7 @@ var ( } as.Pools["ap-test"] = ap am.AddrSpaces["as-test"] = as - err := am.restore() + err := am.restore(false) Expect(err).To(BeNil()) as = am.AddrSpaces["as-test"] ap = as.Pools["ap-test"] @@ -316,7 +316,7 @@ var ( } as.Pools["ap-test"] = ap am.AddrSpaces["as-test"] = as - err := am.restore() + err := am.restore(false) Expect(err).To(BeNil()) as = am.AddrSpaces["as-test"] ap = as.Pools["ap-test"] From 47120f33102397acabe2e2aef1df1ca4f37e6d70 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 16 Jul 2021 14:25:06 -0700 Subject: [PATCH 3/8] fixed ut --- ipam/manager_ipv6Ipam_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipam/manager_ipv6Ipam_test.go b/ipam/manager_ipv6Ipam_test.go index 13d060beed..0c33005781 100644 --- a/ipam/manager_ipv6Ipam_test.go +++ b/ipam/manager_ipv6Ipam_test.go @@ -36,7 +36,7 @@ func createTestIpv6AddressManager() (AddressManager, error) { return nil, err } - err = am.Initialize(&config, options) + err = am.Initialize(&config, false, options) if err != nil { return nil, err } From 1f95a75387d95e92bae75691a24896c10c8ec11a Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Fri, 16 Jul 2021 14:34:08 -0700 Subject: [PATCH 4/8] remove the ut which is not needed anymore --- ipam/manager_test.go | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/ipam/manager_test.go b/ipam/manager_test.go index c0f0979874..879a7180a1 100644 --- a/ipam/manager_test.go +++ b/ipam/manager_test.go @@ -294,38 +294,6 @@ var ( Expect(ar.InUse).To(BeTrue()) }) }) - - Context("When rebooted", func() { - It("Should clear the RefCount and InUse", func() { - am := &addressManager{ - AddrSpaces: make(map[string]*addressSpace), - } - am.store = &testutils.KeyValueStoreMock{} - ap := &addressPool{ - Id: "ap-test", - RefCount: 1, - Addresses: make(map[string]*addressRecord), - } - ap.Addresses["ar-test"] = &addressRecord{ - ID: "ar-test", - InUse: true, - } - as := &addressSpace{ - Id: "as-test", - Pools: make(map[string]*addressPool), - } - as.Pools["ap-test"] = ap - am.AddrSpaces["as-test"] = as - err := am.restore(false) - Expect(err).To(BeNil()) - as = am.AddrSpaces["as-test"] - ap = as.Pools["ap-test"] - ar := ap.addrsByID["ar-test"] - Expect(ar.ID).To(Equal("ar-test")) - Expect(ap.RefCount).To(Equal(0)) - Expect(ar.InUse).To(BeFalse()) - }) - }) }) Describe("Test save", func() { From e75001b2540be24704906155f8a1c2ccd4c038e0 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 20 Jul 2021 10:30:19 -0700 Subject: [PATCH 5/8] addressed comments --- cni/ipam/ipam.go | 2 +- cni/network/network.go | 5 +---- cni/network/network_linux.go | 5 ++--- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cni/ipam/ipam.go b/cni/ipam/ipam.go index 1020abb129..f7e72dd2a1 100644 --- a/cni/ipam/ipam.go +++ b/cni/ipam/ipam.go @@ -21,7 +21,6 @@ import ( const ( ipamV6 = "azure-vnet-ipamv6" - rehydrateIpamInfoOnReboot = false ) var ( @@ -72,6 +71,7 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error { log.Printf("[cni-ipam] Plugin %v version %v.", plugin.Name, plugin.Version) log.Printf("[cni-ipam] Running on %v", platform.GetOSInfo()) + rehydrateIpamInfoOnReboot := false // Initialize address manager. err = plugin.am.Initialize(config, rehydrateIpamInfoOnReboot, plugin.Options) if err != nil { diff --git a/cni/network/network.go b/cni/network/network.go index 69f32e20c0..137700cf74 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -72,10 +72,6 @@ const ( Baremetal ExecutionMode = "baremetal" ) -const ( - rehydrateNetworkInfoOnReboot = false -) - // NetPlugin represents the CNI network plugin. type netPlugin struct { *cni.Plugin @@ -148,6 +144,7 @@ func (plugin *netPlugin) Start(config *common.PluginConfig) error { platform.PrintDependencyPackageDetails() common.LogNetworkInterfaces() + rehydrateNetworkInfoOnReboot := false // Initialize network manager. err = plugin.nm.Initialize(config, rehydrateNetworkInfoOnReboot) if err != nil { diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index a64331efa1..ea114611a4 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -20,9 +20,8 @@ const ( infraInterface = "eth2" ) -const ( - snatConfigFileName = "/tmp/snatConfig" -) +const snatConfigFileName = "/tmp/snatConfig" + // handleConsecutiveAdd is a dummy function for Linux platform. func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo network.NetworkInfo, epInfo *network.EndpointInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { From d772733ca103fa300c1242c291671f1e32b69cf8 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 20 Jul 2021 10:34:10 -0700 Subject: [PATCH 6/8] updated same pattern for cnm --- cnm/ipam/ipam.go | 2 +- cnm/network/network.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cnm/ipam/ipam.go b/cnm/ipam/ipam.go index ad60acd170..dcde26b6ec 100644 --- a/cnm/ipam/ipam.go +++ b/cnm/ipam/ipam.go @@ -21,7 +21,6 @@ const ( requiresRequestReplay = false returnCode = 0 returnStr = "Success" - rehydrateIpamInfoOnReboot = true ) // IpamPlugin represents a CNM (libnetwork) IPAM plugin. @@ -65,6 +64,7 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error { return err } + rehydrateIpamInfoOnReboot := true // Initialize address manager. err = plugin.am.Initialize(config, rehydrateIpamInfoOnReboot, plugin.Options) if err != nil { diff --git a/cnm/network/network.go b/cnm/network/network.go index 077be98ca6..7345d0f242 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -70,8 +70,9 @@ func (plugin *netPlugin) Start(config *common.PluginConfig) error { return err } + rehydrateNetworkInfoOnReboot := true // Initialize network manager. - err = plugin.nm.Initialize(config, true) + err = plugin.nm.Initialize(config, rehydrateNetworkInfoOnReboot) if err != nil { log.Printf("[net] Failed to initialize network manager, err:%v.", err) return err From 5a0affa649a1fc817394fb919948e59acc2c6034 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 20 Jul 2021 13:00:33 -0700 Subject: [PATCH 7/8] added comments --- cni/ipam/ipam.go | 1 + cni/network/network.go | 1 + cnm/ipam/ipam.go | 1 + cnm/network/network.go | 1 + 4 files changed, 4 insertions(+) diff --git a/cni/ipam/ipam.go b/cni/ipam/ipam.go index f7e72dd2a1..e73dd1ee4e 100644 --- a/cni/ipam/ipam.go +++ b/cni/ipam/ipam.go @@ -71,6 +71,7 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error { log.Printf("[cni-ipam] Plugin %v version %v.", plugin.Name, plugin.Version) log.Printf("[cni-ipam] Running on %v", platform.GetOSInfo()) + // rehyrdration required on reboot for cni ipam plugin rehydrateIpamInfoOnReboot := false // Initialize address manager. err = plugin.am.Initialize(config, rehydrateIpamInfoOnReboot, plugin.Options) diff --git a/cni/network/network.go b/cni/network/network.go index 137700cf74..c6759e1c18 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -144,6 +144,7 @@ func (plugin *netPlugin) Start(config *common.PluginConfig) error { platform.PrintDependencyPackageDetails() common.LogNetworkInterfaces() + // rehyrdration not required on reboot for cni plugin rehydrateNetworkInfoOnReboot := false // Initialize network manager. err = plugin.nm.Initialize(config, rehydrateNetworkInfoOnReboot) diff --git a/cnm/ipam/ipam.go b/cnm/ipam/ipam.go index dcde26b6ec..7938f1607c 100644 --- a/cnm/ipam/ipam.go +++ b/cnm/ipam/ipam.go @@ -64,6 +64,7 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error { return err } + // rehyrdration required on reboot for cnm ipam plugin rehydrateIpamInfoOnReboot := true // Initialize address manager. err = plugin.am.Initialize(config, rehydrateIpamInfoOnReboot, plugin.Options) diff --git a/cnm/network/network.go b/cnm/network/network.go index 7345d0f242..5a6e72d1cd 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -70,6 +70,7 @@ func (plugin *netPlugin) Start(config *common.PluginConfig) error { return err } + // rehyrdration required on reboot for cnm plugin rehydrateNetworkInfoOnReboot := true // Initialize network manager. err = plugin.nm.Initialize(config, rehydrateNetworkInfoOnReboot) From 58ffc2871406312572cac991f1ccfd64d626d0d3 Mon Sep 17 00:00:00 2001 From: Tamilmani Manoharan Date: Tue, 20 Jul 2021 15:05:46 -0700 Subject: [PATCH 8/8] addressed comments --- cni/ipam/ipam.go | 6 ++---- cni/network/network.go | 6 ++---- cni/network/network_linux.go | 3 +-- cnm/ipam/ipam.go | 6 ++---- cnm/network/network.go | 6 ++---- 5 files changed, 9 insertions(+), 18 deletions(-) diff --git a/cni/ipam/ipam.go b/cni/ipam/ipam.go index e73dd1ee4e..5989d7e890 100644 --- a/cni/ipam/ipam.go +++ b/cni/ipam/ipam.go @@ -71,10 +71,8 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error { log.Printf("[cni-ipam] Plugin %v version %v.", plugin.Name, plugin.Version) log.Printf("[cni-ipam] Running on %v", platform.GetOSInfo()) - // rehyrdration required on reboot for cni ipam plugin - rehydrateIpamInfoOnReboot := false - // Initialize address manager. - err = plugin.am.Initialize(config, rehydrateIpamInfoOnReboot, plugin.Options) + // Initialize address manager. rehyrdration not required on reboot for cni ipam plugin + err = plugin.am.Initialize(config, false, plugin.Options) if err != nil { log.Printf("[cni-ipam] Failed to initialize address manager, err:%v.", err) return err diff --git a/cni/network/network.go b/cni/network/network.go index c6759e1c18..00e79a96c6 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -144,10 +144,8 @@ func (plugin *netPlugin) Start(config *common.PluginConfig) error { platform.PrintDependencyPackageDetails() common.LogNetworkInterfaces() - // rehyrdration not required on reboot for cni plugin - rehydrateNetworkInfoOnReboot := false - // Initialize network manager. - err = plugin.nm.Initialize(config, rehydrateNetworkInfoOnReboot) + // Initialize network manager. rehyrdration not required on reboot for cni plugin + err = plugin.nm.Initialize(config, false) if err != nil { log.Printf("[cni-net] Failed to initialize network manager, err:%v.", err) return err diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index ea114611a4..12ffa73d03 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -20,8 +20,7 @@ const ( infraInterface = "eth2" ) -const snatConfigFileName = "/tmp/snatConfig" - +const snatConfigFileName = "/tmp/snatConfig" // handleConsecutiveAdd is a dummy function for Linux platform. func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo network.NetworkInfo, epInfo *network.EndpointInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) { diff --git a/cnm/ipam/ipam.go b/cnm/ipam/ipam.go index 7938f1607c..ec15b31aaf 100644 --- a/cnm/ipam/ipam.go +++ b/cnm/ipam/ipam.go @@ -64,10 +64,8 @@ func (plugin *ipamPlugin) Start(config *common.PluginConfig) error { return err } - // rehyrdration required on reboot for cnm ipam plugin - rehydrateIpamInfoOnReboot := true - // Initialize address manager. - err = plugin.am.Initialize(config, rehydrateIpamInfoOnReboot, plugin.Options) + // Initialize address manager. rehyrdration required on reboot for cnm ipam plugin + err = plugin.am.Initialize(config, true, plugin.Options) if err != nil { log.Printf("[ipam] Failed to initialize address manager, err:%v.", err) return err diff --git a/cnm/network/network.go b/cnm/network/network.go index 5a6e72d1cd..01a110b832 100644 --- a/cnm/network/network.go +++ b/cnm/network/network.go @@ -70,10 +70,8 @@ func (plugin *netPlugin) Start(config *common.PluginConfig) error { return err } - // rehyrdration required on reboot for cnm plugin - rehydrateNetworkInfoOnReboot := true - // Initialize network manager. - err = plugin.nm.Initialize(config, rehydrateNetworkInfoOnReboot) + // Initialize network manager. rehyrdration required on reboot for cnm plugin + err = plugin.nm.Initialize(config, true) if err != nil { log.Printf("[net] Failed to initialize network manager, err:%v.", err) return err