From a7d7a073118e075b10ae471e2d01b7f6d053c6f2 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Mon, 7 Jan 2019 16:33:15 -0800 Subject: [PATCH 1/2] Cleanup networks and lock files after detecting reboot --- cni/plugin.go | 37 ++++++++++++++++++++++++++++--------- ipam/manager.go | 2 +- network/manager.go | 8 ++++++++ platform/os_windows.go | 8 ++++++++ store/json.go | 28 ++++++++++++++++++++++++++-- store/store.go | 3 ++- 6 files changed, 73 insertions(+), 13 deletions(-) diff --git a/cni/plugin.go b/cni/plugin.go index f6ac56b7cf..fea9fd24a4 100644 --- a/cni/plugin.go +++ b/cni/plugin.go @@ -34,6 +34,18 @@ func NewPlugin(name, version string) (*Plugin, error) { return nil, err } + // Initialize logging. + log.SetName(plugin.Name) + log.SetLevel(log.LevelInfo) + err = log.SetTarget(log.TargetLogfile) + if err != nil { + log.Printf("[cni] Failed to configure logging, err:%v.\n", err) + return &Plugin{ + Plugin: plugin, + version: version, + }, err + } + return &Plugin{ Plugin: plugin, version: version, @@ -45,14 +57,6 @@ func (plugin *Plugin) Initialize(config *common.PluginConfig) error { // Initialize the base plugin. plugin.Plugin.Initialize(config) - // Initialize logging. - log.SetName(plugin.Name) - log.SetLevel(log.LevelInfo) - err := log.SetTarget(log.TargetLogfile) - if err != nil { - log.Printf("[cni] Failed to configure logging, err:%v.\n", err) - return err - } return nil } @@ -164,6 +168,21 @@ func (plugin *Plugin) InitializeKeyValueStore(config *common.PluginConfig) error log.Printf("[cni] Failed to create store: %v.", err) return err } + + // Force unlock the json store if the lock file is left on the node after reboot + if lockFileModTime, err := plugin.Store.GetLockFileModificationTime(); err == nil { + rebootTime, err := platform.GetLastRebootTime() + log.Printf("[cni] reboot time %v storeLockFile mod time %v", rebootTime, lockFileModTime) + if err == nil && rebootTime.After(lockFileModTime) { + log.Printf("[cni] Detected Reboot") + + if err := plugin.Store.Unlock(true); err != nil { + log.Printf("[cni] Failed to force unlock store due to error %v", err) + } else { + log.Printf("[cni] Force unlocked the store successfully") + } + } + } } // Acquire store lock. @@ -180,7 +199,7 @@ func (plugin *Plugin) InitializeKeyValueStore(config *common.PluginConfig) error // Uninitialize key-value store func (plugin *Plugin) UninitializeKeyValueStore() error { if plugin.Store != nil { - err := plugin.Store.Unlock() + err := plugin.Store.Unlock(false) if err != nil { log.Printf("[cni] Failed to unlock store: %v.", err) return err diff --git a/ipam/manager.go b/ipam/manager.go index a87fcc544e..4cbd34f790 100644 --- a/ipam/manager.go +++ b/ipam/manager.go @@ -105,7 +105,6 @@ func (am *addressManager) restore() error { // Check if the VM is rebooted. modTime, err := am.store.GetModificationTime() if err == nil { - rebootTime, err := platform.GetLastRebootTime() log.Printf("[ipam] reboot time %v store mod time %v", rebootTime, modTime) @@ -147,6 +146,7 @@ func (am *addressManager) restore() error { for _, as := range am.AddrSpaces { for _, ap := range as.Pools { ap.as = as + ap.RefCount = 0 for _, ar := range ap.Addresses { ar.InUse = false diff --git a/network/manager.go b/network/manager.go index 2c41971b00..f4c136388e 100644 --- a/network/manager.go +++ b/network/manager.go @@ -129,6 +129,14 @@ func (nm *networkManager) restore() error { return err } + // Delete the networks left behind after reboot + for _, extIf := range nm.ExternalInterfaces { + for _, nw := range extIf.Networks { + log.Printf("[net] Deleting the network %s on reboot\n", nw.Id) + nm.deleteNetwork(nw.Id) + } + } + // Clear networkManager contents nm.TimeStamp = time.Time{} for extIfName := range nm.ExternalInterfaces { diff --git a/platform/os_windows.go b/platform/os_windows.go index f0467b89c8..fe5bc0fa86 100644 --- a/platform/os_windows.go +++ b/platform/os_windows.go @@ -65,7 +65,15 @@ func GetLastRebootTime() (time.Time, error) { bootPM := strings.Contains(strings.Split(systemBootTime, " ")[2], "PM") month := strings.Split(bootDate, "/")[0] + if len(month) < 2 { + month = "0" + month + } + day := strings.Split(bootDate, "/")[1] + if len(day) < 2 { + day = "0" + day + } + year := strings.Split(bootDate, "/")[2] year = strings.Trim(year, ",") hour := strings.Split(bootTime, ":")[0] diff --git a/store/json.go b/store/json.go index 792c3777aa..373ca74bb9 100644 --- a/store/json.go +++ b/store/json.go @@ -170,11 +170,11 @@ func (kvs *jsonFileStore) Lock(block bool) error { } // Unlock unlocks the store. -func (kvs *jsonFileStore) Unlock() error { +func (kvs *jsonFileStore) Unlock(forceUnlock bool) error { kvs.Mutex.Lock() defer kvs.Mutex.Unlock() - if !kvs.locked { + if !forceUnlock && !kvs.locked { return ErrStoreNotLocked } @@ -202,3 +202,27 @@ func (kvs *jsonFileStore) GetModificationTime() (time.Time, error) { return info.ModTime().UTC(), nil } + +// GetLockFileModificationTime returns the modification time of the lock file of the persistent store. +func (kvs *jsonFileStore) GetLockFileModificationTime() (time.Time, error) { + kvs.Mutex.Lock() + defer kvs.Mutex.Unlock() + + lockFileName := kvs.fileName + lockExtension + + // Check if the file exists. + file, err := os.Open(lockFileName) + if err != nil { + return time.Time{}.UTC(), err + } + + defer file.Close() + + info, err := os.Stat(lockFileName) + if err != nil { + log.Printf("os.stat() for file %v failed: %v", lockFileName, err) + return time.Time{}.UTC(), err + } + + return info.ModTime().UTC(), nil +} diff --git a/store/store.go b/store/store.go index 44a73eefcc..2e505ba2ed 100644 --- a/store/store.go +++ b/store/store.go @@ -14,8 +14,9 @@ type KeyValueStore interface { Write(key string, value interface{}) error Flush() error Lock(block bool) error - Unlock() error + Unlock(forceUnlock bool) error GetModificationTime() (time.Time, error) + GetLockFileModificationTime() (time.Time, error) } var ( From b64927d19ccdf5aa2c1812820a62cc479e64cf93 Mon Sep 17 00:00:00 2001 From: Ashvin Deodhar Date: Mon, 7 Jan 2019 16:41:59 -0800 Subject: [PATCH 2/2] Fix the circleci --- store/json_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/json_test.go b/store/json_test.go index 90e9f9b4ae..39c8a8af64 100644 --- a/store/json_test.go +++ b/store/json_test.go @@ -185,7 +185,7 @@ func TestLockingStoreGivesExclusiveAccess(t *testing.T) { } // Unlock the first store. - err = kvs.Unlock() + err = kvs.Unlock(false) if err != nil { t.Errorf("Failed to unlock first store: %v", err) } @@ -198,7 +198,7 @@ func TestLockingStoreGivesExclusiveAccess(t *testing.T) { } // Unlock the second store. - err = kvs2.Unlock() + err = kvs2.Unlock(false) if err != nil { t.Errorf("Failed to unlock second store: %v", err) }