Cross-platform multi process support (eksctl-io#2230)

* Flock kubeconfig file to allow for safe multiprocess use

* Cross platform file locking

Co-authored-by: Mike Beaumont <mjboamail@gmail.com>

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/fluxcd/flux/pkg/install v0.0.0-20200402142123-873fb9300996 // flux 1.19.0
 	github.com/fluxcd/helm-operator/pkg/install v0.0.0-20200407140510-8d71b0072a3e // helm-operator 1.0.0
 	github.com/gobwas/glob v0.2.3
+	github.com/gofrs/flock v0.7.1
 	github.com/golangci/golangci-lint v1.27.0
 	github.com/goreleaser/goreleaser v0.110.0
 	github.com/instrumenta/kubeval v0.0.0-20190918223246-8d013ec9fc56

go.sum

@@ -404,6 +404,8 @@ github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZ
 github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/flock v0.7.0 h1:pGFUjl501gafK9HBt1VGL1KCOd/YhIooID+xgyJCf3g=
 github.com/gofrs/flock v0.7.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc=
+github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

pkg/utils/kubeconfig/kubeconfig.go

@@ -6,13 +6,13 @@ import (
 	"path"
 	"strings"
 
-	"github.com/weaveworks/eksctl/pkg/utils/file"
-
 	"os/exec"
 
+	"github.com/gofrs/flock"
 	"github.com/kris-nova/logger"
 	"github.com/pkg/errors"
 	api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
+	"github.com/weaveworks/eksctl/pkg/utils/file"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 )
@@ -139,12 +139,43 @@ func AppendAuthenticator(config *clientcmdapi.Config, spec *api.ClusterConfig, a
 	}
 }
 
+func lockConfigFile(filePath string) error {
+	flock := flock.New(filePath)
+	err := flock.Lock()
+	if err != nil {
+		return errors.Wrap(err, "flock: failed to obtain exclusive lock existing kubeconfig file")
+	}
+
+	return nil
+}
+
+func unlockConfigFile(filePath string) error {
+	flock := flock.New(filePath)
+	err := flock.Unlock()
+	if err != nil {
+		return errors.Wrap(err, "flock: failed to release exclusive lock on existing kubeconfig file")
+	}
+
+	return nil
+}
+
 // Write will write Kubernetes client configuration to a file.
 // If path isn't specified then the path will be determined by client-go.
 // If file pointed to by path doesn't exist it will be created.
 // If the file already exists then the configuration will be merged with the existing file.
 func Write(path string, newConfig clientcmdapi.Config, setContext bool) (string, error) {
 	configAccess := getConfigAccess(path)
+	configFileName := configAccess.GetDefaultFilename()
+	err := lockConfigFile(configFileName)
+	if err != nil {
+		return "", err
+	}
+
+	defer func() {
+		if err := unlockConfigFile(configFileName); err != nil {
+			logger.Critical(err.Error())
+		}
+	}()
 
 	config, err := configAccess.GetStartingConfig()
 	if err != nil {
@@ -163,7 +194,7 @@ func Write(path string, newConfig clientcmdapi.Config, setContext bool) (string,
 		return "", errors.Wrapf(err, "unable to modify kubeconfig %s", path)
 	}
 
-	return configAccess.GetDefaultFilename(), nil
+	return configFileName, nil
 }
 
 func getConfigAccess(explicitPath string) clientcmd.ConfigAccess {
@@ -223,6 +254,17 @@ func MaybeDeleteConfig(meta *api.ClusterMeta) {
 	p := AutoPath(meta.Name)
 
 	if file.Exists(p) {
+		err := lockConfigFile(p)
+		if err != nil {
+			logger.Critical(err.Error())
+		}
+
+		defer func() {
+			if err := unlockConfigFile(p); err != nil {
+				logger.Critical(err.Error())
+			}
+		}()
+
 		if err := isValidConfig(p, meta.Name); err != nil {
 			logger.Debug(err.Error())
 			return
@@ -234,6 +276,18 @@ func MaybeDeleteConfig(meta *api.ClusterMeta) {
 	}
 
 	configAccess := getConfigAccess(DefaultPath)
+	defaultFilename := configAccess.GetDefaultFilename()
+	err := lockConfigFile(defaultFilename)
+	if err != nil {
+		logger.Critical(err.Error())
+	}
+
+	defer func() {
+		if err := unlockConfigFile(defaultFilename); err != nil {
+			logger.Critical(err.Error())
+		}
+	}()
+
 	config, err := configAccess.GetStartingConfig()
 	if err != nil {
 		logger.Debug("error reading kubeconfig file %q: %s", DefaultPath, err.Error())

pkg/utils/kubeconfig/kubeconfig_test.go

@@ -3,6 +3,7 @@ package kubeconfig_test
 import (
 	"io/ioutil"
 	"os"
+	"sync"
 
 	eksctlapi "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
 	"github.com/weaveworks/eksctl/pkg/utils/kubeconfig"
@@ -296,5 +297,28 @@ var _ = Describe("Kubeconfig", func() {
 			Expect(err).To(BeNil())
 			Expect(configFileAsBytes).To(MatchYAML(twoClustersAsBytes), "Should not change")
 		})
+
+		It("safely handles concurrent read-modify-write operations", func() {
+			var wg sync.WaitGroup
+			multiplier := 3
+			iters := 100
+			for i := 0; i < multiplier; i++ {
+				for k := 0; k < iters; k++ {
+					wg.Add(2)
+					go func() {
+						defer wg.Done()
+						_, err := configFile.Write(oneClusterAsBytes)
+						Expect(err).To(BeNil())
+					}()
+					go func() {
+						defer wg.Done()
+						_, err := configFile.Write(twoClustersAsBytes)
+						Expect(err).To(BeNil())
+					}()
+				}
+			}
+
+			wg.Wait()
+		})
 	})
 })