Maintain unique kube configs (#13)

This change adds support for managing multiple kube configs to
allow for simultaneous operations on different kubernetes clusters
by a single provisioning server. Previously, kubernetes configuration
was written to a default location by kops which could lead to
race conditions by concurrent operations.

internal/api/kops_test.go

@@ -2,12 +2,13 @@ package api_test
 
 import (
 	"io/ioutil"
+	"path"
 
 	"github.com/mattermost/mattermost-cloud/internal/tools/kops"
 )
 
 type mockKopsCmd struct {
-	outputDirectory string
+	tempDirectory string
 }
 
 func newMockKopsCmd() (*mockKopsCmd, error) {
@@ -17,7 +18,7 @@ func newMockKopsCmd() (*mockKopsCmd, error) {
 	}
 
 	return &mockKopsCmd{
-		outputDirectory: dir,
+		tempDirectory: dir,
 	}, nil
 }
 
@@ -54,7 +55,11 @@ func (m *mockKopsCmd) ValidateCluster(string, bool) error {
 }
 
 func (m *mockKopsCmd) GetOutputDirectory() string {
-	return m.outputDirectory
+	return m.tempDirectory
+}
+
+func (m *mockKopsCmd) GetKubeConfigPath() string {
+	return path.Join(m.tempDirectory, "kubeconfig")
 }
 
 func (m *mockKopsCmd) Close() error {

internal/provisioner/cluster.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"path/filepath"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -174,8 +173,7 @@ func (provisioner *KopsProvisioner) CreateCluster(request *api.CreateClusterRequ
 	logger.WithField("name", kopsMetadata.Name).Info("successfully deployed kubernetes")
 
 	// Begin deploying the mattermost operator.
-	// TODO: remove reliance on kube config being in the default location.
-	k8sClient, err := provisioner.k8sFactory(filepath.Join(os.Getenv("HOME"), ".kube", "config"), logger)
+	k8sClient, err := provisioner.k8sFactory(kops.GetKubeConfigPath(), logger)
 	if err != nil {
 		return &cluster, err
 	}

internal/provisioner/kops_cmd.go

@@ -18,5 +18,6 @@ type KopsCmd interface {
 	WaitForKubernetesReadiness(dns string, timeout int) error
 	ValidateCluster(name string, silent bool) error
 	GetOutputDirectory() string
+	GetKubeConfigPath() string
 	Close() error
 }

internal/tools/kops/cluster.go

@@ -23,7 +23,7 @@ func (c *Cmd) CreateCluster(name, cloud string, clusterSize ClusterSize, zones [
 		arg("node-size", clusterSize.NodeSize),
 		arg("master-size", clusterSize.MasterSize),
 		arg("target", "terraform"),
-		arg("out", c.outputDir),
+		arg("out", c.GetOutputDirectory()),
 		arg("output", "json"),
 	)
 	if err != nil {
@@ -58,7 +58,7 @@ func (c *Cmd) UpdateCluster(name string) error {
 		arg("state", "s3://", c.s3StateStore),
 		"--yes",
 		arg("target", "terraform"),
-		arg("out", c.outputDir),
+		arg("out", c.GetOutputDirectory()),
 	)
 	if err != nil {
 		return errors.Wrap(err, "failed to invoke kops update cluster")

internal/tools/kops/cmd.go

@@ -3,42 +3,52 @@ package kops
 import (
 	"io/ioutil"
 	"os"
+	"path"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
-const defaultKopsPath = "/usr/local/bin/kops"
+const (
+	defaultKopsPath = "/usr/local/bin/kops"
+	outputDirName   = "output"
+	kubeConfigName  = "kubeconfig"
+)
 
 // Cmd is the kops command to execute.
 type Cmd struct {
 	kopsPath     string
 	s3StateStore string
-	outputDir    string
+	tempDir      string
 	logger       log.FieldLogger
 }
 
 // New creates a new instance of Cmd through which to execute kops.
 func New(s3StateStore string, logger log.FieldLogger) (*Cmd, error) {
-	outputDir, err := ioutil.TempDir("", "kops")
+	tempDir, err := ioutil.TempDir("", "kops-")
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to create temporary directory for output")
+		return nil, errors.Wrap(err, "failed to create temporary kops directory")
 	}
 
 	return &Cmd{
 		kopsPath:     defaultKopsPath,
 		s3StateStore: s3StateStore,
-		outputDir:    outputDir,
+		tempDir:      tempDir,
 		logger:       logger,
 	}, nil
 }
 
 // GetOutputDirectory returns the temporary output directory used by kops.
 func (c *Cmd) GetOutputDirectory() string {
-	return c.outputDir
+	return path.Join(c.tempDir, outputDirName)
+}
+
+// GetKubeConfigPath returns the temporary kubeconfig directory used by kops.
+func (c *Cmd) GetKubeConfigPath() string {
+	return path.Join(c.tempDir, kubeConfigName)
 }
 
 // Close cleans up the temporary output directory used by kops.
 func (c *Cmd) Close() error {
-	return os.RemoveAll(c.outputDir)
+	return os.RemoveAll(c.tempDir)
 }

internal/tools/kops/run.go

@@ -3,6 +3,7 @@ package kops
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
 	"os/exec"
 	"regexp"
 	"strings"
@@ -61,12 +62,20 @@ func outputLogger(line string, logger log.FieldLogger) {
 
 func (c *Cmd) run(arg ...string) ([]byte, []byte, error) {
 	cmd := exec.Command(c.kopsPath, arg...)
+	cmd.Env = append(
+		os.Environ(),
+		fmt.Sprintf("KUBECONFIG=%s", c.GetKubeConfigPath()),
+	)
 
 	return exechelper.Run(cmd, c.logger, outputLogger)
 }
 
 func (c *Cmd) runSilent(arg ...string) ([]byte, []byte, error) {
 	cmd := exec.Command(c.kopsPath, arg...)
+	cmd.Env = append(
+		os.Environ(),
+		fmt.Sprintf("KUBECONFIG=%s", c.GetKubeConfigPath()),
+	)
 
 	return exechelper.Run(cmd, silentLogger(), func(string, log.FieldLogger) {})
 }