Add kube credentials lockfile to prevent possibility of excessive log…

…in attempts

api/utils/keypaths/keypaths.go

@@ -52,6 +52,8 @@ const (
 	kubeDirSuffix = "-kube"
 	// kubeConfigSuffix is the suffix of a kubeconfig file stored under the keys directory.
 	kubeConfigSuffix = "-kubeconfig"
+	// fileNameKubeCredLock is file name of lockfile used to prevent excessive login attempts.
+	fileNameKubeCredLock = "kube_credentials.lock"
 	// casDir is the directory name for where clusters certs are stored.
 	casDir = "cas"
 	// fileExtPem is the extension of a file where a public certificate is stored.
@@ -76,6 +78,7 @@ const (
 //    │   ├── foo                      --> Private Key for user "foo"
 //    │   ├── foo.pub                  --> Public Key
 //    │   ├── foo.ppk                  --> PuTTY PPK-formatted keypair for user "foo"
+//    │   ├── kube_credentials.lock    --> Kube credential lockfile, used to prevent excessive relogin attempts
 //    │   ├── foo-x509.pem             --> TLS client certificate for Auth Server
 //    │   ├── foo-ssh                  --> SSH certs for user "foo"
 //    │   │   ├── root-cert.pub        --> SSH cert for Teleport cluster "root"
@@ -311,6 +314,13 @@ func KubeConfigPath(baseDir, proxy, username, cluster, kubename string) string {
 	return filepath.Join(KubeCertDir(baseDir, proxy, username, cluster), kubename+kubeConfigSuffix)
 }
 
+// KubeCredLockfilePath returns the kube credentials lock file for given proxy
+//
+// <baseDir>/keys/<proxy>/kube_credentials.lock
+func KubeCredLockfilePath(baseDir, proxy string) string {
+	return filepath.Join(ProxyKeyDir(baseDir, proxy), fileNameKubeCredLock)
+}
+
 // IsProfileKubeConfigPath makes a best effort attempt to check if the given
 // path is a profile specific kubeconfig path generated by this package.
 func IsProfileKubeConfigPath(path string) (bool, error) {

lib/client/keystore.go

@@ -122,6 +122,11 @@ func (fs *FSKeyStore) ppkFilePath(idx KeyIndex) string {
 	return keypaths.PPKFilePath(fs.KeyDir, idx.ProxyHost, idx.Username)
 }
 
+// kubeCredLockfilePath returns kube credentials lockfile path for the given KeyIndex.
+func (fs *FSKeyStore) kubeCredLockfilePath(idx KeyIndex) string {
+	return keypaths.KubeCredLockfilePath(fs.KeyDir, idx.ProxyHost)
+}
+
 // publicKeyPath returns the public key path for the given KeyIndex.
 func (fs *FSKeyStore) publicKeyPath(idx KeyIndex) string {
 	return keypaths.PublicKeyPath(fs.KeyDir, idx.ProxyHost, idx.Username)
@@ -239,6 +244,8 @@ func (fs *FSKeyStore) DeleteKey(idx KeyIndex) error {
 		os.Remove(fs.ppkFilePath(idx))
 	}
 
+	os.Remove(fs.kubeCredLockfilePath(idx))
+
 	// Clear ClusterName to delete the user certs stored for all clusters.
 	idx.ClusterName = ""
 	return fs.DeleteUserCerts(idx, WithAllCerts...)

lib/utils/fs_windows.go

@@ -3,6 +3,8 @@
 
 package utils
 
+import "strings"
+
 /*
 Copyright 2018 Gravitational, Inc.
 
@@ -29,5 +31,10 @@ limitations under the License.
 const lockPostfix = ".lock.tmp"
 
 func getPlatformLockFilePath(path string) string {
+	// If target file is itself dedicated lockfile, we don't create another lockfile, since
+	// we don't intend to read/write the target file itself.
+	if strings.HasSuffix(path, ".lock") {
+		return path
+	}
 	return path + lockPostfix
 }

tool/tsh/kube.go

@@ -581,6 +581,49 @@ func newKubeCredentialsCommand(parent *kingpin.CmdClause) *kubeCredentialsComman
 	return c
 }
 
+func getKubeCredLockfilePath(homePath, proxy string) (string, error) {
+	profilePath := profile.FullProfilePath(homePath)
+	// tsh stores the profiles using the proxy host as the profile name.
+	profileName, err := utils.Host(proxy)
+	if err != nil {
+		return "", trace.Wrap(err)
+	}
+
+	return keypaths.KubeCredLockfilePath(profilePath, profileName), nil
+}
+
+var ErrKubeCredLockfileFound = trace.AlreadyExists("Having problems with relogin, please use 'tsh login/tsh kube login' manually")
+
+func takeKubeCredLock(ctx context.Context, homePath, proxy string) (func(bool), error) {
+	kubeCredLockfilePath, err := getKubeCredLockfilePath(homePath, proxy)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	// If kube credentials lockfile already exists, it means last time kube credentials was called
+	// we had an error while trying to issue certificate, return an error asking user to login manually.
+	if _, err := os.Stat(kubeCredLockfilePath); err == nil {
+		log.Debugf("Kube credentials lockfile was found at %q, aborting.", kubeCredLockfilePath)
+		return nil, ErrKubeCredLockfileFound
+	}
+
+	if _, err := utils.EnsureLocalPath(kubeCredLockfilePath, "", ""); err != nil {
+		return nil, trace.Wrap(err)
+	}
+	// Take a lock while we're trying to issue certificate and possibly relogin
+	unlock, err := utils.FSTryWriteLockTimeout(ctx, kubeCredLockfilePath, 5*time.Second)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	return func(removeFile bool) {
+		if removeFile {
+			os.Remove(kubeCredLockfilePath)
+		}
+		unlock()
+	}, nil
+}
+
 func (c *kubeCredentialsCommand) run(cf *CLIConf) error {
 	// client.LoadKeysToKubeFromStore function is used to speed up the credentials
 	// loading process since Teleport Store transverses the entire store to find the keys.
@@ -604,6 +647,12 @@ func (c *kubeCredentialsCommand) run(cf *CLIConf) error {
 		}
 	}
 
+	unlockKubeCred, err := takeKubeCredLock(cf.Context, cf.HomePath, cf.Proxy)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+	defer unlockKubeCred(false) // by default (in case of an error) we don't delete lockfile; safe to call twice
+
 	tc, err := makeClient(cf, true)
 	if err != nil {
 		return trace.Wrap(err)
@@ -628,6 +677,9 @@ func (c *kubeCredentialsCommand) run(cf *CLIConf) error {
 		}
 		if crt != nil && time.Until(crt.NotAfter) > time.Minute {
 			log.Debugf("Re-using existing TLS cert for Kubernetes cluster %q", c.kubeCluster)
+			// Unlock and remove the lockfile so subsequent tsh kube credentials calls don't exit early
+			unlockKubeCred(true)
+
 			return c.writeKeyResponse(cf.Stdout(), k, c.kubeCluster)
 		}
 		// Otherwise, cert for this k8s cluster is missing or expired. Request
@@ -662,6 +714,9 @@ func (c *kubeCredentialsCommand) run(cf *CLIConf) error {
 		return trace.Wrap(err)
 	}
 
+	// Unlock and remove the lockfile so subsequent tsh kube credentials calls don't exit early
+	unlockKubeCred(true)
+
 	return c.writeKeyResponse(cf.Stdout(), k, c.kubeCluster)
 }
 

tool/tsh/tsh_test.go

@@ -52,7 +52,9 @@ import (
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/api/types/events"
 	"github.com/gravitational/teleport/api/types/wrappers"
+	apiutils "github.com/gravitational/teleport/api/utils"
 	"github.com/gravitational/teleport/api/utils/keys"
+	"github.com/gravitational/teleport/integration/kube"
 	"github.com/gravitational/teleport/lib"
 	"github.com/gravitational/teleport/lib/auth"
 	"github.com/gravitational/teleport/lib/auth/mocku2f"
@@ -1924,6 +1926,118 @@ func tryCreateTrustedCluster(t *testing.T, authServer *auth.Server, trustedClust
 	require.FailNow(t, "Timeout creating trusted cluster")
 }
 
+func TestKubeCredentialsLock(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	const kubeClusterName = "kube-cluster"
+
+	t.Run("remaining lockfile prevents subsequent calls", func(t *testing.T) {
+		tmpHomePath := t.TempDir()
+
+		err := Run(ctx, []string{
+			"kube",
+			"credentials",
+			"--proxy", "fake-proxy",
+			"--teleport-cluster", "teleport",
+			"--kube-cluster", kubeClusterName,
+		}, setHomePath(tmpHomePath))
+		require.Error(t, err) // First run fails because fake proxy doesn't exist
+
+		err = Run(ctx, []string{
+			"kube",
+			"credentials",
+			"--proxy", "fake-proxy",
+			"--teleport-cluster", "teleport",
+			"--kube-cluster", kubeClusterName,
+		}, setHomePath(tmpHomePath))
+		require.ErrorIs(t, ErrKubeCredLockfileFound, err) // Second run returns error related to the kube cred lockfile, since last run failed
+	})
+
+	t.Run("running kube credentials in parallel", func(t *testing.T) {
+		tmpHomePath := t.TempDir()
+
+		connector := mockConnector(t)
+		alice, err := types.NewUser("alice@example.com")
+		require.NoError(t, err)
+
+		kubeRole, err := types.NewRole("kube-access", types.RoleSpecV6{
+			Allow: types.RoleConditions{
+				KubernetesLabels: types.Labels{types.Wildcard: apiutils.Strings{types.Wildcard}},
+				KubeGroups:       []string{kube.TestImpersonationGroup},
+				KubeUsers:        []string{alice.GetName()},
+				KubernetesResources: []types.KubernetesResource{
+					{
+						Kind: types.KindKubePod, Name: types.Wildcard, Namespace: types.Wildcard,
+					},
+				},
+			},
+		})
+		require.NoError(t, err)
+		alice.SetRoles([]string{"access", kubeRole.GetName()})
+
+		require.NoError(t, err)
+		authProcess, proxyProcess := makeTestServers(t, withBootstrap(connector, alice, kubeRole))
+		authServer := authProcess.GetAuthServer()
+		require.NotNil(t, authServer)
+		proxyAddr, err := proxyProcess.ProxyWebAddr()
+		require.NoError(t, err)
+
+		teleportClusterName, err := authServer.GetClusterName()
+		require.NoError(t, err)
+
+		kubeCluster, err := types.NewKubernetesClusterV3(types.Metadata{
+			Name:   kubeClusterName,
+			Labels: map[string]string{},
+		},
+			types.KubernetesClusterSpecV3{},
+		)
+		kubeServer, err := types.NewKubernetesServerV3FromCluster(kubeCluster, kubeClusterName, kubeClusterName)
+		require.NoError(t, err)
+		_, err = authServer.UpsertKubernetesServer(context.Background(), kubeServer)
+		require.NoError(t, err)
+
+		err = Run(context.Background(), []string{
+			"login",
+			"--insecure",
+			"--debug",
+			"--auth", connector.GetName(),
+			"--proxy", proxyAddr.String(),
+		}, setHomePath(tmpHomePath), func(cf *CLIConf) error {
+			cf.mockSSOLogin = mockSSOLogin(t, authServer, alice)
+			return nil
+		})
+		require.NoError(t, err)
+
+		_, err = profile.FromDir(tmpHomePath, "")
+		require.NoError(t, err)
+
+		errChan := make(chan error)
+		runCreds := func() {
+			err = Run(ctx, []string{
+				"kube",
+				"credentials",
+				"--proxy", proxyAddr.String(),
+				"--teleport-cluster", teleportClusterName.GetClusterName(),
+				"--kube-cluster", kubeClusterName,
+			}, setHomePath(tmpHomePath))
+			errChan <- err
+		}
+
+		runsCount := 3
+		for i := 0; i < runsCount; i++ {
+			go runCreds()
+		}
+		for i := 0; i < runsCount; i++ {
+			select {
+			case err := <-errChan:
+				require.NoError(t, err)
+			case <-time.After(time.Second * 1):
+				require.Fail(t, "Running kube credentials timed out")
+			}
+		}
+	})
+}
+
 func TestSSHHeadless(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()