Add --internal flag for export kubecfg that targets the internal dns …

…name Kops creates an "api.internal.$clustername" dns A record that points to the master IP(s) This adds a flag that will use that name and force the CA cert to be included. This is a workaround for client certificate authentication not working on API ELBs with ACM certificates. The ELB has a TLS listener rather than TCP, so the client certificate is not passed through to the apiserver. Using --internal will bypass the API ELB so that the client certificate will be passed directly to the apiserver. This also requires that the masters' security groups allow 443 access from the client which this does not handle automatically.
kubernetes · Aug 14, 2020 · 7edc7fd · 7edc7fd
1 parent 67e1bf1
commit 7edc7fd
Show file tree

Hide file tree

Showing 7 changed files with 74 additions and 11 deletions.
diff --git a/cmd/kops/export_kubecfg.go b/cmd/kops/export_kubecfg.go
@@ -45,6 +45,9 @@ var (
 
 	# export using a user already existing in the kubeconfig file
 	kops export kubecfg kubernetes-cluster.example.com --user my-oidc-user
+
+	# export using the internal DNS name, bypassing the cloud load balancer
+	kops export kubecfg kubernetes-cluster.example.com --internal
 		`))
 
 	exportKubecfgShort = i18n.T(`Export kubecfg.`)
@@ -55,6 +58,7 @@ type ExportKubecfgOptions struct {
 	all            bool
 	admin          bool
 	user           string
+	internal       bool
 }
 
 func NewCmdExportKubecfg(f *util.Factory, out io.Writer) *cobra.Command {
@@ -78,6 +82,7 @@ func NewCmdExportKubecfg(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().BoolVar(&options.all, "all", options.all, "export all clusters from the kops state store")
 	cmd.Flags().BoolVar(&options.admin, "admin", options.admin, "export the cluster admin user and add it to the context")
 	cmd.Flags().StringVar(&options.user, "user", options.user, "add an existing user to the cluster context")
+	cmd.Flags().BoolVar(&options.internal, "internal", options.internal, "use the cluster's internal DNS name")
 
 	return cmd
 }
@@ -128,7 +133,7 @@ func RunExportKubecfg(ctx context.Context, f *util.Factory, out io.Writer, optio
 			return err
 		}
 
-		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, buildPathOptions(options), options.admin, options.user)
+		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, buildPathOptions(options), options.admin, options.user, options.internal)
 		if err != nil {
 			return err
 		}

diff --git a/cmd/kops/update_cluster.go b/cmd/kops/update_cluster.go
@@ -69,6 +69,7 @@ type UpdateClusterOptions struct {
 	CreateKubecfg bool
 	admin         bool
 	user          string
+	internal      bool
 
 	Phase string
 
@@ -118,6 +119,7 @@ func NewCmdUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().BoolVar(&options.CreateKubecfg, "create-kube-config", options.CreateKubecfg, "Will control automatically creating the kube config file on your local filesystem")
 	cmd.Flags().BoolVar(&options.admin, "admin", options.admin, "Also export the admin user. Implies --create-kube-config")
 	cmd.Flags().StringVar(&options.user, "user", options.user, "Existing user to add to the cluster context. Implies --create-kube-config")
+	cmd.Flags().BoolVar(&options.internal, "internal", options.internal, "Use the cluster's internal DNS name. Implies --create-kube-config")
 	cmd.Flags().BoolVar(&options.AllowKopsDowngrade, "allow-kops-downgrade", options.AllowKopsDowngrade, "Allow an older version of kops to update the cluster than last used")
 	cmd.Flags().StringVar(&options.Phase, "phase", options.Phase, "Subset of tasks to run: "+strings.Join(cloudup.Phases.List(), ", "))
 	cmd.Flags().StringSliceVar(&options.LifecycleOverrides, "lifecycle-overrides", options.LifecycleOverrides, "comma separated list of phase overrides, example: SecurityGroups=Ignore,InternetGateway=ExistsAndWarnIfChanges")
@@ -159,6 +161,11 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, clusterName string,
 		c.CreateKubecfg = true
 	}
 
+	if c.internal && !c.CreateKubecfg {
+		klog.Info("--internal implies --create-kube-config")
+		c.CreateKubecfg = true
+	}
+
 	// direct requires --yes (others do not, because they don't do anything!)
 	if c.Target == cloudup.TargetDirect {
 		if !c.Yes {
@@ -304,7 +311,7 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, clusterName string,
 		}
 		if kubecfgCert != nil {
 			klog.Infof("Exporting kubecfg for cluster")
-			conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, clientcmd.NewDefaultPathOptions(), c.admin, c.user)
+			conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, clientcmd.NewDefaultPathOptions(), c.admin, c.user, c.internal)
 			if err != nil {
 				return nil, err
 			}

diff --git a/docs/cli/kops_export_kubecfg.md b/docs/cli/kops_export_kubecfg.md
@@ -21,6 +21,9 @@ kops export kubecfg CLUSTERNAME [flags]
   
   # export using a user already existing in the kubeconfig file
   kops export kubecfg kubernetes-cluster.example.com --user my-oidc-user
+  
+  # export using the internal DNS name, bypassing the cloud load balancer
+  kops export kubecfg kubernetes-cluster.example.com --internal
 ```
 
 ### Options
@@ -29,6 +32,7 @@ kops export kubecfg CLUSTERNAME [flags]
       --admin               export the cluster admin user and add it to the context
       --all                 export all clusters from the kops state store
   -h, --help                help for kubecfg
+      --internal            use the cluster's internal DNS name
       --kubeconfig string   the location of the kubeconfig file to create.
       --user string         add an existing user to the cluster context
 ```

diff --git a/docs/cli/kops_update_cluster.md b/docs/cli/kops_update_cluster.md
@@ -29,6 +29,7 @@ kops update cluster [flags]
       --allow-kops-downgrade          Allow an older version of kops to update the cluster than last used
       --create-kube-config            Will control automatically creating the kube config file on your local filesystem
   -h, --help                          help for cluster
+      --internal                      Use the cluster's internal DNS name. Implies --create-kube-config
       --lifecycle-overrides strings   comma separated list of phase overrides, example: SecurityGroups=Ignore,InternetGateway=ExistsAndWarnIfChanges
       --out string                    Path to write any local output
       --phase string                  Subset of tasks to run: assets, cluster, network, security

diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
@@ -48,7 +48,9 @@ spec:
       idleTimeoutSeconds: 300
 ```
 
-You can use a valid SSL Certificate for your API Server Load Balancer. Currently, only AWS is supported:
+You can use a valid SSL Certificate for your API Server Load Balancer. Currently, only AWS is supported.
+
+Note that when using `sslCertificate`, client certificate authentication, such as with the credentials generated via `kops export kubecfg`, will not work through the load balancer. A `kubecfg` that bypasses the load balancer may be created with the `--internal` flag to `kops update cluster` or `kops export kubecfg`. Security groups may need to be opened to allow access from the clients to the master instances' port TCP/443, for example by using the `additionalSecurityGroups` field on the master instance groups.
 
 ```yaml
 spec:

diff --git a/pkg/kubeconfig/create_kubecfg.go b/pkg/kubeconfig/create_kubecfg.go
@@ -28,12 +28,20 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 )
 
-func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, configAccess clientcmd.ConfigAccess, admin bool, user string) (*KubeconfigBuilder, error) {
+func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, configAccess clientcmd.ConfigAccess, admin bool, user string, internal bool) (*KubeconfigBuilder, error) {
 	clusterName := cluster.ObjectMeta.Name
 
-	master := cluster.Spec.MasterPublicName
-	if master == "" {
-		master = "api." + clusterName
+	var master string
+	if internal {
+		master = cluster.Spec.MasterInternalName
+		if master == "" {
+			master = "api.internal." + clusterName
+		}
+	} else {
+		master = cluster.Spec.MasterPublicName
+		if master == "" {
+			master = "api." + clusterName
+		}
 	}
 
 	server := "https://" + master
@@ -88,8 +96,8 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se
 	b.Context = clusterName
 	b.Server = server
 
-	// add the CA Cert to the kubeconfig only if we didn't specify a SSL cert for the LB
-	if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" {
+	// add the CA Cert to the kubeconfig only if we didn't specify a SSL cert for the LB or are targeting the internal DNS name
+	if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || internal {
 		cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA)
 		if err != nil {
 			return nil, fmt.Errorf("error fetching CA keypair: %v", err)

diff --git a/pkg/kubeconfig/create_kubecfg_test.go b/pkg/kubeconfig/create_kubecfg_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package kubeconfig
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 
@@ -77,6 +78,7 @@ func buildMinimalCluster(clusterName string, masterPublicName string) *kops.Clus
 	}
 
 	c.Spec.MasterPublicName = masterPublicName
+	c.Spec.MasterInternalName = fmt.Sprintf("internal.%v", masterPublicName)
 	c.Spec.KubernetesAPIAccess = []string{"0.0.0.0/0"}
 	c.Spec.SSHAccess = []string{"0.0.0.0/0"}
 
@@ -121,6 +123,7 @@ func TestBuildKubecfg(t *testing.T) {
 		configAccess clientcmd.ConfigAccess
 		admin        bool
 		user         string
+		internal     bool
 	}
 
 	publiccluster := buildMinimalCluster("testcluster", "testcluster.test.com")
@@ -150,6 +153,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				true,
 				"",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context:    "testcluster",
@@ -178,6 +182,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				false,
 				"myuser",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context:    "testcluster",
@@ -206,6 +211,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				true,
 				"",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context:    "emptyMasterPublicNameCluster",
@@ -242,6 +248,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				true,
 				"",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context:    "testgossipcluster.k8s.local",
@@ -253,16 +260,45 @@ func TestBuildKubecfg(t *testing.T) {
 			},
 			false,
 		},
+		{
+			"Test Kube Config Data For internal DNS name with admin",
+			args{
+				publiccluster,
+				fakeKeyStore{
+					FindKeypairFn: func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
+						return fakeCertificate(),
+							fakePrivateKey(),
+							true,
+							nil
+					},
+				},
+				nil,
+				fakeStatusStore{},
+				nil,
+				true,
+				"",
+				true,
+			},
+			&KubeconfigBuilder{
+				Context:    "testcluster",
+				Server:     "https://internal.testcluster.test.com",
+				CACert:     []byte(certData),
+				ClientCert: []byte(certData),
+				ClientKey:  []byte(privatekeyData),
+				User:       "testcluster",
+			},
+			false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := BuildKubecfg(tt.args.cluster, tt.args.keyStore, tt.args.secretStore, tt.args.status, tt.args.configAccess, tt.args.admin, tt.args.user)
+			got, err := BuildKubecfg(tt.args.cluster, tt.args.keyStore, tt.args.secretStore, tt.args.status, tt.args.configAccess, tt.args.admin, tt.args.user, tt.args.internal)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("BuildKubecfg() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("BuildKubecfg() = %v, want %v", got, tt.want)
+				t.Errorf("BuildKubecfg() = %+v, want %+v", got, tt.want)
 			}
 		})
 	}