Merge pull request #9732 from rifelpet/export-kubecfg-internal

Add --internal flag for export kubecfg that targets the internal dns name

cmd/kops/export_kubecfg.go

@@ -46,6 +46,9 @@ var (
 
 	# export using a user already existing in the kubeconfig file
 	kops export kubecfg kubernetes-cluster.example.com --user my-oidc-user
+
+	# export using the internal DNS name, bypassing the cloud load balancer
+	kops export kubecfg kubernetes-cluster.example.com --internal
 		`))
 
 	exportKubecfgShort = i18n.T(`Export kubecfg.`)
@@ -56,6 +59,7 @@ type ExportKubecfgOptions struct {
 	all            bool
 	admin          time.Duration
 	user           string
+	internal       bool
 }
 
 func NewCmdExportKubecfg(f *util.Factory, out io.Writer) *cobra.Command {
@@ -80,6 +84,7 @@ func NewCmdExportKubecfg(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().DurationVar(&options.admin, "admin", options.admin, "export a cluster admin user credential with the given lifetime and add it to the cluster context")
 	cmd.Flags().Lookup("admin").NoOptDefVal = kubeconfig.DefaultKubecfgAdminLifetime.String()
 	cmd.Flags().StringVar(&options.user, "user", options.user, "add an existing user to the cluster context")
+	cmd.Flags().BoolVar(&options.internal, "internal", options.internal, "use the cluster's internal DNS name")
 
 	return cmd
 }
@@ -130,7 +135,7 @@ func RunExportKubecfg(ctx context.Context, f *util.Factory, out io.Writer, optio
 			return err
 		}
 
-		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, buildPathOptions(options), options.admin, options.user)
+		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, buildPathOptions(options), options.admin, options.user, options.internal)
 		if err != nil {
 			return err
 		}

cmd/kops/update_cluster.go

@@ -70,6 +70,7 @@ type UpdateClusterOptions struct {
 	CreateKubecfg bool
 	admin         time.Duration
 	user          string
+	internal      bool
 
 	Phase string
 
@@ -120,6 +121,7 @@ func NewCmdUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().DurationVar(&options.admin, "admin", options.admin, "Also export a cluster admin user credential with the specified lifetime and add it to the cluster context")
 	cmd.Flags().Lookup("admin").NoOptDefVal = kubeconfig.DefaultKubecfgAdminLifetime.String()
 	cmd.Flags().StringVar(&options.user, "user", options.user, "Existing user to add to the cluster context. Implies --create-kube-config")
+	cmd.Flags().BoolVar(&options.internal, "internal", options.internal, "Use the cluster's internal DNS name. Implies --create-kube-config")
 	cmd.Flags().BoolVar(&options.AllowKopsDowngrade, "allow-kops-downgrade", options.AllowKopsDowngrade, "Allow an older version of kops to update the cluster than last used")
 	cmd.Flags().StringVar(&options.Phase, "phase", options.Phase, "Subset of tasks to run: "+strings.Join(cloudup.Phases.List(), ", "))
 	cmd.Flags().StringSliceVar(&options.LifecycleOverrides, "lifecycle-overrides", options.LifecycleOverrides, "comma separated list of phase overrides, example: SecurityGroups=Ignore,InternetGateway=ExistsAndWarnIfChanges")
@@ -161,6 +163,11 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, clusterName string,
 		c.CreateKubecfg = true
 	}
 
+	if c.internal && !c.CreateKubecfg {
+		klog.Info("--internal implies --create-kube-config")
+		c.CreateKubecfg = true
+	}
+
 	// direct requires --yes (others do not, because they don't do anything!)
 	if c.Target == cloudup.TargetDirect {
 		if !c.Yes {
@@ -299,7 +306,7 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, clusterName string,
 		firstRun = !hasKubecfg
 
 		klog.Infof("Exporting kubecfg for cluster")
-		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, clientcmd.NewDefaultPathOptions(), c.admin, c.user)
+		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, clientcmd.NewDefaultPathOptions(), c.admin, c.user, c.internal)
 		if err != nil {
 			return nil, err
 		}

docs/cli/kops_export_kubecfg.md

@@ -21,6 +21,9 @@ kops export kubecfg CLUSTERNAME [flags]
   
   # export using a user already existing in the kubeconfig file
   kops export kubecfg kubernetes-cluster.example.com --user my-oidc-user
+  
+  # export using the internal DNS name, bypassing the cloud load balancer
+  kops export kubecfg kubernetes-cluster.example.com --internal
 ```
 
 ### Options
@@ -29,6 +32,7 @@ kops export kubecfg CLUSTERNAME [flags]
       --admin duration[=18h0m0s]   export a cluster admin user credential with the given lifetime and add it to the cluster context
       --all                        export all clusters from the kops state store
   -h, --help                       help for kubecfg
+      --internal                   use the cluster's internal DNS name
       --kubeconfig string          the location of the kubeconfig file to create.
       --user string                add an existing user to the cluster context
 ```

docs/cli/kops_update_cluster.md

@@ -29,6 +29,7 @@ kops update cluster [flags]
       --allow-kops-downgrade          Allow an older version of kops to update the cluster than last used
       --create-kube-config            Will control automatically creating the kube config file on your local filesystem
   -h, --help                          help for cluster
+      --internal                      Use the cluster's internal DNS name. Implies --create-kube-config
       --lifecycle-overrides strings   comma separated list of phase overrides, example: SecurityGroups=Ignore,InternetGateway=ExistsAndWarnIfChanges
       --out string                    Path to write any local output
       --phase string                  Subset of tasks to run: assets, cluster, network, security

docs/cluster_spec.md

@@ -48,7 +48,9 @@ spec:
       idleTimeoutSeconds: 300
 ```
 
-You can use a valid SSL Certificate for your API Server Load Balancer. Currently, only AWS is supported:
+You can use a valid SSL Certificate for your API Server Load Balancer. Currently, only AWS is supported.
+
+Note that when using `sslCertificate`, client certificate authentication, such as with the credentials generated via `kops export kubecfg`, will not work through the load balancer. As of Kops 1.19, a `kubecfg` that bypasses the load balancer may be created with the `--internal` flag to `kops update cluster` or `kops export kubecfg`. Security groups may need to be opened to allow access from the clients to the master instances' port TCP/443, for example by using the `additionalSecurityGroups` field on the master instance groups.
 
 ```yaml
 spec:

pkg/kubeconfig/create_kubecfg.go

@@ -35,12 +35,20 @@ import (
 
 const DefaultKubecfgAdminLifetime = 18 * time.Hour
 
-func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, configAccess clientcmd.ConfigAccess, admin time.Duration, configUser string) (*KubeconfigBuilder, error) {
+func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, configAccess clientcmd.ConfigAccess, admin time.Duration, configUser string, internal bool) (*KubeconfigBuilder, error) {
 	clusterName := cluster.ObjectMeta.Name
 
-	master := cluster.Spec.MasterPublicName
-	if master == "" {
-		master = "api." + clusterName
+	var master string
+	if internal {
+		master = cluster.Spec.MasterInternalName
+		if master == "" {
+			master = "api.internal." + clusterName
+		}
+	} else {
+		master = cluster.Spec.MasterPublicName
+		if master == "" {
+			master = "api." + clusterName
+		}
 	}
 
 	server := "https://" + master
@@ -95,8 +103,8 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se
 	b.Context = clusterName
 	b.Server = server
 
-	// add the CA Cert to the kubeconfig only if we didn't specify a SSL cert for the LB
-	if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" {
+	// add the CA Cert to the kubeconfig only if we didn't specify a SSL cert for the LB or are targeting the internal DNS name
+	if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" || internal {
 		cert, _, _, err := keyStore.FindKeypair(fi.CertificateIDCA)
 		if err != nil {
 			return nil, fmt.Errorf("error fetching CA keypair: %v", err)

pkg/kubeconfig/create_kubecfg_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package kubeconfig
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 	"time"
@@ -78,6 +79,7 @@ func buildMinimalCluster(clusterName string, masterPublicName string) *kops.Clus
 	}
 
 	c.Spec.MasterPublicName = masterPublicName
+	c.Spec.MasterInternalName = fmt.Sprintf("internal.%v", masterPublicName)
 	c.Spec.KubernetesAPIAccess = []string{"0.0.0.0/0"}
 	c.Spec.SSHAccess = []string{"0.0.0.0/0"}
 
@@ -128,6 +130,7 @@ func TestBuildKubecfg(t *testing.T) {
 		configAccess clientcmd.ConfigAccess
 		admin        time.Duration
 		user         string
+		internal     bool
 	}
 
 	publiccluster := buildMinimalCluster("testcluster", "testcluster.test.com")
@@ -157,6 +160,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				DefaultKubecfgAdminLifetime,
 				"",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context: "testcluster",
@@ -183,6 +187,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				0,
 				"myuser",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context: "testcluster",
@@ -209,6 +214,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				0,
 				"",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context: "emptyMasterPublicNameCluster",
@@ -243,6 +249,7 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				0,
 				"",
+				false,
 			},
 			&KubeconfigBuilder{
 				Context: "testgossipcluster.k8s.local",
@@ -252,10 +259,39 @@ func TestBuildKubecfg(t *testing.T) {
 			},
 			false,
 		},
+		{
+			"Test Kube Config Data For internal DNS name with admin",
+			args{
+				publiccluster,
+				fakeKeyStore{
+					FindKeypairFn: func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
+						return fakeCertificate(),
+							fakePrivateKey(),
+							true,
+							nil
+					},
+				},
+				nil,
+				fakeStatusStore{},
+				nil,
+				DefaultKubecfgAdminLifetime,
+				"",
+				true,
+			},
+			&KubeconfigBuilder{
+				Context:    "testcluster",
+				Server:     "https://internal.testcluster.test.com",
+				CACert:     []byte(certData),
+				ClientCert: []byte(certData),
+				ClientKey:  []byte(privatekeyData),
+				User:       "testcluster",
+			},
+			false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := BuildKubecfg(tt.args.cluster, tt.args.keyStore, tt.args.secretStore, tt.args.status, tt.args.configAccess, tt.args.admin, tt.args.user)
+			got, err := BuildKubecfg(tt.args.cluster, tt.args.keyStore, tt.args.secretStore, tt.args.status, tt.args.configAccess, tt.args.admin, tt.args.user, tt.args.internal)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("BuildKubecfg() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -271,7 +307,7 @@ func TestBuildKubecfg(t *testing.T) {
 				tt.want.ClientKey = got.ClientKey
 			}
 			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("BuildKubecfg() = %v, want %v", got, tt.want)
+				t.Errorf("BuildKubecfg() = %+v, want %+v", got, tt.want)
 			}
 		})
 	}