Adds a gce-service-account flag so you BYO service-account

cmd/kops/create_cluster.go

@@ -161,6 +161,9 @@ type CreateClusterOptions struct {
 	// OpenstackLBOctavia is boolean value should we use octavia or old loadbalancer api
 	OpenstackLBOctavia bool
 
+	// GCEServiceAccount specifies the service account with which the GCE VM runs
+	GCEServiceAccount string
+
 	// ConfigBase is the location where we will store the configuration, it defaults to the state store
 	ConfigBase string
 
@@ -298,7 +301,6 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringSliceVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster")
 	cmd.Flags().StringSliceVar(&options.MasterZones, "master-zones", options.MasterZones, "Zones in which to run masters (must be an odd number)")
 
-	cmd.Flags().StringVar(&options.Project, "project", options.Project, "Project to use (must be set on GCE)")
 	cmd.Flags().StringVar(&options.KubernetesVersion, "kubernetes-version", options.KubernetesVersion, "Version of kubernetes to run (defaults to version in channel)")
 
 	cmd.Flags().StringVar(&options.ContainerRuntime, "container-runtime", options.ContainerRuntime, "Container runtime to use: containerd, docker")
@@ -373,6 +375,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 		cmd.Flags().StringSliceVar(&options.Overrides, "override", options.Overrides, "Directly configure values in the spec")
 	}
 
+	// GCE flags
+	cmd.Flags().StringVar(&options.Project, "project", options.Project, "Project to use (must be set on GCE)")
+	cmd.Flags().StringVar(&options.GCEServiceAccount, "gce-service-account", options.GCEServiceAccount, "Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account.")
+
 	if featureflag.VSphereCloudProvider.Enabled() {
 		// vSphere flags
 		cmd.Flags().StringVar(&options.VSphereServer, "vsphere-server", options.VSphereServer, "vsphere-server is required for vSphere. Set vCenter URL Ex: 10.192.10.30 or myvcenter.io (without https://)")
@@ -990,6 +996,17 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e
 		}
 	}
 
+	if c.GCEServiceAccount != "" {
+		klog.Infof("VMs will be configured to use specified Service Account: %v", c.GCEServiceAccount)
+		cluster.Spec.GCEServiceAccount = c.GCEServiceAccount
+	} else {
+		if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderGCE {
+			klog.Warning("VMs will be configured to use the GCE default compute Service Account! This is an anti-pattern")
+			klog.Warning("Use a pre-create Service Account with the flag: --gce-service-account=account@projectname.iam.gserviceaccount.com")
+			cluster.Spec.GCEServiceAccount = "default"
+		}
+	}
+
 	if c.KubernetesVersion != "" {
 		cluster.Spec.KubernetesVersion = c.KubernetesVersion
 	}

cmd/kops/create_cluster_integration_test.go

@@ -67,6 +67,11 @@ func TestCreateClusterHAGCE(t *testing.T) {
 	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_gce", "v1alpha2")
 }
 
+// TestCreateClusterGCE runs kops create cluster gce.example.com --cloud gce --zones us-test1-a --gce-service-account=test-account@testproject.iam.gserviceaccounts.com
+func TestCreateClusterGCE(t *testing.T) {
+	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/gce_byo_sa", "v1alpha2")
+}
+
 // TestCreateClusterHASharedZones tests kops create cluster when the master count is bigger than the number of zones
 func TestCreateClusterHASharedZones(t *testing.T) {
 	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_shared_zones", "v1alpha2")

docs/cli/kops_create_cluster.md

@@ -81,6 +81,7 @@ kops create cluster [flags]
       --dry-run                          If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.
       --encrypt-etcd-storage             Generate key in aws kms and use it for encrypt etcd volumes
       --etcd-storage-type string         The default storage type for etc members
+      --gce-service-account string       Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account.
   -h, --help                             help for cluster
       --image string                     Image to use for all instances.
       --kubernetes-version string        Version of kubernetes to run (defaults to version in channel)

docs/releases/1.18-NOTES.md

@@ -12,6 +12,8 @@
 
 * New clusters in GCE are configured to run the [metadata-proxy](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/metadata-proxy) by default. The proxy runs as a DaemonSet and lands on nodes with the nodeLabel `cloud.google.com/metadata-proxy-ready: "true"`. If you want to enable metadata-proxy on an existing cluster/instance group, add that nodeLabel to your instancegroup specs (`kops edit ig ...`) and run `kops update cluster`. When the changes are applied, the proxy will roll out to those targeted nodes.
 
+* GCE has a new flag: `--gce-service-account`. This takes the email of an existing GCP service account and launches the instances with it. This setting applies to the whole cluster (ie: it is not currently designed to support Instance Groups with different service accounts). If you do not specify a service account during cluster creation, the default compute service account will be used which matches the prior behavior.
+
 # Breaking changes
 
 * Terraform users on AWS may need to rename some resources in their state file in order to prepare for Terraform 0.12 support. See Required Actions below.

k8s/crds/kops.k8s.io_clusters.yaml

@@ -738,6 +738,10 @@ spec:
                     type: array
                 type: object
               type: array
+            gceServiceAccount:
+              description: GCEServiceAccount specifies the service account with which
+                the GCE VM runs
+              type: string
             gossipConfig:
               description: GossipConfig for the cluster assuming the use of gossip
                 DNS

pkg/apis/kops/cluster.go

@@ -191,6 +191,8 @@ type ClusterSpec struct {
 	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 	// RollingUpdate defines the default rolling-update settings for instance groups
 	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+	// GCEServiceAccount specifies the service account with which the GCE VM runs
+	GCEServiceAccount string `json:"gceServiceAccount,omitempty"`
 }
 
 // NodeAuthorizationSpec is used to node authorization

pkg/apis/kops/v1alpha2/cluster.go

@@ -189,6 +189,8 @@ type ClusterSpec struct {
 	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 	// RollingUpdate defines the default rolling-update settings for instance groups
 	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+	// GCEServiceAccount specifies the service account with which the GCE VM runs
+	GCEServiceAccount string `json:"gceServiceAccount,omitempty"`
 }
 
 // NodeAuthorizationSpec is used to node authorization

pkg/model/gcemodel/autoscalinggroup.go

@@ -35,6 +35,7 @@ const (
 	DefaultVolumeType = "pd-standard"
 )
 
+// TODO: rework these parts to be more GCE native. ie: Managed Instance Groups > ASGs
 // AutoscalingGroupModelBuilder configures AutoscalingGroup objects
 type AutoscalingGroupModelBuilder struct {
 	*GCEModelContext
@@ -70,7 +71,6 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 			}
 
 			namePrefix := gce.LimitedLengthName(name, gcetasks.InstanceTemplateNamePrefixMaxLength)
-
 			t := &gcetasks.InstanceTemplate{
 				Name:           s(name),
 				NamePrefix:     s(namePrefix),
@@ -89,7 +89,6 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 					"monitoring",
 					"logging-write",
 				},
-
 				Metadata: map[string]*fi.ResourceHolder{
 					"startup-script": startupScript,
 					//"config": resources/config.yaml $nodeset.Name
@@ -121,6 +120,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 			switch ig.Spec.Role {
 			case kops.InstanceGroupRoleMaster:
 				// Grant DNS permissions
+				// TODO: migrate to IAM permissions instead of oldschool scopes?
 				t.Scopes = append(t.Scopes, "https://www.googleapis.com/auth/ndev.clouddns.readwrite")
 				t.Tags = append(t.Tags, b.GCETagForRole(kops.InstanceGroupRoleMaster))
 
@@ -139,6 +139,17 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error {
 				t.CanIPForward = fi.Bool(true)
 			}
 
+			if b.Cluster.Spec.GCEServiceAccount != "" {
+				klog.Infof("VMs using Service Account: %v", b.Cluster.Spec.GCEServiceAccount)
+				// b.Cluster.Spec.GCEServiceAccount = c.GCEServiceAccount
+			} else {
+				klog.Warning("VMs will be configured to use the GCE default compute Service Account! This is an anti-pattern")
+				klog.Warning("Use a pre-created Service Account with the flag: --gce-service-account=account@projectname.iam.gserviceaccount.com")
+				b.Cluster.Spec.GCEServiceAccount = "default"
+			}
+
+			klog.Infof("gsa: %v", b.Cluster.Spec.GCEServiceAccount)
+			t.ServiceAccounts = []string{b.Cluster.Spec.GCEServiceAccount}
 			//labels, err := b.CloudTagsForInstanceGroup(ig)
 			//if err != nil {
 			//	return fmt.Errorf("error building cloud tags: %v", err)

tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml

@@ -0,0 +1,98 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  name: gce.example.com
+spec:
+  api:
+    dns: {}
+  authorization:
+    rbac: {}
+  channel: stable
+  cloudProvider: gce
+  configBase: memfs://tests/gce.example.com
+  containerRuntime: docker
+  etcdClusters:
+  - cpuRequest: 200m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    memoryRequest: 100Mi
+    name: main
+  - cpuRequest: 100m
+    etcdMembers:
+    - instanceGroup: master-us-test1-a
+      name: a
+    memoryRequest: 100Mi
+    name: events
+  gceServiceAccount: test-account@testproject.iam.gserviceaccount.com
+  iam:
+    allowContainerRegistry: true
+    legacy: false
+  kubelet:
+    anonymousAuth: false
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  kubernetesVersion: v1.15.6-beta.1
+  masterPublicName: api.gce.example.com
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  project: testproject
+  sshAccess:
+  - 0.0.0.0/0
+  subnets:
+  - name: us-test1
+    region: us-test1
+    type: Public
+  topology:
+    dns:
+      type: Public
+    masters: public
+    nodes: public
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: gce.example.com
+  name: master-us-test1-a
+spec:
+  image: cos-cloud/cos-stable-65-10323-99-0
+  machineType: n1-standard-1
+  maxSize: 1
+  minSize: 1
+  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
+    kops.k8s.io/instancegroup: master-us-test1-a
+  role: Master
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2017-01-01T00:00:00Z"
+  labels:
+    kops.k8s.io/cluster: gce.example.com
+  name: nodes
+spec:
+  image: cos-cloud/cos-stable-65-10323-99-0
+  machineType: n1-standard-2
+  maxSize: 2
+  minSize: 2
+  nodeLabels:
+    cloud.google.com/metadata-proxy-ready: "true"
+    kops.k8s.io/instancegroup: nodes
+  role: Node
+  subnets:
+  - us-test1
+  zones:
+  - us-test1-a

tests/integration/create_cluster/gce_byo_sa/options.yaml

@@ -0,0 +1,9 @@
+ClusterName: gce.example.com
+Zones:
+- us-test1-a
+MasterZones:
+- us-test1-a
+Cloud: gce
+KubernetesVersion: v1.15.6-beta.1
+Project: testproject
+GCEServiceAccount: test-account@testproject.iam.gserviceaccount.com

tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml

@@ -33,6 +33,7 @@ spec:
       name: c
     memoryRequest: 100Mi
     name: events
+  gceServiceAccount: default
   iam:
     allowContainerRegistry: true
     legacy: false

tests/integration/update_cluster/ha_gce/in-v1alpha2.yaml

@@ -28,6 +28,7 @@ spec:
     - instanceGroup: master-us-test1-c
       name: "3"
     name: events
+  gceServiceAccount: default
   iam:
     legacy: false
   kubelet:

tests/integration/update_cluster/ha_gce/kubernetes.tf

@@ -398,6 +398,7 @@ resource "google_compute_instance_template" "master-us-test1-a-ha-gce-example-co
   machine_type   = "n1-standard-1"
 
   service_account = {
+    email  = "default"
     scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
   }
 
@@ -439,6 +440,7 @@ resource "google_compute_instance_template" "master-us-test1-b-ha-gce-example-co
   machine_type   = "n1-standard-1"
 
   service_account = {
+    email  = "default"
     scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
   }
 
@@ -480,6 +482,7 @@ resource "google_compute_instance_template" "master-us-test1-c-ha-gce-example-co
   machine_type   = "n1-standard-1"
 
   service_account = {
+    email  = "default"
     scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
   }
 
@@ -521,6 +524,7 @@ resource "google_compute_instance_template" "nodes-ha-gce-example-com" {
   machine_type   = "n1-standard-2"
 
   service_account = {
+    email  = "default"
     scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_only"]
   }
 

tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml

@@ -20,6 +20,7 @@ spec:
     - instanceGroup: master-us-test1-a
       name: "1"
     name: events
+  gceServiceAccount: default
   iam:
     legacy: false
   kubelet:

tests/integration/update_cluster/minimal_gce/kubernetes.tf

@@ -298,6 +298,7 @@ resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-examp
   machine_type   = "n1-standard-1"
 
   service_account = {
+    email  = "default"
     scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
   }
 
@@ -339,6 +340,7 @@ resource "google_compute_instance_template" "nodes-minimal-gce-example-com" {
   machine_type   = "n1-standard-2"
 
   service_account = {
+    email  = "default"
     scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_only"]
   }
 

upup/pkg/fi/cloudup/gcetasks/instance.go

@@ -35,11 +35,12 @@ type Instance struct {
 	Name      *string
 	Lifecycle *fi.Lifecycle
 
-	Network     *Network
-	Tags        []string
-	Preemptible *bool
-	Image       *string
-	Disks       map[string]*Disk
+	Network        *Network
+	Tags           []string
+	Preemptible    *bool
+	Image          *string
+	Disks          map[string]*Disk
+	ServiceAccount *string
 
 	CanIPForward *bool
 	IPAddress    *Address
@@ -253,17 +254,18 @@ func (e *Instance) mapToGCE(project string, ipAddressResolver func(*Address) (*s
 	}
 
 	var serviceAccounts []*compute.ServiceAccount
-	if e.Scopes != nil {
-		var scopes []string
-		for _, s := range e.Scopes {
-			s = scopeToLongForm(s)
-
-			scopes = append(scopes, s)
+	if e.ServiceAccount != nil {
+		if e.Scopes != nil {
+			var scopes []string
+			for _, s := range e.Scopes {
+				s = scopeToLongForm(s)
+				scopes = append(scopes, s)
+			}
+			serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{
+				Email:  fi.StringValue(e.ServiceAccount),
+				Scopes: scopes,
+			})
 		}
-		serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{
-			Email:  "default",
-			Scopes: scopes,
-		})
 	}
 
 	var metadataItems []*compute.MetadataItems