diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index 38437b5c6361f..5c006d914b9c0 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -161,6 +161,9 @@ type CreateClusterOptions struct { // OpenstackLBOctavia is boolean value should we use octavia or old loadbalancer api OpenstackLBOctavia bool + // GCEServiceAccount specifies the service account with which the GCE VM runs + GCEServiceAccount string + // ConfigBase is the location where we will store the configuration, it defaults to the state store ConfigBase string @@ -298,7 +301,6 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringSliceVar(&options.Zones, "zones", options.Zones, "Zones in which to run the cluster") cmd.Flags().StringSliceVar(&options.MasterZones, "master-zones", options.MasterZones, "Zones in which to run masters (must be an odd number)") - cmd.Flags().StringVar(&options.Project, "project", options.Project, "Project to use (must be set on GCE)") cmd.Flags().StringVar(&options.KubernetesVersion, "kubernetes-version", options.KubernetesVersion, "Version of kubernetes to run (defaults to version in channel)") cmd.Flags().StringVar(&options.ContainerRuntime, "container-runtime", options.ContainerRuntime, "Container runtime to use: containerd, docker") @@ -373,6 +375,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringSliceVar(&options.Overrides, "override", options.Overrides, "Directly configure values in the spec") } + // GCE flags + cmd.Flags().StringVar(&options.Project, "project", options.Project, "Project to use (must be set on GCE)") + cmd.Flags().StringVar(&options.GCEServiceAccount, "gce-service-account", options.GCEServiceAccount, "Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account.") + if featureflag.VSphereCloudProvider.Enabled() { // vSphere flags cmd.Flags().StringVar(&options.VSphereServer, "vsphere-server", options.VSphereServer, "vsphere-server is required for vSphere. Set vCenter URL Ex: 10.192.10.30 or myvcenter.io (without https://)") @@ -990,6 +996,17 @@ func RunCreateCluster(f *util.Factory, out io.Writer, c *CreateClusterOptions) e } } + if c.GCEServiceAccount != "" { + klog.Infof("VMs will be configured to use specified Service Account: %v", c.GCEServiceAccount) + cluster.Spec.GCEServiceAccount = c.GCEServiceAccount + } else { + if api.CloudProviderID(cluster.Spec.CloudProvider) == api.CloudProviderGCE { + klog.Warning("VMs will be configured to use the GCE default compute Service Account! This is an anti-pattern") + klog.Warning("Use a pre-create Service Account with the flag: --gce-service-account=account@projectname.iam.gserviceaccount.com") + cluster.Spec.GCEServiceAccount = "default" + } + } + if c.KubernetesVersion != "" { cluster.Spec.KubernetesVersion = c.KubernetesVersion } diff --git a/cmd/kops/create_cluster_integration_test.go b/cmd/kops/create_cluster_integration_test.go index 62ab69d4514be..e28a1f98eb6ea 100644 --- a/cmd/kops/create_cluster_integration_test.go +++ b/cmd/kops/create_cluster_integration_test.go @@ -67,6 +67,11 @@ func TestCreateClusterHAGCE(t *testing.T) { runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_gce", "v1alpha2") } +// TestCreateClusterGCE runs kops create cluster gce.example.com --cloud gce --zones us-test1-a --gce-service-account=test-account@testproject.iam.gserviceaccounts.com +func TestCreateClusterGCE(t *testing.T) { + runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/gce_byo_sa", "v1alpha2") +} + // TestCreateClusterHASharedZones tests kops create cluster when the master count is bigger than the number of zones func TestCreateClusterHASharedZones(t *testing.T) { runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_shared_zones", "v1alpha2") diff --git a/docs/cli/kops_create_cluster.md b/docs/cli/kops_create_cluster.md index 6c02e8b00ee48..b0d4ed5631644 100644 --- a/docs/cli/kops_create_cluster.md +++ b/docs/cli/kops_create_cluster.md @@ -81,6 +81,7 @@ kops create cluster [flags] --dry-run If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest. --encrypt-etcd-storage Generate key in aws kms and use it for encrypt etcd volumes --etcd-storage-type string The default storage type for etc members + --gce-service-account string Service account with which the GCE VM runs. Warning: if not set, VMs will run as default compute service account. -h, --help help for cluster --image string Image to use for all instances. --kubernetes-version string Version of kubernetes to run (defaults to version in channel) diff --git a/docs/releases/1.18-NOTES.md b/docs/releases/1.18-NOTES.md index 870c6c539494a..2232909324037 100644 --- a/docs/releases/1.18-NOTES.md +++ b/docs/releases/1.18-NOTES.md @@ -12,6 +12,8 @@ * New clusters in GCE are configured to run the [metadata-proxy](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/metadata-proxy) by default. The proxy runs as a DaemonSet and lands on nodes with the nodeLabel `cloud.google.com/metadata-proxy-ready: "true"`. If you want to enable metadata-proxy on an existing cluster/instance group, add that nodeLabel to your instancegroup specs (`kops edit ig ...`) and run `kops update cluster`. When the changes are applied, the proxy will roll out to those targeted nodes. +* GCE has a new flag: `--gce-service-account`. This takes the email of an existing GCP service account and launches the instances with it. This setting applies to the whole cluster (ie: it is not currently designed to support Instance Groups with different service accounts). If you do not specify a service account during cluster creation, the default compute service account will be used which matches the prior behavior. + # Breaking changes * Terraform users on AWS may need to rename some resources in their state file in order to prepare for Terraform 0.12 support. See Required Actions below. diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 9a1a66792ca2c..1d0e11f013eff 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -738,6 +738,10 @@ spec: type: array type: object type: array + gceServiceAccount: + description: GCEServiceAccount specifies the service account with which + the GCE VM runs + type: string gossipConfig: description: GossipConfig for the cluster assuming the use of gossip DNS diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 6bfeed8622474..5896b2c86fb45 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -191,6 +191,8 @@ type ClusterSpec struct { SysctlParameters []string `json:"sysctlParameters,omitempty"` // RollingUpdate defines the default rolling-update settings for instance groups RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` + // GCEServiceAccount specifies the service account with which the GCE VM runs + GCEServiceAccount string `json:"gceServiceAccount,omitempty"` } // NodeAuthorizationSpec is used to node authorization diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 769bbd4af5995..af05ba7d7178a 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -189,6 +189,8 @@ type ClusterSpec struct { SysctlParameters []string `json:"sysctlParameters,omitempty"` // RollingUpdate defines the default rolling-update settings for instance groups RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` + // GCEServiceAccount specifies the service account with which the GCE VM runs + GCEServiceAccount string `json:"gceServiceAccount,omitempty"` } // NodeAuthorizationSpec is used to node authorization diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 1769d3ed1769c..fd203e5c10db7 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1995,6 +1995,7 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * } else { out.RollingUpdate = nil } + out.GCEServiceAccount = in.GCEServiceAccount return nil } @@ -2318,6 +2319,7 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec, } else { out.RollingUpdate = nil } + out.GCEServiceAccount = in.GCEServiceAccount return nil } diff --git a/pkg/model/gcemodel/autoscalinggroup.go b/pkg/model/gcemodel/autoscalinggroup.go index 806f20d4d2303..b36e745ce8641 100644 --- a/pkg/model/gcemodel/autoscalinggroup.go +++ b/pkg/model/gcemodel/autoscalinggroup.go @@ -35,6 +35,7 @@ const ( DefaultVolumeType = "pd-standard" ) +// TODO: rework these parts to be more GCE native. ie: Managed Instance Groups > ASGs // AutoscalingGroupModelBuilder configures AutoscalingGroup objects type AutoscalingGroupModelBuilder struct { *GCEModelContext @@ -70,7 +71,6 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { } namePrefix := gce.LimitedLengthName(name, gcetasks.InstanceTemplateNamePrefixMaxLength) - t := &gcetasks.InstanceTemplate{ Name: s(name), NamePrefix: s(namePrefix), @@ -89,7 +89,6 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { "monitoring", "logging-write", }, - Metadata: map[string]*fi.ResourceHolder{ "startup-script": startupScript, //"config": resources/config.yaml $nodeset.Name @@ -121,6 +120,7 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { switch ig.Spec.Role { case kops.InstanceGroupRoleMaster: // Grant DNS permissions + // TODO: migrate to IAM permissions instead of oldschool scopes? t.Scopes = append(t.Scopes, "https://www.googleapis.com/auth/ndev.clouddns.readwrite") t.Tags = append(t.Tags, b.GCETagForRole(kops.InstanceGroupRoleMaster)) @@ -139,6 +139,17 @@ func (b *AutoscalingGroupModelBuilder) Build(c *fi.ModelBuilderContext) error { t.CanIPForward = fi.Bool(true) } + if b.Cluster.Spec.GCEServiceAccount != "" { + klog.Infof("VMs using Service Account: %v", b.Cluster.Spec.GCEServiceAccount) + // b.Cluster.Spec.GCEServiceAccount = c.GCEServiceAccount + } else { + klog.Warning("VMs will be configured to use the GCE default compute Service Account! This is an anti-pattern") + klog.Warning("Use a pre-created Service Account with the flag: --gce-service-account=account@projectname.iam.gserviceaccount.com") + b.Cluster.Spec.GCEServiceAccount = "default" + } + + klog.Infof("gsa: %v", b.Cluster.Spec.GCEServiceAccount) + t.ServiceAccounts = []string{b.Cluster.Spec.GCEServiceAccount} //labels, err := b.CloudTagsForInstanceGroup(ig) //if err != nil { // return fmt.Errorf("error building cloud tags: %v", err) diff --git a/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml new file mode 100644 index 0000000000000..2f06093502259 --- /dev/null +++ b/tests/integration/create_cluster/gce_byo_sa/expected-v1alpha2.yaml @@ -0,0 +1,98 @@ +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + name: gce.example.com +spec: + api: + dns: {} + authorization: + rbac: {} + channel: stable + cloudProvider: gce + configBase: memfs://tests/gce.example.com + containerRuntime: docker + etcdClusters: + - cpuRequest: 200m + etcdMembers: + - instanceGroup: master-us-test1-a + name: a + memoryRequest: 100Mi + name: main + - cpuRequest: 100m + etcdMembers: + - instanceGroup: master-us-test1-a + name: a + memoryRequest: 100Mi + name: events + gceServiceAccount: test-account@testproject.iam.gserviceaccount.com + iam: + allowContainerRegistry: true + legacy: false + kubelet: + anonymousAuth: false + kubernetesApiAccess: + - 0.0.0.0/0 + kubernetesVersion: v1.15.6-beta.1 + masterPublicName: api.gce.example.com + networking: + kubenet: {} + nonMasqueradeCIDR: 100.64.0.0/10 + project: testproject + sshAccess: + - 0.0.0.0/0 + subnets: + - name: us-test1 + region: us-test1 + type: Public + topology: + dns: + type: Public + masters: public + nodes: public + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: gce.example.com + name: master-us-test1-a +spec: + image: cos-cloud/cos-stable-65-10323-99-0 + machineType: n1-standard-1 + maxSize: 1 + minSize: 1 + nodeLabels: + cloud.google.com/metadata-proxy-ready: "true" + kops.k8s.io/instancegroup: master-us-test1-a + role: Master + subnets: + - us-test1 + zones: + - us-test1-a + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + creationTimestamp: "2017-01-01T00:00:00Z" + labels: + kops.k8s.io/cluster: gce.example.com + name: nodes +spec: + image: cos-cloud/cos-stable-65-10323-99-0 + machineType: n1-standard-2 + maxSize: 2 + minSize: 2 + nodeLabels: + cloud.google.com/metadata-proxy-ready: "true" + kops.k8s.io/instancegroup: nodes + role: Node + subnets: + - us-test1 + zones: + - us-test1-a diff --git a/tests/integration/create_cluster/gce_byo_sa/options.yaml b/tests/integration/create_cluster/gce_byo_sa/options.yaml new file mode 100644 index 0000000000000..56adba68a4cff --- /dev/null +++ b/tests/integration/create_cluster/gce_byo_sa/options.yaml @@ -0,0 +1,9 @@ +ClusterName: gce.example.com +Zones: +- us-test1-a +MasterZones: +- us-test1-a +Cloud: gce +KubernetesVersion: v1.15.6-beta.1 +Project: testproject +GCEServiceAccount: test-account@testproject.iam.gserviceaccount.com \ No newline at end of file diff --git a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml index 2efae9c1aecb0..0ee09fe0a1884 100644 --- a/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml +++ b/tests/integration/create_cluster/ha_gce/expected-v1alpha2.yaml @@ -33,6 +33,7 @@ spec: name: c memoryRequest: 100Mi name: events + gceServiceAccount: default iam: allowContainerRegistry: true legacy: false diff --git a/tests/integration/update_cluster/ha_gce/in-v1alpha2.yaml b/tests/integration/update_cluster/ha_gce/in-v1alpha2.yaml index 95864074fc78b..6f954d061a09f 100644 --- a/tests/integration/update_cluster/ha_gce/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/ha_gce/in-v1alpha2.yaml @@ -28,6 +28,7 @@ spec: - instanceGroup: master-us-test1-c name: "3" name: events + gceServiceAccount: default iam: legacy: false kubelet: diff --git a/tests/integration/update_cluster/ha_gce/kubernetes.tf b/tests/integration/update_cluster/ha_gce/kubernetes.tf index eb9525e356681..c874bc3231f9e 100644 --- a/tests/integration/update_cluster/ha_gce/kubernetes.tf +++ b/tests/integration/update_cluster/ha_gce/kubernetes.tf @@ -398,6 +398,7 @@ resource "google_compute_instance_template" "master-us-test1-a-ha-gce-example-co machine_type = "n1-standard-1" service_account = { + email = "default" scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"] } @@ -439,6 +440,7 @@ resource "google_compute_instance_template" "master-us-test1-b-ha-gce-example-co machine_type = "n1-standard-1" service_account = { + email = "default" scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"] } @@ -480,6 +482,7 @@ resource "google_compute_instance_template" "master-us-test1-c-ha-gce-example-co machine_type = "n1-standard-1" service_account = { + email = "default" scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"] } @@ -521,6 +524,7 @@ resource "google_compute_instance_template" "nodes-ha-gce-example-com" { machine_type = "n1-standard-2" service_account = { + email = "default" scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_only"] } diff --git a/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml b/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml index ddd16166e6781..0a780d5132807 100644 --- a/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml +++ b/tests/integration/update_cluster/minimal_gce/in-v1alpha2.yaml @@ -20,6 +20,7 @@ spec: - instanceGroup: master-us-test1-a name: "1" name: events + gceServiceAccount: default iam: legacy: false kubelet: diff --git a/tests/integration/update_cluster/minimal_gce/kubernetes.tf b/tests/integration/update_cluster/minimal_gce/kubernetes.tf index b1da433d047d1..3addd97fc69f6 100644 --- a/tests/integration/update_cluster/minimal_gce/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce/kubernetes.tf @@ -298,6 +298,7 @@ resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-examp machine_type = "n1-standard-1" service_account = { + email = "default" scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_write", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"] } @@ -339,6 +340,7 @@ resource "google_compute_instance_template" "nodes-minimal-gce-example-com" { machine_type = "n1-standard-2" service_account = { + email = "default" scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/devstorage.read_only"] } diff --git a/upup/pkg/fi/cloudup/gcetasks/instance.go b/upup/pkg/fi/cloudup/gcetasks/instance.go index a3e76878f53f2..37a4759a6345c 100644 --- a/upup/pkg/fi/cloudup/gcetasks/instance.go +++ b/upup/pkg/fi/cloudup/gcetasks/instance.go @@ -35,11 +35,12 @@ type Instance struct { Name *string Lifecycle *fi.Lifecycle - Network *Network - Tags []string - Preemptible *bool - Image *string - Disks map[string]*Disk + Network *Network + Tags []string + Preemptible *bool + Image *string + Disks map[string]*Disk + ServiceAccount *string CanIPForward *bool IPAddress *Address @@ -253,17 +254,18 @@ func (e *Instance) mapToGCE(project string, ipAddressResolver func(*Address) (*s } var serviceAccounts []*compute.ServiceAccount - if e.Scopes != nil { - var scopes []string - for _, s := range e.Scopes { - s = scopeToLongForm(s) - - scopes = append(scopes, s) + if e.ServiceAccount != nil { + if e.Scopes != nil { + var scopes []string + for _, s := range e.Scopes { + s = scopeToLongForm(s) + scopes = append(scopes, s) + } + serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{ + Email: fi.StringValue(e.ServiceAccount), + Scopes: scopes, + }) } - serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{ - Email: "default", - Scopes: scopes, - }) } var metadataItems []*compute.MetadataItems diff --git a/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go b/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go index 934af39e48a53..9a24f0fe613fc 100644 --- a/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go +++ b/upup/pkg/fi/cloudup/gcetasks/instancetemplate.go @@ -58,7 +58,8 @@ type InstanceTemplate struct { Subnet *Subnet AliasIPRanges map[string]string - Scopes []string + Scopes []string + ServiceAccounts []string Metadata map[string]*fi.ResourceHolder MachineType *string @@ -269,19 +270,32 @@ func (e *InstanceTemplate) mapToGCE(project string, region string) (*compute.Ins } networkInterfaces = append(networkInterfaces, ni) - var serviceAccounts []*compute.ServiceAccount + scopes := make([]string, 0) if e.Scopes != nil { - var scopes []string for _, s := range e.Scopes { s = scopeToLongForm(s) - scopes = append(scopes, s) } - serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{ - Email: "default", + } + serviceAccounts := []*compute.ServiceAccount{ + { + Email: e.ServiceAccounts[0], Scopes: scopes, - }) + }, } + // if e.ServiceAccounts != nil { + // for _, s := range e.ServiceAccounts { + // serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{ + // Email: s, + // Scopes: scopes, + // }) + // } + // } else { + // serviceAccounts = append(serviceAccounts, &compute.ServiceAccount{ + // Email: "default", + // Scopes: scopes, + // }) + // } var metadataItems []*compute.MetadataItems for key, r := range e.Metadata { @@ -423,6 +437,7 @@ type terraformInstanceCommon struct { } type terraformServiceAccount struct { + Email string `json:"email"` Scopes []string `json:"scopes"` } @@ -514,16 +529,23 @@ func (t *terraformInstanceCommon) AddMetadata(target *terraform.TerraformTarget, } func (t *terraformInstanceCommon) AddServiceAccounts(serviceAccounts []*compute.ServiceAccount) { - for _, g := range serviceAccounts { - for _, scope := range g.Scopes { - if t.ServiceAccount == nil { - t.ServiceAccount = &terraformServiceAccount{} - } - t.ServiceAccount.Scopes = append(t.ServiceAccount.Scopes, scope) + // there's an inconsistency here- GCP only lets you have one service account per VM + // terraform gets it right, but the golang api doesn't. womp womp :( + if len(serviceAccounts) != 1 { + klog.Fatal("Instances can only have 1 service account assigned.") + } else { + klog.Infof("adding csa: %v", serviceAccounts[0].Email) + csa := serviceAccounts[0] + tsa := &terraformServiceAccount{ + Email: csa.Email, + Scopes: csa.Scopes, } + // for _, scope := range csa.Scopes { + // tsa.Scopes = append(tsa.Scopes, scope) + // } + t.ServiceAccount = tsa } } - func (_ *InstanceTemplate) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *InstanceTemplate) error { project := t.Project