feat: Create a workspace for 0-bootstrap (#866)

* create workspace for 0-bootstrap * update integration tests * move assignment of roles to the bootstrap service account in the CI/CD project to the sa.tf file
terraform-google-modules · Nov 9, 2022 · 6e9c575 · 6e9c575
1 parent 9c17c13
commit 6e9c575
Show file tree

Hide file tree

Showing 8 changed files with 134 additions and 54 deletions.
diff --git a/0-bootstrap/README.md b/0-bootstrap/README.md
@@ -203,19 +203,49 @@ your current Jenkins manager (controller) environment.
    ```
 
 1. (Optional) Run `terraform plan` to verify that state is configured correctly. You should see no changes from the previous state.
-1. Save `0-bootstrap` Terraform configuration to `gcp-bootstrap` source repository:
+1. Clone the policy repo and copy contents of policy-library to new repo. Clone the repo at the same level of the `terraform-example-foundation` folder, the next instructions assume that layout.
 
    ```bash
    cd ../..
 
+   gcloud source repos clone gcp-policies --project=${cloudbuild_project_id}
+
+   cd gcp-policies
+   git checkout -b main
+   cp -RT ../terraform-example-foundation/policy-library/ .
+   ```
+
+1. Commit changes and push your main branch to the policy repo.
+
+   ```bash
+   git add .
+   git commit -m 'Initialize policy library'
+   git push --set-upstream origin main
+   ```
+
+1. Navigate out of the repo.
+
+   ```bash
+   cd ..
+   ```
+
+1. Save `0-bootstrap` Terraform configuration to `gcp-bootstrap` source repository:
+
+   ```bash
    gcloud source repos clone gcp-bootstrap --project=${cloudbuild_project_id}
+
    cd gcp-bootstrap
-   git checkout -b production
-   cp -RT ../terraform-example-foundation/0-bootstrap/ .
+   git checkout -b plan
+   mkdir -p envs/shared
+
+   cp -RT ../terraform-example-foundation/0-bootstrap/ ./envs/shared
+   cp ../terraform-example-foundation/build/cloudbuild-tf-* .
+   cp ../terraform-example-foundation/build/tf-wrapper.sh .
+   chmod 755 ./tf-wrapper.sh
 
    git add .
    git commit -m 'Initialize bootstrap'
-   git push --set-upstream origin production
+   git push --set-upstream origin plan
    ```
 
 1. You can now move to the instructions in the [1-org](../1-org/README.md) step.
@@ -256,6 +286,7 @@ Each step has instructions for this change.
 
 | Name | Description |
 |------|-------------|
+| bootstrap\_step\_terraform\_service\_account\_email | Bootstrap Step Terraform Account |
 | cloud\_builder\_artifact\_repo | GAR Repo created to store TF Cloud Builder images. |
 | cloudbuild\_project\_id | Project where CloudBuild configuration and terraform container image will reside. |
 | common\_config | Common configuration data to be used in other steps. |

diff --git a/0-bootstrap/cb.tf b/0-bootstrap/cb.tf
@@ -20,33 +20,42 @@ locals {
   // The version of the terraform docker image to be used in the workspace builds
   docker_tag_version_terraform = "v1"
 
+  cicd_project_id = module.tf_source.cloudbuild_project_id
+
+  default_state_bucket_self_link      = "${local.bucket_self_link_prefix}${module.seed_bootstrap.gcs_bucket_tfstate}"
+  gcp_projects_state_bucket_self_link = module.gcp_projects_state_bucket.bucket.self_link
+
   cb_config = {
+    "bootstrap" = {
+      source       = "gcp-bootstrap",
+      state_bucket = local.default_state_bucket_self_link,
+    },
     "org" = {
-      source        = "gcp-org",
-      create_bucket = false,
+      source       = "gcp-org",
+      state_bucket = local.default_state_bucket_self_link,
     },
     "env" = {
-      source        = "gcp-environments",
-      create_bucket = false,
+      source       = "gcp-environments",
+      state_bucket = local.default_state_bucket_self_link,
     },
     "net" = {
-      source        = "gcp-networks",
-      create_bucket = false,
+      source       = "gcp-networks",
+      state_bucket = local.default_state_bucket_self_link,
     },
     "proj" = {
-      source        = "gcp-projects",
-      create_bucket = true,
+      source       = "gcp-projects",
+      state_bucket = local.gcp_projects_state_bucket_self_link,
     },
   }
+
   cloud_source_repos = [for v in local.cb_config : v.source]
   cloudbuilder_repo  = "tf-cloudbuilder"
   base_cloud_source_repos = [
     "gcp-policies",
     "gcp-bootstrap",
     local.cloudbuilder_repo,
   ]
-  gar_repository              = split("/", module.tf_cloud_builder.artifact_repo)[length(split("/", module.tf_cloud_builder.artifact_repo)) - 1]
-  projects_gcs_bucket_tfstate = split("/", module.tf_workspace["proj"].state_bucket)[length(split("/", module.tf_workspace["proj"].state_bucket)) - 1]
+  gar_repository = split("/", module.tf_cloud_builder.artifact_repo)[length(split("/", module.tf_cloud_builder.artifact_repo)) - 1]
 }
 
 resource "random_string" "suffix" {
@@ -55,6 +64,16 @@ resource "random_string" "suffix" {
   upper   = false
 }
 
+module "gcp_projects_state_bucket" {
+  source  = "terraform-google-modules/cloud-storage/google//modules/simple_bucket"
+  version = "~> 3.2"
+
+  name          = "bkt-b-gcp-projects-tfstate-${module.seed_bootstrap.seed_project_id}"
+  project_id    = module.seed_bootstrap.seed_project_id
+  location      = var.default_region
+  force_destroy = var.bucket_force_destroy
+}
+
 module "tf_source" {
   source  = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_source"
   version = "~> 6.2"
@@ -155,14 +174,14 @@ module "tf_workspace" {
 
   project_id                = module.tf_source.cloudbuild_project_id
   location                  = var.default_region
-  state_bucket_self_link    = "${local.bucket_self_link_prefix}${module.seed_bootstrap.gcs_bucket_tfstate}"
+  state_bucket_self_link    = local.cb_config[each.key].state_bucket
   cloudbuild_plan_filename  = "cloudbuild-tf-plan.yaml"
   cloudbuild_apply_filename = "cloudbuild-tf-apply.yaml"
   tf_repo_uri               = module.tf_source.csr_repos[local.cb_config[each.key].source].url
   cloudbuild_sa             = google_service_account.terraform-env-sa[each.key].id
   create_cloudbuild_sa      = false
   diff_sa_project           = true
-  create_state_bucket       = local.cb_config[each.key].create_bucket
+  create_state_bucket       = false
   buckets_force_destroy     = var.bucket_force_destroy
 
   substitutions = {

diff --git a/0-bootstrap/jenkins.tf.example b/0-bootstrap/jenkins.tf.example
@@ -16,6 +16,8 @@
 
 locals {
   sa_names = { for k, v in google_service_account.terraform-env-sa : k => v.id }
+
+  cicd_project_id = module.jenkins_bootstrap.cicd_project_id
 }
 
 module "jenkins_bootstrap" {

diff --git a/0-bootstrap/main.tf b/0-bootstrap/main.tf
@@ -22,6 +22,7 @@ locals {
   // in the list "org_project_creators" will have the Project Creator role,
   // so the granular service accounts for each step need to be added to the list.
   step_terraform_sa = [
+    "serviceAccount:${google_service_account.terraform-env-sa["bootstrap"].email}",
     "serviceAccount:${google_service_account.terraform-env-sa["org"].email}",
     "serviceAccount:${google_service_account.terraform-env-sa["env"].email}",
     "serviceAccount:${google_service_account.terraform-env-sa["net"].email}",

diff --git a/0-bootstrap/outputs.tf b/0-bootstrap/outputs.tf
@@ -19,6 +19,11 @@ output "seed_project_id" {
   value       = module.seed_bootstrap.seed_project_id
 }
 
+output "bootstrap_step_terraform_service_account_email" {
+  description = "Bootstrap Step Terraform Account"
+  value       = google_service_account.terraform-env-sa["bootstrap"].email
+}
+
 output "projects_step_terraform_service_account_email" {
   description = "Projects Step Terraform Account"
   value       = google_service_account.terraform-env-sa["proj"].email
@@ -94,7 +99,7 @@ output "gcs_bucket_cloudbuild_artifacts" {
 
 output "projects_gcs_bucket_tfstate" {
   description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project."
-  value       = local.projects_gcs_bucket_tfstate
+  value       = module.gcp_projects_state_bucket.bucket.name
 }
 
 output "cloud_builder_artifact_repo" {

diff --git a/0-bootstrap/sa.tf b/0-bootstrap/sa.tf
@@ -20,17 +20,23 @@ locals {
   parent_id   = var.parent_folder == "" ? var.org_id : var.parent_folder
 
   granular_sa = {
-    "org"  = "Foundation Organization SA. Managed by Terraform.",
-    "env"  = "Foundation Environment SA. Managed by Terraform.",
-    "net"  = "Foundation Network SA. Managed by Terraform.",
-    "proj" = "Foundation Projects SA. Managed by Terraform.",
+    "bootstrap" = "Foundation Bootstrap SA. Managed by Terraform.",
+    "org"       = "Foundation Organization SA. Managed by Terraform.",
+    "env"       = "Foundation Environment SA. Managed by Terraform.",
+    "net"       = "Foundation Network SA. Managed by Terraform.",
+    "proj"      = "Foundation Projects SA. Managed by Terraform.",
   }
 
   common_roles = [
     "roles/browser", // Required for gcloud beta terraform vet to be able to read the ancestry of folders
   ]
 
   granular_sa_org_level_roles = {
+    "bootstrap" = distinct(concat([
+      "roles/resourcemanager.organizationAdmin",
+      "roles/accesscontextmanager.policyAdmin",
+      "roles/serviceusage.serviceUsageConsumer",
+    ], local.common_roles)),
     "org" = distinct(concat([
       "roles/orgpolicy.policyAdmin",
       "roles/logging.configWriter",
@@ -57,6 +63,9 @@ locals {
   }
 
   granular_sa_parent_level_roles = {
+    "bootstrap" = [
+      "roles/resourcemanager.folderAdmin",
+    ],
     "org" = [
       "roles/resourcemanager.folderAdmin",
     ],
@@ -80,7 +89,13 @@ locals {
     ],
   }
 
+  // Roles required to manage resources in the Seed project
   granular_sa_seed_project = {
+    "bootstrap" = [
+      "roles/storage.admin",
+      "roles/iam.serviceAccountAdmin",
+      "roles/resourcemanager.projectDeleter",
+    ],
     "org" = [
       "roles/storage.objectAdmin",
     ],
@@ -94,6 +109,22 @@ locals {
       "roles/storage.objectAdmin",
     ],
   }
+
+  // Roles required to manage resources in the CI/CD project
+  granular_sa_cicd_project = {
+    "bootstrap" = [
+      "roles/storage.admin",
+      "roles/compute.networkAdmin",
+      "roles/cloudbuild.builds.editor",
+      "roles/cloudbuild.workerPoolOwner",
+      "roles/artifactregistry.admin",
+      "roles/source.admin",
+      "roles/iam.serviceAccountAdmin",
+      "roles/workflows.admin",
+      "roles/cloudscheduler.admin",
+      "roles/resourcemanager.projectDeleter",
+    ],
+  }
 }
 
 resource "google_service_account" "terraform-env-sa" {
@@ -124,7 +155,7 @@ module "parent_iam_member" {
   roles       = each.value
 }
 
-module "project_iam_member" {
+module "seed_project_iam_member" {
   source   = "./modules/parent-iam-member"
   for_each = local.granular_sa_seed_project
 
@@ -134,6 +165,16 @@ module "project_iam_member" {
   roles       = each.value
 }
 
+module "cicd_project_iam_member" {
+  source   = "./modules/parent-iam-member"
+  for_each = local.granular_sa_cicd_project
+
+  member      = "serviceAccount:${google_service_account.terraform-env-sa[each.key].email}"
+  parent_type = "project"
+  parent_id   = local.cicd_project_id
+  roles       = each.value
+}
+
 resource "google_billing_account_iam_member" "tf_billing_user" {
   for_each = local.granular_sa
 

diff --git a/1-org/README.md b/1-org/README.md
@@ -105,51 +105,21 @@ commands. The `-T` flag is needed for Linux, but causes problems for MacOS.
 
 ### Deploying with Cloud Build
 
-1. Clone the policy repo based on the Terraform output from the previous section.
+1. Clone the `gcp-org` repo based on the Terraform output from the previous step.
 Clone the repo at the same level of the `terraform-example-foundation` folder, the next instructions assume that layout.
 Run `terraform output cloudbuild_project_id` in the `0-bootstrap` folder to see the project again.
 
    ```bash
    export CLOUD_BUILD_PROJECT_ID=$(terraform -chdir="terraform-example-foundation/0-bootstrap/" output -raw cloudbuild_project_id)
    echo ${CLOUD_BUILD_PROJECT_ID}
-   gcloud source repos clone gcp-policies --project=${CLOUD_BUILD_PROJECT_ID}
-   ```
-
-1. Navigate into the repo and copy contents of policy-library to new repo.
-   All subsequent steps assume you are running them
-   from the gcp-policies directory. If you run them from another directory,
-   adjust your copy paths accordingly.
-
-   ```bash
-   cd gcp-policies
-   git checkout -b main
-   cp -RT ../terraform-example-foundation/policy-library/ .
-   ```
 
-1. Commit changes and push your main branch to the new repo.
-
-   ```bash
-   git add .
-   git commit -m 'Your message'
-   git push --set-upstream origin main
-   ```
-
-1. Navigate out of the repo.
-
-   ```bash
-   cd ..
-   ```
-
-1. Clone the repo.
-
-   ```bash
    gcloud source repos clone gcp-org --project=${CLOUD_BUILD_PROJECT_ID}
    ```
 
    - The message `warning: You appear to have cloned an empty repository.` is
    normal and can be ignored.
 1. Navigate into the repo, change to a non-production branch and copy contents of foundation to new repo.
-   All subsequent steps assume you are running them from the gcp-org directory.
+   All subsequent steps assume you are running them from the `gcp-org` directory.
    If you run them from another directory, adjust your copy paths accordingly.
 
    ```bash

diff --git a/test/integration/bootstrap/bootstrap_test.go b/test/integration/bootstrap/bootstrap_test.go
@@ -72,6 +72,7 @@ func TestBootstrap(t *testing.T) {
 	}
 
 	triggerRepos := []string{
+		"gcp-bootstrap",
 		"gcp-org",
 		"gcp-environments",
 		"gcp-networks",
@@ -153,6 +154,7 @@ func TestBootstrap(t *testing.T) {
 			assert.True(prj.Exists(), "project %s should exist", cbProjectID)
 
 			for _, env := range []string{
+				"bootstrap",
 				"org",
 				"env",
 				"net",
@@ -234,6 +236,15 @@ func TestBootstrap(t *testing.T) {
 						"roles/browser",
 					},
 				},
+				{
+					output: "bootstrap_step_terraform_service_account_email",
+					orgRoles: []string{
+						"roles/resourcemanager.organizationAdmin",
+						"roles/accesscontextmanager.policyAdmin",
+						"roles/serviceusage.serviceUsageConsumer",
+						"roles/browser",
+					},
+				},
 			} {
 				terraformSAEmail := bootstrap.GetStringOutput(sa.output)
 				terraformSAName := fmt.Sprintf("projects/%s/serviceAccounts/%s", seedProjectID, terraformSAEmail)