feat: migrate to GAR for runner images (#94)

examples/cloudbuild_enabled/README.md

@@ -27,5 +27,6 @@ This example combines the Organization bootstrap module with the Cloud Build sub
 | seed\_project\_id | Project where service accounts and core APIs will be enabled. |
 | terraform\_sa\_email | Email for privileged service account for Terraform. |
 | terraform\_sa\_name | Fully qualified name for privileged service account for Terraform. |
+| tf\_runner\_artifact\_repo | GAR Repo created to store runner images |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/cloudbuild_enabled/main.tf

@@ -19,6 +19,10 @@ provider "google" {
   version = ">= 3.50"
 }
 
+provider "google-beta" {
+  version = ">= 3.50"
+}
+
 provider "null" {
   version = "~> 2.1"
 }

examples/cloudbuild_enabled/outputs.tf

@@ -58,3 +58,8 @@ output "kms_crypto_key" {
   description = "KMS key created by the module."
   value       = module.cloudbuild_bootstrap.kms_crypto_key
 }
+
+output "tf_runner_artifact_repo" {
+  description = "GAR Repo created to store runner images"
+  value       = module.cloudbuild_bootstrap.tf_runner_artifact_repo
+}

kitchen.yml

@@ -58,3 +58,7 @@ suites:
           controls:
             - bootstrap
             - cloudbuild
+        - name: cloudbuild_enabled gcloud
+          backend: local
+          controls:
+            - gcloud

modules/cloudbuild/README.md

@@ -59,6 +59,7 @@ Functional examples and sample Cloud Build definitions are included in the [exam
 | create\_cloud\_source\_repos | If shared Cloud Source Repos should be created. | `bool` | `true` | no |
 | default\_region | Default region to create resources where applicable. | `string` | `"us-central1"` | no |
 | folder\_id | The ID of a folder to host this project | `string` | `""` | no |
+| gar\_repo\_name | Custom name to use for GAR repo. | `string` | `""` | no |
 | group\_org\_admins | Google Group for GCP Organization Administrators | `string` | n/a | yes |
 | org\_id | GCP Organization ID | `string` | n/a | yes |
 | project\_id | Custom project ID to use for project created. | `string` | `""` | no |
@@ -83,6 +84,7 @@ Functional examples and sample Cloud Build definitions are included in the [exam
 | gcs\_bucket\_cloudbuild\_artifacts | Bucket used to store Cloud/Build artefacts in CloudBuild project. |
 | kms\_crypto\_key | KMS key created by the module. |
 | kms\_keyring | KMS Keyring created by the module. |
+| tf\_runner\_artifact\_repo | GAR Repo created to store runner images |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 

modules/cloudbuild/cloudbuild_builder/cloudbuild.yaml

@@ -18,16 +18,16 @@ steps:
 - name: 'gcr.io/cloud-builders/docker'
   args: [
     'build',
-    '--tag=gcr.io/${PROJECT_ID}/terraform',
+    '--tag=${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/terraform',
     '--build-arg=TERRAFORM_VERSION=${_TERRAFORM_VERSION}',
     '--build-arg=TERRAFORM_VERSION_SHA256SUM=${_TERRAFORM_VERSION_SHA256SUM}',
     '--build-arg=TERRAFORM_VALIDATOR_RELEASE=${_TERRAFORM_VALIDATOR_RELEASE}',
     '.'
     ]
-- name: 'gcr.io/${PROJECT_ID}/terraform'
+- name: '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/terraform'
   args: ['version']
 substitutions:
   _TERRAFORM_VERSION: '0.13.6' # default value
   _TERRAFORM_VERSION_SHA256SUM: '55f2db00b05675026be9c898bdd3e8230ff0c5c78dd12d743ca38032092abfc9' # default value
   _TERRAFORM_VALIDATOR_RELEASE: '2020-09-24'
-images: ['gcr.io/${PROJECT_ID}/terraform']
+images: ['${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/terraform']

modules/cloudbuild/main.tf

@@ -16,10 +16,12 @@
 
 locals {
   cloudbuild_project_id       = var.project_id != "" ? var.project_id : format("%s-%s", var.project_prefix, "cloudbuild")
-  cloudbuild_apis             = ["cloudbuild.googleapis.com", "sourcerepo.googleapis.com", "cloudkms.googleapis.com"]
+  gar_repo_name               = var.gar_repo_name != "" ? var.gar_repo_name : format("%s-%s", var.project_prefix, "tf-runners")
+  cloudbuild_apis             = ["cloudbuild.googleapis.com", "sourcerepo.googleapis.com", "cloudkms.googleapis.com", "artifactregistry.googleapis.com"]
   impersonation_enabled_count = var.sa_enable_impersonation == true ? 1 : 0
-  activate_apis               = distinct(var.activate_apis)
+  activate_apis               = distinct(concat(var.activate_apis, local.cloudbuild_apis))
   apply_branches_regex        = "^(${join("|", var.terraform_apply_branches)})$"
+  gar_name                    = split("/", google_artifact_registry_repository.tf-image-repo.name)[length(split("/", google_artifact_registry_repository.tf-image-repo.name)) - 1]
 }
 
 resource "random_id" "suffix" {
@@ -48,13 +50,6 @@ module "cloudbuild_project" {
   labels                      = var.project_labels
 }
 
-resource "google_project_service" "cloudbuild_apis" {
-  for_each           = toset(local.cloudbuild_apis)
-  project            = module.cloudbuild_project.project_id
-  service            = each.value
-  disable_on_destroy = false
-}
-
 /******************************************
   Cloudbuild IAM for admins
 *******************************************/
@@ -94,9 +89,6 @@ resource "google_kms_key_ring" "tf_keyring" {
   project  = module.cloudbuild_project.project_id
   name     = "tf-keyring"
   location = var.default_region
-  depends_on = [
-    google_project_service.cloudbuild_apis,
-  ]
 }
 
 /******************************************
@@ -120,9 +112,6 @@ resource "google_kms_crypto_key_iam_binding" "cloudbuild_crypto_key_decrypter" {
     "serviceAccount:${module.cloudbuild_project.project_number}@cloudbuild.gserviceaccount.com",
     "serviceAccount:${var.terraform_sa_email}"
   ]
-  depends_on = [
-    google_project_service.cloudbuild_apis,
-  ]
 }
 
 /******************************************
@@ -146,9 +135,6 @@ resource "google_sourcerepo_repository" "gcp_repo" {
   for_each = var.create_cloud_source_repos ? toset(var.cloud_source_repos) : []
   project  = module.cloudbuild_project.project_id
   name     = each.value
-  depends_on = [
-    google_project_service.cloudbuild_apis,
-  ]
 }
 
 /******************************************
@@ -223,6 +209,19 @@ resource "google_cloudbuild_trigger" "non_master_trigger" {
   ]
 }
 
+/***********************************************
+ Cloud Build - Terraform Image Repo
+ ***********************************************/
+resource "google_artifact_registry_repository" "tf-image-repo" {
+  provider = google-beta
+  project  = module.cloudbuild_project.project_id
+
+  location      = var.default_region
+  repository_id = local.gar_repo_name
+  description   = "Docker repository for Terraform runner images used by Cloud Build"
+  format        = "DOCKER"
+}
+
 /***********************************************
  Cloud Build - Terraform builder
  ***********************************************/
@@ -232,18 +231,20 @@ resource "null_resource" "cloudbuild_terraform_builder" {
     project_id_cloudbuild_project = module.cloudbuild_project.project_id
     terraform_version_sha256sum   = var.terraform_version_sha256sum
     terraform_version             = var.terraform_version
+    gar_name                      = local.gar_name
+    gar_location                  = google_artifact_registry_repository.tf-image-repo.location
   }
 
   provisioner "local-exec" {
     command = <<EOT
       gcloud builds submit ${path.module}/cloudbuild_builder/ \
       --project ${module.cloudbuild_project.project_id} \
       --config=${path.module}/cloudbuild_builder/cloudbuild.yaml \
-      --substitutions=_TERRAFORM_VERSION=${var.terraform_version},_TERRAFORM_VERSION_SHA256SUM=${var.terraform_version_sha256sum},_TERRAFORM_VALIDATOR_RELEASE=${var.terraform_validator_release}
+      --substitutions=_TERRAFORM_VERSION=${var.terraform_version},_TERRAFORM_VERSION_SHA256SUM=${var.terraform_version_sha256sum},_TERRAFORM_VALIDATOR_RELEASE=${var.terraform_validator_release},_REGION=${google_artifact_registry_repository.tf-image-repo.location},_REPOSITORY=${local.gar_name}
   EOT
   }
   depends_on = [
-    google_project_service.cloudbuild_apis,
+    google_artifact_registry_repository_iam_member.terraform-image-iam
   ]
 }
 
@@ -255,9 +256,16 @@ resource "google_storage_bucket_iam_member" "cloudbuild_artifacts_iam" {
   bucket = google_storage_bucket.cloudbuild_artifacts.name
   role   = "roles/storage.admin"
   member = "serviceAccount:${module.cloudbuild_project.project_number}@cloudbuild.gserviceaccount.com"
-  depends_on = [
-    google_project_service.cloudbuild_apis,
-  ]
+}
+
+resource "google_artifact_registry_repository_iam_member" "terraform-image-iam" {
+  provider = google-beta
+  project  = module.cloudbuild_project.project_id
+
+  location   = google_artifact_registry_repository.tf-image-repo.location
+  repository = google_artifact_registry_repository.tf-image-repo.name
+  role       = "roles/artifactregistry.writer"
+  member     = "serviceAccount:${module.cloudbuild_project.project_number}@cloudbuild.gserviceaccount.com"
 }
 
 resource "google_service_account_iam_member" "cloudbuild_terraform_sa_impersonate_permissions" {
@@ -266,9 +274,6 @@ resource "google_service_account_iam_member" "cloudbuild_terraform_sa_impersonat
   service_account_id = var.terraform_sa_name
   role               = "roles/iam.serviceAccountTokenCreator"
   member             = "serviceAccount:${module.cloudbuild_project.project_number}@cloudbuild.gserviceaccount.com"
-  depends_on = [
-    google_project_service.cloudbuild_apis,
-  ]
 }
 
 resource "google_organization_iam_member" "cloudbuild_serviceusage_consumer" {
@@ -277,9 +282,6 @@ resource "google_organization_iam_member" "cloudbuild_serviceusage_consumer" {
   org_id = var.org_id
   role   = "roles/serviceusage.serviceUsageConsumer"
   member = "serviceAccount:${module.cloudbuild_project.project_number}@cloudbuild.gserviceaccount.com"
-  depends_on = [
-    google_project_service.cloudbuild_apis,
-  ]
 }
 
 # Required to allow cloud build to access state with impersonation.
@@ -289,7 +291,4 @@ resource "google_storage_bucket_iam_member" "cloudbuild_state_iam" {
   bucket = var.terraform_state_bucket
   role   = "roles/storage.admin"
   member = "serviceAccount:${module.cloudbuild_project.project_number}@cloudbuild.gserviceaccount.com"
-  depends_on = [
-    google_project_service.cloudbuild_apis,
-  ]
 }

modules/cloudbuild/outputs.tf

@@ -38,3 +38,8 @@ output "kms_crypto_key" {
   description = "KMS key created by the module."
   value       = google_kms_crypto_key.tf_key
 }
+
+output "tf_runner_artifact_repo" {
+  description = "GAR Repo created to store runner images"
+  value       = google_artifact_registry_repository.tf-image-repo.name
+}

modules/cloudbuild/variables.tf

@@ -169,3 +169,9 @@ variable "terraform_apply_branches" {
     "master"
   ]
 }
+
+variable "gar_repo_name" {
+  description = "Custom name to use for GAR repo."
+  default     = ""
+  type        = string
+}

modules/cloudbuild/versions.tf

@@ -22,6 +22,10 @@ terraform {
       source  = "hashicorp/google"
       version = ">= 3.50"
     }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = ">= 3.50"
+    }
   }
 
   provider_meta "google" {

test/fixtures/cloudbuild_enabled/outputs.tf

@@ -63,3 +63,8 @@ output "kms_crypto_key" {
   description = "KMS key created by the module."
   value       = module.cloudbuild_enabled.kms_crypto_key
 }
+
+output "tf_runner_artifact_repo" {
+  description = "GAR Repo created to store runner images"
+  value       = module.cloudbuild_enabled.tf_runner_artifact_repo
+}

test/integration/cloudbuild_enabled/controls/gcloud.rb

@@ -0,0 +1,49 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+control "gcloud" do
+    title "GAR Repo"
+    describe command("gcloud --project=#{attribute("cloudbuild_project_id")} artifacts repositories describe #{attribute("tf_runner_artifact_repo")} --location=us-central1 --format=json") do
+      its(:exit_status) { should eq 0 }
+      let!(:data) do
+        if subject.exit_status == 0
+          JSON.parse(subject.stdout)
+        else
+          {}
+        end
+      end
+      describe "GAR repo" do
+        it "exists" do
+          expect(data['name']).to eq "projects/#{attribute("cloudbuild_project_id")}/locations/us-central1/repositories/#{attribute("tf_runner_artifact_repo")}"
+        end
+      end
+    end
+
+    title "GAR Repo TF Runner Image"
+    describe command("gcloud --project=#{attribute("cloudbuild_project_id")} artifacts tags list --repository=#{attribute("tf_runner_artifact_repo")} --location=us-central1 --package=terraform --format=json") do
+      its(:exit_status) { should eq 0 }
+      let!(:data) do
+        if subject.exit_status == 0
+          JSON.parse(subject.stdout)
+        else
+          {}
+        end
+      end
+      describe "TF runner image" do
+        it "exists" do
+          expect(data[0]['name']).to eq "projects/#{attribute("cloudbuild_project_id")}/locations/us-central1/repositories/#{attribute("tf_runner_artifact_repo")}/packages/terraform/tags/latest"
+        end
+      end
+    end
+  end

test/integration/cloudbuild_enabled/inspec.yml

@@ -33,3 +33,6 @@ attributes:
   - name: kms_crypto_key
     required: true
     type: hash
+  - name: tf_runner_artifact_repo
+    required: true
+    type: string

test/setup/main.tf

@@ -32,7 +32,8 @@ module "project" {
     "serviceusage.googleapis.com",
     "cloudbuild.googleapis.com",
     "sourcerepo.googleapis.com",
-    "cloudkms.googleapis.com"
+    "cloudkms.googleapis.com",
+    "artifactregistry.googleapis.com"
   ]
 }