feat: CloudBuild workspace blueprint (#162)

* feat: CloudBuild workspace blueprint

* add example

* add test

* add test to CI

* add versions, attrib

* workaround for dynamic project id

* typo

* fix optional bucket, subst

* regen

* support custom bucket/sa in same config

* update docs

* sleep after main push in example

* Update versions.tf

* Update versions.tf

build/int.cloudbuild.yaml

@@ -93,6 +93,26 @@ steps:
       - verify-tfbuilder
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilder --stage teardown --verbose']
+- id: init-tfworkspace
+  waitFor:
+      - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildWorkspaceSimple --stage init --verbose']
+- id: apply-tfworkspace
+  waitFor:
+      - init-tfworkspace
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildWorkspaceSimple --stage apply --verbose']
+- id: verify-tfworkspace
+  waitFor:
+      - apply-tfworkspace
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildWorkspaceSimple --stage verify --verbose']
+- id: teardown-tfworkspace
+  waitFor:
+      - verify-tfworkspace
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildWorkspaceSimple --stage teardown --verbose']
 tags:
 - 'ci'
 - 'integration'

examples/tf_cloudbuild_workspace_simple/README.md

@@ -0,0 +1,24 @@
+## Overview
+
+This example demonstrates the simplest usage of the [tf_cloudbuild_workspace](../../modules/tf_cloudbuild_workspace/) module.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cloudbuild\_apply\_trigger\_id | Trigger used for running TF apply |
+| cloudbuild\_plan\_trigger\_id | Trigger used for running TF plan |
+| cloudbuild\_sa | SA used by Cloud Build triggers |
+| csr\_repo\_url | CSR repo for storing TF configs |
+| logs\_bucket | Bucket for storing TF logs/plans |
+| project\_id | n/a |
+| state\_bucket | Bucket for storing TF state |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/tf_cloudbuild_workspace_simple/apis.tf

@@ -0,0 +1,30 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "enabled_google_apis" {
+  source  = "terraform-google-modules/project-factory/google//modules/project_services"
+  version = "~> 11.3"
+
+  project_id                  = var.project_id
+  disable_services_on_destroy = false
+
+  activate_apis = [
+    "iam.googleapis.com",
+    "compute.googleapis.com",
+    "sourcerepo.googleapis.com",
+    "cloudbuild.googleapis.com",
+  ]
+}

examples/tf_cloudbuild_workspace_simple/files/.gitignore

@@ -0,0 +1,2 @@
+# Local .terraform directories
+**/.terraform*

examples/tf_cloudbuild_workspace_simple/files/backend.tf

@@ -0,0 +1,22 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  backend "gcs" {
+    bucket = "tf-state-prod"
+    prefix = "terraform/state"
+  }
+}

examples/tf_cloudbuild_workspace_simple/files/main.tf

@@ -0,0 +1,48 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "test-vpc-module" {
+  source       = "terraform-google-modules/network/google"
+  version      = "~> 4.0.1"
+  project_id   = var.project_id
+  network_name = "my-custom-mode-network"
+  mtu          = 1460
+
+  subnets = [
+    {
+      subnet_name   = "subnet-01"
+      subnet_ip     = "10.10.10.0/24"
+      subnet_region = "us-west1"
+    },
+    {
+      subnet_name           = "subnet-02"
+      subnet_ip             = "10.10.20.0/24"
+      subnet_region         = "us-west1"
+      subnet_private_access = "true"
+      subnet_flow_logs      = "true"
+    },
+    {
+      subnet_name               = "subnet-03"
+      subnet_ip                 = "10.10.30.0/24"
+      subnet_region             = "us-west1"
+      subnet_flow_logs          = "true"
+      subnet_flow_logs_interval = "INTERVAL_10_MIN"
+      subnet_flow_logs_sampling = 0.7
+      subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA"
+      subnet_flow_logs_filter   = "false"
+    }
+  ]
+}

examples/tf_cloudbuild_workspace_simple/files/variables.tf

@@ -0,0 +1,19 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the network in"
+}

examples/tf_cloudbuild_workspace_simple/main.tf

@@ -0,0 +1,47 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "tf_workspace" {
+  source = "../../modules/tf_cloudbuild_workspace"
+
+  project_id  = module.enabled_google_apis.project_id
+  tf_repo_uri = google_sourcerepo_repository.tf_config_repo.url
+  # allow log/state buckets to be destroyed
+  buckets_force_destroy = true
+  cloudbuild_sa_roles = { (module.enabled_google_apis.project_id) = {
+    project_id = module.enabled_google_apis.project_id,
+    roles      = ["roles/compute.networkAdmin"]
+    }
+  }
+  cloudbuild_env_vars = ["TF_VAR_project_id=${var.project_id}"]
+
+}
+
+# CSR for storing TF configs
+resource "google_sourcerepo_repository" "tf_config_repo" {
+  project = module.enabled_google_apis.project_id
+  name    = "tf-configs"
+}
+
+# # Bootstrap CSR with TF configs
+module "bootstrap_csr_repo" {
+  source  = "terraform-google-modules/gcloud/google"
+  version = "~> 3.1.0"
+  upgrade = false
+
+  create_cmd_entrypoint = "${path.module}/scripts/push-to-repo.sh"
+  create_cmd_body       = "${module.enabled_google_apis.project_id} ${split("/", google_sourcerepo_repository.tf_config_repo.id)[3]} ${path.module}/files"
+}

examples/tf_cloudbuild_workspace_simple/outputs.tf

@@ -0,0 +1,49 @@
+# /**
+#  * Copyright 2022 Google LLC
+#  *
+#  * Licensed under the Apache License, Version 2.0 (the "License");
+#  * you may not use this file except in compliance with the License.
+#  * You may obtain a copy of the License at
+#  *
+#  *      http://www.apache.org/licenses/LICENSE-2.0
+#  *
+#  * Unless required by applicable law or agreed to in writing, software
+#  * distributed under the License is distributed on an "AS IS" BASIS,
+#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  * See the License for the specific language governing permissions and
+#  * limitations under the License.
+#  */
+
+output "cloudbuild_plan_trigger_id" {
+  description = "Trigger used for running TF plan"
+  value       = module.tf_workspace.cloudbuild_plan_trigger_id
+}
+
+output "cloudbuild_apply_trigger_id" {
+  description = "Trigger used for running TF apply"
+  value       = module.tf_workspace.cloudbuild_apply_trigger_id
+}
+
+output "cloudbuild_sa" {
+  description = "SA used by Cloud Build triggers"
+  value       = module.tf_workspace.cloudbuild_sa
+}
+
+output "state_bucket" {
+  description = "Bucket for storing TF state"
+  value       = module.tf_workspace.state_bucket
+}
+
+output "logs_bucket" {
+  description = "Bucket for storing TF logs/plans"
+  value       = module.tf_workspace.logs_bucket
+}
+
+output "csr_repo_url" {
+  description = "CSR repo for storing TF configs"
+  value       = google_sourcerepo_repository.tf_config_repo.url
+}
+
+output "project_id" {
+  value = var.project_id
+}

examples/tf_cloudbuild_workspace_simple/scripts/push-to-repo.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+set -ex
+
+if [ "$#" -lt 3 ]; then
+    >&2 echo "Not all expected arguments set."
+    exit 1
+fi
+
+CSR_PROJECT_ID=$1
+CSR_NAME=$2
+TF_CONFIG_PATH=$3
+
+# create temp dir, cleanup at exit
+tmp_dir=$(mktemp -d)
+# shellcheck disable=SC2064
+trap "rm -rf $tmp_dir" EXIT
+gcloud source repos clone "${CSR_NAME}" "${tmp_dir}" --project "${CSR_PROJECT_ID}"
+cp -r "${TF_CONFIG_PATH}/." "${tmp_dir}"
+pushd "${tmp_dir}"
+git config credential.helper gcloud.sh
+git config init.defaultBranch main
+git config user.email "terraform-robot@example.com"
+git config user.name "TF Robot"
+git checkout plan || git checkout -b plan
+git add -A
+git commit -m "init tf configs"
+git push origin plan -f
+sleep 60
+git checkout main || git checkout -b main
+git push origin main -f
+sleep 120

examples/tf_cloudbuild_workspace_simple/variables.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The ID of the project in which to provision resources."
+  type        = string
+}