feat: create Cloudbuild Source submodule (#167)

* cloudbuild core module

* update README.md

* remove private pool configuration

* remove reference to private pool from readme

* add test for tf_cloudbuild_core

* fix length of project id for tf core test

* fix gcloud command in core test

* fix assert input

* add depends on to module outputs

* review fixes

* rename tf_cloudbuild_core to tf_cloudbuild_source

* update build steps

* remove extra lines

* remove cloudbuild artifacts bucket

* remove random_id resource and add explanation for the cloudbuild bucket

Co-authored-by: Andrew Peabody <andrewpeabody@google.com>

build/int.cloudbuild.yaml

@@ -73,6 +73,28 @@ steps:
 - id: destroy-simple-folder
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-folder-default']
+
+- id: init-tfsource
+  waitFor:
+      - prepare
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage init --verbose']
+- id: apply-tfsource
+  waitFor:
+      - init-tfsource
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage apply --verbose']
+- id: verify-tfsource
+  waitFor:
+      - apply-tfsource
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage verify --verbose']
+- id: teardown-tfsource
+  waitFor:
+      - verify-tfsource
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage teardown --verbose']
+
 - id: init-tfbuilder
   waitFor:
       - prepare
@@ -93,6 +115,7 @@ steps:
       - verify-tfbuilder
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilder --stage teardown --verbose']
+
 - id: init-tfworkspace
   waitFor:
       - prepare
@@ -113,6 +136,7 @@ steps:
       - verify-tfworkspace
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildWorkspaceSimple --stage teardown --verbose']
+
 tags:
 - 'ci'
 - 'integration'

examples/tf_cloudbuild_source_simple/README.md

@@ -0,0 +1,23 @@
+## Overview
+
+This example demonstrates the simplest usage of the [tf_cloudbuild_source](../../modules/tf_cloudbuild_source/) module.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| billing\_account | The ID of the billing account to associate projects with. | `string` | n/a | yes |
+| group\_org\_admins | Google Group for GCP Organization Administrators | `string` | n/a | yes |
+| org\_id | GCP Organization ID | `string` | n/a | yes |
+| parent\_folder | The bootstrap parent folder | `string` | `""` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cloudbuild\_project\_id | Project where CloudBuild configuration and terraform container image will reside. |
+| csr\_repos | List of Cloud Source Repos created by the module. |
+| gcs\_cloudbuild\_default\_bucket | Bucket used to store temporary files in CloudBuild project. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/tf_cloudbuild_source_simple/main.tf

@@ -0,0 +1,25 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module "tf_source" {
+  source = "../../modules/tf_cloudbuild_source"
+
+  org_id                = var.org_id
+  folder_id             = var.parent_folder
+  billing_account       = var.billing_account
+  group_org_admins      = var.group_org_admins
+  buckets_force_destroy = true
+}

examples/tf_cloudbuild_source_simple/outputs.tf

@@ -0,0 +1,30 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "cloudbuild_project_id" {
+  description = "Project where CloudBuild configuration and terraform container image will reside."
+  value       = module.tf_source.cloudbuild_project_id
+}
+
+output "csr_repos" {
+  description = "List of Cloud Source Repos created by the module."
+  value       = module.tf_source.csr_repos
+}
+
+output "gcs_cloudbuild_default_bucket" {
+  description = "Bucket used to store temporary files in CloudBuild project."
+  value       = module.tf_source.gcs_cloudbuild_default_bucket
+}

examples/tf_cloudbuild_source_simple/variables.tf

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "org_id" {
+  description = "GCP Organization ID"
+  type        = string
+}
+
+variable "parent_folder" {
+  description = "The bootstrap parent folder"
+  type        = string
+  default     = ""
+}
+
+variable "billing_account" {
+  description = "The ID of the billing account to associate projects with."
+  type        = string
+}
+
+variable "group_org_admins" {
+  description = "Google Group for GCP Organization Administrators"
+  type        = string
+}

modules/tf_cloudbuild_source/README.md

@@ -0,0 +1,70 @@
+## Overview
+
+## Usage
+
+Basic usage of this module is as follows:
+
+```hcl
+module "tf-cloudbuild-core" {
+  source  = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_source"
+  version = "~> 6.1"
+
+  org_id           = var.org_id
+  billing_account  = var.billing_account
+  group_org_admins = var.group_org_admins
+}
+```
+
+Functional examples are included in the [examples](../../examples/) directory.
+
+## Resources created
+
+This module creates:
+
+- Project for Cloud Build.
+- Default Cloud Build bucket.
+- Set of Cloud Source Repos.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| activate\_apis | List of APIs to enable in the Cloudbuild project. | `list(string)` | <pre>[<br>  "serviceusage.googleapis.com",<br>  "servicenetworking.googleapis.com",<br>  "compute.googleapis.com",<br>  "logging.googleapis.com",<br>  "iam.googleapis.com",<br>  "admin.googleapis.com"<br>]</pre> | no |
+| billing\_account | The ID of the billing account to associate projects with. | `string` | n/a | yes |
+| buckets\_force\_destroy | When deleting CloudBuild buckets, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects. | `bool` | `false` | no |
+| cloud\_source\_repos | List of Cloud Source Repos to create with CloudBuild triggers. | `list(string)` | <pre>[<br>  "gcp-policies",<br>  "gcp-org",<br>  "gcp-envs",<br>  "gcp-networks",<br>  "gcp-projects"<br>]</pre> | no |
+| folder\_id | The ID of a folder to host this project | `string` | `""` | no |
+| group\_org\_admins | Google Group for GCP Organization Administrators | `string` | n/a | yes |
+| location | Location for build artifacts bucket | `string` | `"us-central1"` | no |
+| org\_id | GCP Organization ID | `string` | n/a | yes |
+| project\_id | Custom project ID to use for project created. | `string` | `""` | no |
+| project\_labels | Labels to apply to the project. | `map(string)` | `{}` | no |
+| storage\_bucket\_labels | Labels to apply to the storage bucket. | `map(string)` | `{}` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| cloudbuild\_project\_id | Project for CloudBuild and Cloud Source Repositories. |
+| csr\_repos | List of Cloud Source Repos created by the module. |
+| gcs\_cloudbuild\_default\_bucket | Bucket used to store temporary files in CloudBuild project. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+## Requirements
+
+### Software
+
+- [Terraform](https://www.terraform.io/downloads.html) >= 0.13.0
+- [terraform-provider-google] plugin >= 3.50.x
+
+### Permissions
+
+- `roles/resourcemanager.projectCreator`
+- `roles/billing.user`
+
+## Contributing
+
+Refer to the [contribution guidelines](../../CONTRIBUTING.md) for
+information on contributing to this module.

modules/tf_cloudbuild_source/main.tf

@@ -0,0 +1,93 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  cloudbuild_project_id = var.project_id != "" ? var.project_id : "tf-cloudbuild-"
+  use_random_suffix     = var.project_id == ""
+
+  cloudbuild_apis = [
+    "cloudbuild.googleapis.com",
+    "sourcerepo.googleapis.com",
+    "storage-api.googleapis.com",
+    "iam.googleapis.com",
+    "cloudresourcemanager.googleapis.com",
+    "cloudbilling.googleapis.com"
+  ]
+
+  activate_apis = distinct(concat(var.activate_apis, local.cloudbuild_apis))
+}
+
+module "cloudbuild_project" {
+  source  = "terraform-google-modules/project-factory/google"
+  version = "~> 13.0"
+
+  name                        = local.cloudbuild_project_id
+  random_project_id           = local.use_random_suffix
+  disable_services_on_destroy = false
+  folder_id                   = var.folder_id
+  org_id                      = var.org_id
+  billing_account             = var.billing_account
+  activate_apis               = local.activate_apis
+  labels                      = var.project_labels
+}
+
+// On the first run of cloud build submit, a bucket is automaticaly created with name "[PROJECT_ID]_cloudbuild"
+// https://cloud.google.com/sdk/gcloud/reference/builds/submit#:~:text=%5BPROJECT_ID%5D_cloudbuild
+// This bucket is create in the default region "US"
+// https://cloud.google.com/storage/docs/json_api/v1/buckets/insert#:~:text=or%20multi%2Dregion.-,Defaults%20to%20%22US%22,-.%20See%20Cloud%20Storage
+// Creating the bucket beforehand make it is possible to define a custom location.
+module "cloudbuild_bucket" {
+  source  = "terraform-google-modules/cloud-storage/google//modules/simple_bucket"
+  version = "~> 3.2"
+
+  name          = "${module.cloudbuild_project.project_id}_cloudbuild"
+  project_id    = module.cloudbuild_project.project_id
+  location      = var.location
+  labels        = var.storage_bucket_labels
+  force_destroy = var.buckets_force_destroy
+}
+
+resource "google_sourcerepo_repository" "gcp_repo" {
+  for_each = length(var.cloud_source_repos) > 0 ? toset(var.cloud_source_repos) : []
+
+  project = module.cloudbuild_project.project_id
+  name    = each.value
+}
+
+resource "google_project_iam_member" "org_admins_cloudbuild_editor" {
+  project = module.cloudbuild_project.project_id
+  role    = "roles/cloudbuild.builds.editor"
+  member  = "group:${var.group_org_admins}"
+}
+
+resource "google_project_iam_member" "org_admins_cloudbuild_viewer" {
+  project = module.cloudbuild_project.project_id
+  role    = "roles/viewer"
+  member  = "group:${var.group_org_admins}"
+}
+
+resource "google_project_iam_member" "org_admins_source_repo_admin" {
+  count   = length(var.cloud_source_repos) > 0 ? 1 : 0
+  project = module.cloudbuild_project.project_id
+  role    = "roles/source.admin"
+  member  = "group:${var.group_org_admins}"
+}
+
+resource "google_storage_bucket_iam_member" "cloudbuild_iam" {
+  bucket = module.cloudbuild_bucket.bucket.name
+  role   = "roles/storage.admin"
+  member = "serviceAccount:${module.cloudbuild_project.project_number}@cloudbuild.gserviceaccount.com"
+}

modules/tf_cloudbuild_source/outputs.tf

@@ -0,0 +1,41 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "cloudbuild_project_id" {
+  description = "Project for CloudBuild and Cloud Source Repositories."
+  value       = module.cloudbuild_project.project_id
+
+  depends_on = [
+    google_storage_bucket_iam_member.cloudbuild_iam,
+    google_project_iam_member.org_admins_cloudbuild_editor,
+    google_project_iam_member.org_admins_cloudbuild_viewer,
+    google_project_iam_member.org_admins_source_repo_admin
+  ]
+}
+
+output "csr_repos" {
+  description = "List of Cloud Source Repos created by the module."
+  value       = google_sourcerepo_repository.gcp_repo
+}
+
+output "gcs_cloudbuild_default_bucket" {
+  description = "Bucket used to store temporary files in CloudBuild project."
+  value       = module.cloudbuild_bucket.bucket.name
+
+  depends_on = [
+    google_storage_bucket_iam_member.cloudbuild_iam
+  ]
+}