Add support for billing budgets to project factory

fast/stages/3-project-factory/dev/README.md

@@ -79,8 +79,8 @@ terraform apply
 | name | description | type | required | default | producer |
 |---|---|:---:|:---:|:---:|:---:|
 | [billing_account](variables.tf#L19) | Billing account id. If billing account is not part of the same org set `is_org_level` to false. | <code title="object&#40;&#123;&#10;  id           &#61; string&#10;  is_org_level &#61; optional&#40;bool, true&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  | <code>0-bootstrap</code> |
-| [prefix](variables.tf#L39) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ |  | <code>0-bootstrap</code> |
-| [factory_data_path](variables.tf#L32) | Path to folder containing YAML project data files. | <code>string</code> |  | <code>&#34;data&#47;projects&#34;</code> |  |
+| [factories_config](variables.tf#L32) | Path to folder with YAML resource description data files. | <code title="object&#40;&#123;&#10;  projects_data_path &#61; string&#10;  budgets &#61; optional&#40;object&#40;&#123;&#10;    billing_account       &#61; string&#10;    budgets_data_path     &#61; string&#10;    notification_channels &#61; optional&#40;map&#40;any&#41;, &#123;&#125;&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |  |
+| [prefix](variables.tf#L45) | Prefix used for resources that need unique names. Use 9 characters or less. | <code>string</code> | ✓ |  | <code>0-bootstrap</code> |
 
 ## Outputs
 

fast/stages/3-project-factory/dev/main.tf

@@ -33,7 +33,7 @@ module "projects" {
   data_overrides = {
     prefix = "${var.prefix}-dev"
   }
-  factory_data_path = var.factory_data_path
+  factories_config = var.factories_config
 }
 
 

fast/stages/3-project-factory/dev/variables.tf

@@ -29,11 +29,17 @@ variable "billing_account" {
   }
 }
 
-variable "factory_data_path" {
-  description = "Path to folder containing YAML project data files."
-  type        = string
-  nullable    = false
-  default     = "data/projects"
+variable "factories_config" {
+  description = "Path to folder with YAML resource description data files."
+  type = object({
+    projects_data_path = string
+    budgets = optional(object({
+      billing_account       = string
+      budgets_data_path     = string
+      notification_channels = optional(map(any), {})
+    }))
+  })
+  nullable = false
 }
 
 variable "prefix" {

modules/billing-account/README.md

@@ -230,6 +230,9 @@ module "billing-account" {
       }
     }
   }
+  factories_config = {
+    budgets_data_path = "data/billing-budgets"
+  }
 }
 # tftest modules=1 resources=2 files=test-1  inventory=budget-monitoring-channel.yaml
 ```
@@ -268,7 +271,7 @@ update_rules:
 | [iam_bindings](variables-iam.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | <code title="map&#40;object&#40;&#123;&#10;  members &#61; list&#40;string&#41;&#10;  role    &#61; string&#10;  condition &#61; optional&#40;object&#40;&#123;&#10;    expression  &#61; string&#10;    title       &#61; string&#10;    description &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [iam_bindings_additive](variables-iam.tf#L39) | Individual additive IAM bindings. Keys are arbitrary. | <code title="map&#40;object&#40;&#123;&#10;  member &#61; string&#10;  role   &#61; string&#10;  condition &#61; optional&#40;object&#40;&#123;&#10;    expression  &#61; string&#10;    title       &#61; string&#10;    description &#61; optional&#40;string&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [iam_by_principals](variables-iam.tf#L54) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | <code>map&#40;list&#40;string&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [logging_sinks](variables.tf#L135) | Logging sinks to create for the organization. | <code title="map&#40;object&#40;&#123;&#10;  destination          &#61; string&#10;  type                 &#61; string&#10;  bq_partitioned_table &#61; optional&#40;bool, false&#41;&#10;  description          &#61; optional&#40;string&#41;&#10;  disabled             &#61; optional&#40;bool, false&#41;&#10;  exclusions &#61; optional&#40;map&#40;object&#40;&#123;&#10;    filter      &#61; string&#10;    description &#61; optional&#40;string&#41;&#10;    disabled    &#61; optional&#40;bool&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;  filter &#61; optional&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [logging_sinks](variables.tf#L135) | Logging sinks to create for the billing account. | <code title="map&#40;object&#40;&#123;&#10;  destination          &#61; string&#10;  type                 &#61; string&#10;  bq_partitioned_table &#61; optional&#40;bool, false&#41;&#10;  description          &#61; optional&#40;string&#41;&#10;  disabled             &#61; optional&#40;bool, false&#41;&#10;  exclusions &#61; optional&#40;map&#40;object&#40;&#123;&#10;    filter      &#61; string&#10;    description &#61; optional&#40;string&#41;&#10;    disabled    &#61; optional&#40;bool&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;  filter &#61; optional&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [projects](variables.tf#L168) | Projects associated with this billing account. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
 
 ## Outputs

modules/billing-account/factory.tf

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+# any changes to this factory should be mirrored in the project factory
+
 locals {
   _factory_data = {
     for f in fileset("${local._factory_path}", "**/*.yaml") :

modules/billing-account/variables.tf

@@ -133,7 +133,7 @@ variable "id" {
 }
 
 variable "logging_sinks" {
-  description = "Logging sinks to create for the organization."
+  description = "Logging sinks to create for the billing account."
   type = map(object({
     destination          = string
     type                 = string

modules/project-factory/.gitignore

@@ -0,0 +1 @@
+data

modules/project-factory/README.md

@@ -4,12 +4,10 @@ This module implements in code the end-to-end project creation process for multi
 
 It supports
 
-- all project-level attributes exposed by the [project module](../project/), including Shared VPC host/service configuration
-- optional service account creation in the project, including basic IAM grants
-- KMS key encrypt/decrypt permissions for service identities in the project
-- membership in VPC SC standard or bridge perimeters
-- billing budgets (TODO)
-- per-project IaC configuration (TODO)
+- multiple project creation and management exposing the full configuration options available in the [project module](../project/), including KMS key grants and VPC-SC perimeter membership
+- optional per-project [service account management](#service-accounts) including basic IAM grants
+- optional [billing budgets](#billing-budgets) factory and budget/project associations
+- optional per-project IaC configuration (TODO)
 
 The factory is implemented as a thin wrapping layer, so that no "magic" or hidden side effects are implemented in code, and debugging or integration of new features are simple.
 
@@ -18,6 +16,17 @@ The code is meant to be executed by a high level service accounts with powerful
 - Shared VPC connection if service project attachment is desired
 - project creation on the nodes (folder or org) where projects will be defined
 
+<!-- BEGIN TOC -->
+- [Leveraging data defaults, merges, optionals](#leveraging-data-defaults-merges-optionals)
+- [Additional resources](#additional-resources)
+  - [Service accounts](#service-accounts)
+  - [Billing budgets](#billing-budgets)
+- [Example](#example)
+- [Variables](#variables)
+- [Outputs](#outputs)
+- [Tests](#tests)
+<!-- END TOC -->
+
 ## Leveraging data defaults, merges, optionals
 
 In addition to the YAML-based project configurations, the factory accepts three additional sets of inputs via Terraform variables:
@@ -26,7 +35,49 @@ In addition to the YAML-based project configurations, the factory accepts three
 - the `data_overrides` variable works similarly to defaults, but the values specified here take precedence over those in YAML files
 - the `data_merges` variable allows specifying additional values for map or set based variables, which are merged with the data coming from YAML
 
-Some examples on where to use each of the three sets are provided below.
+Some examples on where to use each of the three sets are [provided below](#example).
+
+## Additional resources
+
+### Service accounts
+
+Service accounts can be managed as part of each project's YAML configuration. This allows creation of default service accounts used for GCE instances, in firewall rules, or for application-level credentials without resorting to a separate Terraform configuration.
+
+Each service account is represented by one key and a set of optional key/value pairs in the `service_accounts` top-level YAML map, like in this example:
+
+```yaml
+service_accounts:
+  be-0: {}
+  fe-1:
+    display_name: GCE frontend service account.
+    iam_project_roles:
+      - roles/storage.objectViewer 
+```
+
+Both the `display_name` and `iam_project_roles` attributes are optional.
+
+### Billing budgets
+
+The project factory integrates the billing budgets factory exposed by the `[`billing-account`](../billing-account/) module, and adds support for easy referencing budgets in project files.
+
+To enable support for billing budgets, set the billing account id, optional notification channels, and the data folder for budgets in the `factories_config.budgets` variable, then create billing budgets using YAML definitions following the format described in the `billing-account` module.
+
+Once budgets are defined, they can be referenced in a project file using their file name:
+
+```yaml
+billing_account: 012345-67890A-BCDEF0
+labels:
+ app: app-1
+ team: foo
+parent: folders/12345678
+services:
+  - container.googleapis.com
+  - storage.googleapis.com
+billing_budgets:
+  - test-100
+```
+
+The example below shows how to use the billing budgets factory.
 
 ## Example
 
@@ -35,7 +86,7 @@ module "project-factory" {
   source = "./fabric/modules/project-factory"
   # use a default billing account if none is specified via yaml
   data_defaults = {
-    billing_account = "012345-67890A-ABCDEF"
+    billing_account = var.billing_account_id
   }
   # make sure the environment label and stackdriver service are always added
   data_merges = {
@@ -54,12 +105,28 @@ module "project-factory" {
     prefix = "test-pf"
   }
   # location where the yaml files are read from
-  factory_data_path = "data"
+  factories_config = {
+    budgets = {
+      billing_account   = var.billing_account_id
+      budgets_data_path = "data/budgets"
+      notification_channels = {
+        billing-default = {
+          project_id = "foo-billing-audit"
+          type       = "email"
+          labels = {
+            email_address = "gcp-billing-admins@example.com"
+          }
+        }
+      }
+    }
+    projects_data_path = "data/projects"
+  }
 }
-# tftest modules=7 resources=33 files=prj-app-1,prj-app-2,prj-app-3 inventory=example.yaml
+# tftest modules=8 resources=35 files=prj-app-1,prj-app-2,prj-app-3,budget-test-100 inventory=example.yaml
 ```
 
 ```yaml
+# project app-1
 billing_account: 012345-67890A-BCDEF0
 labels:
  app: app-1
@@ -78,11 +145,13 @@ service_accounts:
     - roles/monitoring.metricWriter
   app-1-fe:
     display_name: "Test app 1 frontend."
-
-# tftest-file id=prj-app-1 path=data/prj-app-1.yaml
+billing_budgets:
+  - test-100
+# tftest-file id=prj-app-1 path=data/projects/prj-app-1.yaml
 ```
 
 ```yaml
+# project app-2
 labels:
   app: app-2
   team: foo
@@ -115,24 +184,45 @@ shared_vpc_service_config:
     europe-west1/prod-default-ew1:
     - group:team-1@example.com
 
-# tftest-file id=prj-app-2 path=data/prj-app-2.yaml
+# tftest-file id=prj-app-2 path=data/projects/prj-app-2.yaml
 ```
 
 ```yaml
+# project app-3
 parent: folders/12345678
 services:
 - run.googleapis.com
 - storage.googleapis.com
 
-# tftest-file id=prj-app-3 path=data/prj-app-3.yaml
+# tftest-file id=prj-app-3 path=data/projects/prj-app-3.yaml
 ```
 
+```yaml
+# billing budget test-100
+display_name: 100 dollars in current spend
+amount:
+  units: 100
+filter:
+  period:
+    calendar: MONTH
+  resource_ancestors:
+  - folders/1234567890
+threshold_rules:
+- percent: 0.5
+- percent: 0.75
+update_rules:
+  default:
+    disable_default_iam_recipients: true
+    monitoring_notification_channels:
+    - billing-default
+# tftest-file id=budget-test-100 path=data/budgets/test-100.yaml
+```
 <!-- BEGIN TFDOC -->
 ## Variables
 
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [factory_data_path](variables.tf#L91) | Path to folder with YAML project description data files. | <code>string</code> | ✓ |  |
+| [factories_config](variables.tf#L91) | Path to folder with YAML resource description data files. | <code title="object&#40;&#123;&#10;  projects_data_path &#61; string&#10;  budgets &#61; optional&#40;object&#40;&#123;&#10;    billing_account       &#61; string&#10;    budgets_data_path     &#61; string&#10;    notification_channels &#61; optional&#40;map&#40;any&#41;, &#123;&#125;&#41;&#10;  &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ |  |
 | [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | <code title="object&#40;&#123;&#10;  billing_account            &#61; optional&#40;string&#41;&#10;  contacts                   &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  labels                     &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;  metric_scopes              &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  parent                     &#61; optional&#40;string&#41;&#10;  prefix                     &#61; optional&#40;string&#41;&#10;  service_encryption_key_ids &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  service_perimeter_bridges  &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  service_perimeter_standard &#61; optional&#40;string&#41;&#10;  services                   &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  shared_vpc_service_config &#61; optional&#40;object&#40;&#123;&#10;    host_project                &#61; string&#10;    network_users               &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;    service_identity_iam        &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;    service_identity_subnet_iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;    service_iam_grants          &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;    network_subnet_users        &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  &#125;&#41;, &#123; host_project &#61; null &#125;&#41;&#10;  tag_bindings &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;  service_accounts &#61; optional&#40;map&#40;object&#40;&#123;&#10;    display_name      &#61; optional&#40;string, &#34;Terraform-managed.&#34;&#41;&#10;    iam_project_roles &#61; optional&#40;list&#40;string&#41;&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [data_merges](variables.tf#L49) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | <code title="object&#40;&#123;&#10;  contacts                   &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  labels                     &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;  metric_scopes              &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  service_encryption_key_ids &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10;  service_perimeter_bridges  &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  services                   &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10;  tag_bindings               &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10;  service_accounts &#61; optional&#40;map&#40;object&#40;&#123;&#10;    display_name      &#61; optional&#40;string, &#34;Terraform-managed.&#34;&#41;&#10;    iam_project_roles &#61; optional&#40;list&#40;string&#41;&#41;&#10;  &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [data_overrides](variables.tf#L69) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | <code title="object&#40;&#123;&#10;  billing_account            &#61; optional&#40;string&#41;&#10;  contacts                   &#61; optional&#40;map&#40;list&#40;string&#41;&#41;&#41;&#10;  parent                     &#61; optional&#40;string&#41;&#10;  prefix                     &#61; optional&#40;string&#41;&#10;  service_encryption_key_ids &#61; optional&#40;map&#40;list&#40;string&#41;&#41;&#41;&#10;  service_perimeter_bridges  &#61; optional&#40;list&#40;string&#41;&#41;&#10;  service_perimeter_standard &#61; optional&#40;string&#41;&#10;  tag_bindings               &#61; optional&#40;map&#40;string&#41;&#41;&#10;  services                   &#61; optional&#40;list&#40;string&#41;&#41;&#10;  service_accounts &#61; optional&#40;map&#40;object&#40;&#123;&#10;    display_name      &#61; optional&#40;string, &#34;Terraform-managed.&#34;&#41;&#10;    iam_project_roles &#61; optional&#40;list&#40;string&#41;&#41;&#10;  &#125;&#41;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>&#123;&#125;</code> |
@@ -144,7 +234,6 @@ services:
 | [projects](outputs.tf#L17) | Project module outputs. |  |
 | [service_accounts](outputs.tf#L22) | Service account emails. |  |
 <!-- END TFDOC -->
-
 ## Tests
 
 These tests validate fixes to the project factory.
@@ -166,7 +255,9 @@ module "project-factory" {
   data_overrides = {
     prefix = "foo"
   }
-  factory_data_path = "data"
+  factories_config = {
+    projects_data_path = "data/projects"
+  }
 }
 # tftest modules=4 resources=14 files=test-0,test-1,test-2
 ```
@@ -177,21 +268,21 @@ services:
   - iam.googleapis.com
   - contactcenteraiplatform.googleapis.com
   - container.googleapis.com
-# tftest-file id=test-0 path=data/test-0.yaml
+# tftest-file id=test-0 path=data/projects/test-0.yaml
 ```
 
 ```yaml
 parent: folders/1234567890
 services:
   - iam.googleapis.com
   - contactcenteraiplatform.googleapis.com
-# tftest-file id=test-1 path=data/test-1.yaml
+# tftest-file id=test-1 path=data/projects/test-1.yaml
 ```
 
 ```yaml
 parent: folders/1234567890
 services:
   - iam.googleapis.com
   - storage.googleapis.com
-# tftest-file id=test-2 path=data/test-2.yaml
+# tftest-file id=test-2 path=data/projects/test-2.yaml
 ```