Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for billing budgets to project factory #2112

Merged
merged 7 commits into from
Feb 27, 2024
Merged

Add support for billing budgets to project factory #2112

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6e2dbe4
align factory variable name in project factory module
ludoo Feb 27, 2024
d033e82
tested
ludoo Feb 27, 2024
e4be5cc
align fast stage
ludoo Feb 27, 2024
19ff107
Merge branch 'master' into ludo/pf-billing-budgets
ludoo Feb 27, 2024
162cbcd
Update modules/project-factory/README.md
ludoo Feb 27, 2024
a2dc4aa
Update README.md
ludoo Feb 27, 2024
867e2ec
add toc to project factory README
ludoo Feb 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 4 additions & 1 deletion modules/billing-account/README.md
View file
Open in desktop
Expand Up @@ -230,6 +230,9 @@ module "billing-account" {
}
}
}
factories_config = {
budgets_data_path = "data/billing-budgets"
}
}
# tftest modules=1 resources=2 files=test-1 inventory=budget-monitoring-channel.yaml
```
Expand Down Expand Up @@ -268,7 +271,7 @@ update_rules:
| [iam_bindings](variables-iam.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | <code title="map&#40;object&#40;&#123;&#10; members &#61; list&#40;string&#41;&#10; role &#61; string&#10; condition &#61; optional&#40;object&#40;&#123;&#10; expression &#61; string&#10; title &#61; string&#10; description &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam_bindings_additive](variables-iam.tf#L39) | Individual additive IAM bindings. Keys are arbitrary. | <code title="map&#40;object&#40;&#123;&#10; member &#61; string&#10; role &#61; string&#10; condition &#61; optional&#40;object&#40;&#123;&#10; expression &#61; string&#10; title &#61; string&#10; description &#61; optional&#40;string&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [iam_by_principals](variables-iam.tf#L54) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | <code>map&#40;list&#40;string&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [logging_sinks](variables.tf#L135) | Logging sinks to create for the organization. | <code title="map&#40;object&#40;&#123;&#10; destination &#61; string&#10; type &#61; string&#10; bq_partitioned_table &#61; optional&#40;bool, false&#41;&#10; description &#61; optional&#40;string&#41;&#10; disabled &#61; optional&#40;bool, false&#41;&#10; exclusions &#61; optional&#40;map&#40;object&#40;&#123;&#10; filter &#61; string&#10; description &#61; optional&#40;string&#41;&#10; disabled &#61; optional&#40;bool&#41;&#10; &#125;&#41;&#41;, &#123;&#125;&#41;&#10; filter &#61; optional&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [logging_sinks](variables.tf#L135) | Logging sinks to create for the billing account. | <code title="map&#40;object&#40;&#123;&#10; destination &#61; string&#10; type &#61; string&#10; bq_partitioned_table &#61; optional&#40;bool, false&#41;&#10; description &#61; optional&#40;string&#41;&#10; disabled &#61; optional&#40;bool, false&#41;&#10; exclusions &#61; optional&#40;map&#40;object&#40;&#123;&#10; filter &#61; string&#10; description &#61; optional&#40;string&#41;&#10; disabled &#61; optional&#40;bool&#41;&#10; &#125;&#41;&#41;, &#123;&#125;&#41;&#10; filter &#61; optional&#40;string&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> | | <code>&#123;&#125;</code> |
| [projects](variables.tf#L168) | Projects associated with this billing account. | <code>list&#40;string&#41;</code> | | <code>&#91;&#93;</code> |

## Outputs
Expand Down
2 changes: 2 additions & 0 deletions modules/billing-account/factory.tf
View file
Open in desktop
Expand Up @@ -14,6 +14,8 @@
* limitations under the License.
*/

# any changes to this factory should be mirrored in the project factory

locals {
_factory_data = {
for f in fileset("${local._factory_path}", "**/*.yaml") :
Expand Down
2 changes: 1 addition & 1 deletion modules/billing-account/variables.tf
View file
Open in desktop
Expand Up @@ -133,7 +133,7 @@ variable "id" {
}

variable "logging_sinks" {
description = "Logging sinks to create for the organization."
description = "Logging sinks to create for the billing account."
type = map(object({
destination = string
type = string
Expand Down
1 change: 1 addition & 0 deletions modules/project-factory/.gitignore
View file
Open in desktop
@@ -0,0 +1 @@
data
121 changes: 100 additions & 21 deletions modules/project-factory/README.md
View file
Open in desktop
Expand Up @@ -4,12 +4,10 @@ This module implements in code the end-to-end project creation process for multi

It supports

- all project-level attributes exposed by the [project module](../project/), including Shared VPC host/service configuration
- optional service account creation in the project, including basic IAM grants
- KMS key encrypt/decrypt permissions for service identities in the project
- membership in VPC SC standard or bridge perimeters
- billing budgets (TODO)
- per-project IaC configuration (TODO)
- multiple project creation and management exposing the full configuration options available in the [project module](../project/), including KMS key grants and VPC-SC perimeter membership
- optional per-project [service account management](#service-accounts) including basic IAM grants
- optional billing budgets factory and budget/project associations (TODO)
- optional per-project IaC configuration (TODO)

The factory is implemented as a thin wrapping layer, so that no "magic" or hidden side effects are implemented in code, and debugging or integration of new features are simple.

Expand All @@ -26,7 +24,49 @@ In addition to the YAML-based project configurations, the factory accepts three
- the `data_overrides` variable works similarly to defaults, but the values specified here take precedence over those in YAML files
- the `data_merges` variable allows specifying additional values for map or set based variables, which are merged with the data coming from YAML

Some examples on where to use each of the three sets are provided below.
Some examples on where to use each of the three sets are [provided below](#example).

## Additional resources

### Service accounts

Service accounts can be managed as part of each project's YAML configuration. This allows creation of default service accounts used for GCE instances, in firewall rules, or for application-level credentials without resorting to a separate Terraform configuration.

Each service account is represented by one key and a set of optional key/value pairs in the `service_accounts` top-evel YAML map, like in this example:
ludoo marked this conversation as resolved.
Show resolved Hide resolved

```yaml
service_accounts:
be-0: {}
fe-1:
display_name: GCE frontend service account.
iam_project_roles:
- roles/storage.objectViewer
```

Both the `display_name` and `iam_project_roles` attributes are optional.

### Billing budgets

The project factory integrates the billing budgets factory exposed by the `[`billing-account`](../billing-account/) module, and adds support for easy referencing budgets in project files.

To enable support for billing budgets, set the billing account id, optional notification channels, and the data folder for budgets in the `factories_config.budgets` variable, then create billing budgets using YAML definitions following the format described in the `billing-account` module.

Once budgets are defined, they can be referenced in a project file using their file name:

```yaml
billing_account: 012345-67890A-BCDEF0
labels:
app: app-1
team: foo
parent: folders/12345678
services:
- container.googleapis.com
- storage.googleapis.com
billing_budgets:
- test-100
```

The example below shows how to use the billing budgets factory.

## Example

Expand All @@ -35,7 +75,7 @@ module "project-factory" {
source = "./fabric/modules/project-factory"
# use a default billing account if none is specified via yaml
data_defaults = {
billing_account = "012345-67890A-ABCDEF"
billing_account = var.billing_account_id
}
# make sure the environment label and stackdriver service are always added
data_merges = {
Expand All @@ -54,12 +94,28 @@ module "project-factory" {
prefix = "test-pf"
}
# location where the yaml files are read from
factory_data_path = "data"
factories_config = {
budgets = {
billing_account = var.billing_account_id
budgets_data_path = "data/budgets"
notification_channels = {
billing-default = {
project_id = "foo-billing-audit"
type = "email"
labels = {
email_address = "gcp-billing-admins@example.com"
}
}
}
}
projects_data_path = "data/projects"
}
}
# tftest modules=7 resources=33 files=prj-app-1,prj-app-2,prj-app-3 inventory=example.yaml
# tftest modules=8 resources=35 files=prj-app-1,prj-app-2,prj-app-3,budget-test-100 inventory=example.yaml
```

```yaml
# project app-1
billing_account: 012345-67890A-BCDEF0
labels:
app: app-1
Expand All @@ -78,11 +134,13 @@ service_accounts:
- roles/monitoring.metricWriter
app-1-fe:
display_name: "Test app 1 frontend."

# tftest-file id=prj-app-1 path=data/prj-app-1.yaml
billing_budgets:
- test-100
# tftest-file id=prj-app-1 path=data/projects/prj-app-1.yaml
```

```yaml
# project app-2
labels:
app: app-2
team: foo
Expand Down Expand Up @@ -115,24 +173,45 @@ shared_vpc_service_config:
europe-west1/prod-default-ew1:
- group:team-1@example.com

# tftest-file id=prj-app-2 path=data/prj-app-2.yaml
# tftest-file id=prj-app-2 path=data/projects/prj-app-2.yaml
```

```yaml
# project app-3
parent: folders/12345678
services:
- run.googleapis.com
- storage.googleapis.com

# tftest-file id=prj-app-3 path=data/prj-app-3.yaml
# tftest-file id=prj-app-3 path=data/projects/prj-app-3.yaml
```

```yaml
# billing budget test-100
display_name: 100 dollars in current spend
amount:
units: 100
filter:
period:
calendar: MONTH
resource_ancestors:
- folders/1234567890
threshold_rules:
- percent: 0.5
- percent: 0.75
update_rules:
default:
disable_default_iam_recipients: true
monitoring_notification_channels:
- billing-default
# tftest-file id=budget-test-100 path=data/budgets/test-100.yaml
```
<!-- BEGIN TFDOC -->
## Variables

| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [factory_data_path](variables.tf#L91) | Path to folder with YAML project description data files. | <code>string</code> | ✓ | |
| [factories_config](variables.tf#L91) | Path to folder with YAML resource description data files. | <code title="object&#40;&#123;&#10; projects_data_path &#61; string&#10; budgets &#61; optional&#40;object&#40;&#123;&#10; billing_account &#61; string&#10; budgets_data_path &#61; string&#10; notification_channels &#61; optional&#40;map&#40;any&#41;, &#123;&#125;&#41;&#10; &#125;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | ✓ | |
| [data_defaults](variables.tf#L17) | Optional default values used when corresponding project data from files are missing. | <code title="object&#40;&#123;&#10; billing_account &#61; optional&#40;string&#41;&#10; contacts &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; labels &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; metric_scopes &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; parent &#61; optional&#40;string&#41;&#10; prefix &#61; optional&#40;string&#41;&#10; service_encryption_key_ids &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; service_perimeter_bridges &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; service_perimeter_standard &#61; optional&#40;string&#41;&#10; services &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; shared_vpc_service_config &#61; optional&#40;object&#40;&#123;&#10; host_project &#61; string&#10; network_users &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; service_identity_iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; service_identity_subnet_iam &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; service_iam_grants &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; network_subnet_users &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; &#125;&#41;, &#123; host_project &#61; null &#125;&#41;&#10; tag_bindings &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; service_accounts &#61; optional&#40;map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string, &#34;Terraform-managed.&#34;&#41;&#10; iam_project_roles &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> |
| [data_merges](variables.tf#L49) | Optional values that will be merged with corresponding data from files. Combines with `data_defaults`, file data, and `data_overrides`. | <code title="object&#40;&#123;&#10; contacts &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; labels &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; metric_scopes &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; service_encryption_key_ids &#61; optional&#40;map&#40;list&#40;string&#41;&#41;, &#123;&#125;&#41;&#10; service_perimeter_bridges &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; services &#61; optional&#40;list&#40;string&#41;, &#91;&#93;&#41;&#10; tag_bindings &#61; optional&#40;map&#40;string&#41;, &#123;&#125;&#41;&#10; service_accounts &#61; optional&#40;map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string, &#34;Terraform-managed.&#34;&#41;&#10; iam_project_roles &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;, &#123;&#125;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> |
| [data_overrides](variables.tf#L69) | Optional values that override corresponding data from files. Takes precedence over file data and `data_defaults`. | <code title="object&#40;&#123;&#10; billing_account &#61; optional&#40;string&#41;&#10; contacts &#61; optional&#40;map&#40;list&#40;string&#41;&#41;&#41;&#10; parent &#61; optional&#40;string&#41;&#10; prefix &#61; optional&#40;string&#41;&#10; service_encryption_key_ids &#61; optional&#40;map&#40;list&#40;string&#41;&#41;&#41;&#10; service_perimeter_bridges &#61; optional&#40;list&#40;string&#41;&#41;&#10; service_perimeter_standard &#61; optional&#40;string&#41;&#10; tag_bindings &#61; optional&#40;map&#40;string&#41;&#41;&#10; services &#61; optional&#40;list&#40;string&#41;&#41;&#10; service_accounts &#61; optional&#40;map&#40;object&#40;&#123;&#10; display_name &#61; optional&#40;string, &#34;Terraform-managed.&#34;&#41;&#10; iam_project_roles &#61; optional&#40;list&#40;string&#41;&#41;&#10; &#125;&#41;&#41;&#41;&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> | | <code>&#123;&#125;</code> |
Expand All @@ -141,10 +220,8 @@ services:

| name | description | sensitive |
|---|---|:---:|
| [projects](outputs.tf#L17) | Project module outputs. | |
| [service_accounts](outputs.tf#L22) | Service account emails. | |
<!-- END TFDOC -->

## Tests

These tests validate fixes to the project factory.
Expand All @@ -166,7 +243,9 @@ module "project-factory" {
data_overrides = {
prefix = "foo"
}
factory_data_path = "data"
factories_config = {
projects_data_path = "data/projects"
}
}
# tftest modules=4 resources=14 files=test-0,test-1,test-2
```
Expand All @@ -177,21 +256,21 @@ services:
- iam.googleapis.com
- contactcenteraiplatform.googleapis.com
- container.googleapis.com
# tftest-file id=test-0 path=data/test-0.yaml
# tftest-file id=test-0 path=data/projects/test-0.yaml
```

```yaml
parent: folders/1234567890
services:
- iam.googleapis.com
- contactcenteraiplatform.googleapis.com
# tftest-file id=test-1 path=data/test-1.yaml
# tftest-file id=test-1 path=data/projects/test-1.yaml
```

```yaml
parent: folders/1234567890
services:
- iam.googleapis.com
- storage.googleapis.com
# tftest-file id=test-2 path=data/test-2.yaml
# tftest-file id=test-2 path=data/projects/test-2.yaml
```
74 changes: 74 additions & 0 deletions modules/project-factory/factory-budgets.tf
View file
Open in desktop
@@ -0,0 +1,74 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

locals {
# reimplement the billing account factory here to interpolate projects
_budget_path = try(pathexpand(var.factories_config.budgets.budgets_data_path), null)
_budgets = (
{
for f in try(fileset(local._budget_path, "**/*.yaml"), []) :
trimsuffix(f, ".yaml") => yamldecode(file("${local._budget_path}/${f}"))
}
)
budgets = {
for k, v in local._budgets : k => merge(v, {
amount = merge(
{
currency_code = null
nanos = null
units = null
use_last_period = null
},
try(v.amount, {})
)
display_name = try(v.display_name, null)
filter = try(v.filter, null) == null ? null : {
credit_types_treatment = (
try(v.filter.credit_types_treatment, null) == null
? null
: merge(
{ exclude_all = null, include_specified = null },
v.filter.credit_types_treatment
)
)
label = try(v.filter.label, null)
projects = concat(
try(v.projects, []),
[
for p in lookup(local.project_budgets, k, []) :
"projects/${module.projects[p].project_id}"
]
)
resource_ancestors = try(v.filter.resource_ancestors, null)
services = try(v.filter.services, null)
subaccounts = try(v.filter.subaccounts, null)
}
threshold_rules = [
for vv in try(v.threshold_rules, []) : merge({
percent = null
forecasted_spend = null
}, vv)
]
update_rules = {
for kk, vv in try(v.update_rules, {}) : kk => merge({
disable_default_iam_recipients = null
monitoring_notification_channels = null
pubsub_topic = null
}, vv)
}
})
}
}
24 changes: 16 additions & 8 deletions modules/project-factory/factory.tf → modules/project-factory/factory-projects.tf
View file
Open in desktop
Expand Up @@ -13,19 +13,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

locals {
_data = (
_project_path = try(pathexpand(var.factories_config.projects_data_path), null)
_projects = (
{
for f in fileset(local._data_path, "**/*.yaml") :
trimsuffix(f, ".yaml") => yamldecode(file("${local._data_path}/${f}"))
for f in try(fileset(local._project_path, "**/*.yaml"), []) :
trimsuffix(f, ".yaml") => yamldecode(file("${local._project_path}/${f}"))
}
)
_data_path = var.factory_data_path == null ? null : pathexpand(
var.factory_data_path
)
_project_budgets = flatten([
for k, v in local._projects : [
for b in try(v.billing_budgets, []) : {
budget = b
project = k
}
]
])
project_budgets = {
for v in local._project_budgets : v.budget => v.project...
}
projects = {
for k, v in local._data : k => merge(v, {
for k, v in local._projects : k => merge(v, {
billing_account = try(coalesce(
var.data_overrides.billing_account,
try(v.billing_account, null),
Expand Down
10 changes: 10 additions & 0 deletions modules/project-factory/main.tf
View file
Open in desktop
Expand Up @@ -76,3 +76,13 @@ module "service-accounts" {
(module.projects[each.value.project].project_id) = each.value.iam_project_roles
}
}

module "billing-account" {
source = "../billing-account"
count = var.factories_config.budgets == null ? 0 : 1
id = var.factories_config.budgets.billing_account
budget_notification_channels = (
var.factories_config.budgets.notification_channels
)
budgets = local.budgets
}