From 2347d0a19f7f83b6e803fe7f1957e6e0ddb65bfb Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Wed, 20 Jul 2022 19:07:43 -0300 Subject: [PATCH 01/21] Fix Copyright year --- modules/logbucket/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/logbucket/outputs.tf b/modules/logbucket/outputs.tf index 5b48d2eb..410cb900 100644 --- a/modules/logbucket/outputs.tf +++ b/modules/logbucket/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From cd18efc3a18a5cad8d709542bc209fa1348700a1 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Wed, 20 Jul 2022 19:08:40 -0300 Subject: [PATCH 02/21] New variable to deal with sink and logbucket in same project --- modules/logbucket/README.md | 1 + modules/logbucket/variables.tf | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/logbucket/README.md b/modules/logbucket/README.md index f96dc91f..912d874b 100644 --- a/modules/logbucket/README.md +++ b/modules/logbucket/README.md @@ -42,6 +42,7 @@ module "destination" { | name | The name of the log bucket to be created and used for log entries matching the filter. | `string` | n/a | yes | | project\_id | The ID of the project in which the log bucket will be created. | `string` | n/a | yes | | retention\_days | The number of days data should be retained for the log bucket. | `number` | `30` | no | +| sink\_and\_bucket\_in\_same\_project | (Optional) Indicates if the sink and logging bucket are in the same project. When the sink route logs between Logging buckets in the same Cloud project, no new service account need to be created. | `bool` | `false` | no | ## Outputs diff --git a/modules/logbucket/variables.tf b/modules/logbucket/variables.tf index f39133a4..4cf54195 100644 --- a/modules/logbucket/variables.tf +++ b/modules/logbucket/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2019 Google LLC + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,3 +40,9 @@ variable "retention_days" { type = number default = 30 } + +variable "sink_and_bucket_in_same_project" { + description = "(Optional) Indicates if the sink and logging bucket are in the same project. When the sink route logs between Logging buckets in the same Cloud project, no new service account need to be created." + type = bool + default = false +} From 40bf3065af1d4bd139b40a181f505520dad0d91b Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Wed, 20 Jul 2022 19:12:33 -0300 Subject: [PATCH 03/21] When sink and logbucket in same project no new service account is created --- modules/logbucket/main.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/logbucket/main.tf b/modules/logbucket/main.tf index 2b758824..75c0a89f 100644 --- a/modules/logbucket/main.tf +++ b/modules/logbucket/main.tf @@ -43,6 +43,8 @@ resource "google_logging_project_bucket_config" "bucket" { # Service account IAM membership # #--------------------------------# resource "google_project_iam_member" "logbucket_sink_member" { + count = !var.sink_and_bucket_in_same_project ? 1 : 0 + project = google_logging_project_bucket_config.bucket.project role = "roles/logging.bucketWriter" member = var.log_sink_writer_identity From a047e253b5b7fc15623ccb3e7f8daa493ff10ba9 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Wed, 20 Jul 2022 19:13:41 -0300 Subject: [PATCH 04/21] New examples for logbucket at project-level --- examples/logbucket/project/README.md | 26 ++++++++++ examples/logbucket/project/main.tf | 67 +++++++++++++++++++++++++ examples/logbucket/project/outputs.tf | 58 +++++++++++++++++++++ examples/logbucket/project/variables.tf | 25 +++++++++ examples/logbucket/project/versions.tf | 29 +++++++++++ 5 files changed, 205 insertions(+) create mode 100644 examples/logbucket/project/README.md create mode 100644 examples/logbucket/project/main.tf create mode 100644 examples/logbucket/project/outputs.tf create mode 100644 examples/logbucket/project/variables.tf create mode 100644 examples/logbucket/project/versions.tf diff --git a/examples/logbucket/project/README.md b/examples/logbucket/project/README.md new file mode 100644 index 00000000..c3bcb7d0 --- /dev/null +++ b/examples/logbucket/project/README.md @@ -0,0 +1,26 @@ +# Log Export: Log Bucket destination at Folder level + +These examples configures a project-level log sink that feeds a logging log bucket destination with log bucket and log sink in the same project and in separated projects. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| parent\_resource\_project | The ID of the project in which the log export will be created. | `string` | n/a | yes | +| project\_id | The ID of the project in which log bucket destination will be created. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| log\_bucket\_name | The name for the log bucket. | +| log\_bucket\_name\_same\_project\_example | The name for the log bucket for sink and logbucket in same project example. | +| log\_bucket\_project | The project where the log bucket is created. | +| log\_bucket\_project\_same\_project\_example | The project where the log bucket is created for sink and logbucket in same project example. | +| log\_sink\_destination\_uri | A fully qualified URI for the log sink. | +| log\_sink\_destination\_uri\_same\_project\_example | A fully qualified URI for the log sink for sink and logbucket in same project example. | +| log\_sink\_folder\_id\_same\_project\_example | The folder id where the log sink is created for sink and logbucket in same project example. | +| log\_sink\_project\_id | The project id where the log sink is created. | + + diff --git a/examples/logbucket/project/main.tf b/examples/logbucket/project/main.tf new file mode 100644 index 00000000..c5e0e47c --- /dev/null +++ b/examples/logbucket/project/main.tf @@ -0,0 +1,67 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + upper = false + special = false +} + +resource "random_string" "suffix_same_project_example" { + length = 4 + upper = false + special = false +} + +module "log_export" { + source = "../../../" + destination_uri = module.destination.destination_uri + filter = "resource.type = gce_instance" + log_sink_name = "logbucket_project_${random_string.suffix.result}" + parent_resource_id = var.parent_resource_project + parent_resource_type = "project" + unique_writer_identity = true +} + +module "destination" { + source = "../../..//modules/logbucket" + project_id = var.project_id + name = "logbucket_project_${random_string.suffix.result}" + location = "global" + log_sink_writer_identity = module.log_export.writer_identity +} + +#-------------------------------------# +# Log Bucket and Sink in same project # +#-------------------------------------# +module "log_export_same_project_example" { + source = "../../../" + destination_uri = module.destination_same_project_example.destination_uri + filter = "resource.type = gce_instance" + log_sink_name = "logbucket_folder_${random_string.suffix_same_project_example.result}" + parent_resource_id = var.project_id + parent_resource_type = "project" + unique_writer_identity = true +} + +module "destination_same_project_example" { + source = "../../..//modules/logbucket" + project_id = var.project_id + name = "logbucket_project_${random_string.suffix.result}" + location = "global" + log_sink_writer_identity = module.log_export_same_project_example.writer_identity + sink_and_bucket_in_same_project = true +} diff --git a/examples/logbucket/project/outputs.tf b/examples/logbucket/project/outputs.tf new file mode 100644 index 00000000..f83be2be --- /dev/null +++ b/examples/logbucket/project/outputs.tf @@ -0,0 +1,58 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "log_bucket_project" { + description = "The project where the log bucket is created." + value = module.destination.project +} + +output "log_bucket_name" { + description = "The name for the log bucket." + value = module.destination.resource_name +} + +output "log_sink_project_id" { + description = "The project id where the log sink is created." + value = module.log_export.parent_resource_id +} + +output "log_sink_destination_uri" { + description = "A fully qualified URI for the log sink." + value = module.destination.destination_uri +} + +#-------------------------------------# +# Log Bucket and Sink in same project # +#-------------------------------------# +output "log_bucket_project_same_project_example" { + description = "The project where the log bucket is created for sink and logbucket in same project example." + value = module.destination_same_project_example.project +} + +output "log_bucket_name_same_project_example" { + description = "The name for the log bucket for sink and logbucket in same project example." + value = module.destination_same_project_example.resource_name +} + +output "log_sink_folder_id_same_project_example" { + description = "The folder id where the log sink is created for sink and logbucket in same project example." + value = module.log_export_same_project_example.parent_resource_id +} + +output "log_sink_destination_uri_same_project_example" { + description = "A fully qualified URI for the log sink for sink and logbucket in same project example." + value = module.destination_same_project_example.destination_uri +} diff --git a/examples/logbucket/project/variables.tf b/examples/logbucket/project/variables.tf new file mode 100644 index 00000000..8578b52a --- /dev/null +++ b/examples/logbucket/project/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project in which log bucket destination will be created." + type = string +} + +variable "parent_resource_project" { + description = "The ID of the project in which the log export will be created." + type = string +} diff --git a/examples/logbucket/project/versions.tf b/examples/logbucket/project/versions.tf new file mode 100644 index 00000000..9bd2103e --- /dev/null +++ b/examples/logbucket/project/versions.tf @@ -0,0 +1,29 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +terraform { + required_version = ">= 0.13" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + random = { + source = "hashicorp/random" + } + } +} From 79dc77b9844ea8fe9ceef09b6ef387f1e8cd9b95 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Thu, 28 Jul 2022 16:33:38 -0300 Subject: [PATCH 05/21] Changed name of new variable to a meaningful name --- modules/logbucket/README.md | 2 +- modules/logbucket/main.tf | 2 +- modules/logbucket/variables.tf | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/logbucket/README.md b/modules/logbucket/README.md index 912d874b..81799f85 100644 --- a/modules/logbucket/README.md +++ b/modules/logbucket/README.md @@ -37,12 +37,12 @@ module "destination" { | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| grant\_write\_permission\_on\_bkt | (Optional) Indicates whether the module is responsible for granting write permission on the logbucket. This permission will be given by default, but if the user wants, this module can skip this step. This is the case when the sink route logs to a log bucket in the same Cloud project, no new service account will be created and this module will need to bypass granting permissions. | `bool` | `true` | no | | location | The location of the log bucket. | `string` | `"global"` | no | | log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. (This is available as an output coming from the root module). | `string` | n/a | yes | | name | The name of the log bucket to be created and used for log entries matching the filter. | `string` | n/a | yes | | project\_id | The ID of the project in which the log bucket will be created. | `string` | n/a | yes | | retention\_days | The number of days data should be retained for the log bucket. | `number` | `30` | no | -| sink\_and\_bucket\_in\_same\_project | (Optional) Indicates if the sink and logging bucket are in the same project. When the sink route logs between Logging buckets in the same Cloud project, no new service account need to be created. | `bool` | `false` | no | ## Outputs diff --git a/modules/logbucket/main.tf b/modules/logbucket/main.tf index 75c0a89f..356e1a1e 100644 --- a/modules/logbucket/main.tf +++ b/modules/logbucket/main.tf @@ -43,7 +43,7 @@ resource "google_logging_project_bucket_config" "bucket" { # Service account IAM membership # #--------------------------------# resource "google_project_iam_member" "logbucket_sink_member" { - count = !var.sink_and_bucket_in_same_project ? 1 : 0 + count = var.grant_write_permission_on_bkt ? 1 : 0 project = google_logging_project_bucket_config.bucket.project role = "roles/logging.bucketWriter" diff --git a/modules/logbucket/variables.tf b/modules/logbucket/variables.tf index 4cf54195..a6380378 100644 --- a/modules/logbucket/variables.tf +++ b/modules/logbucket/variables.tf @@ -41,8 +41,8 @@ variable "retention_days" { default = 30 } -variable "sink_and_bucket_in_same_project" { - description = "(Optional) Indicates if the sink and logging bucket are in the same project. When the sink route logs between Logging buckets in the same Cloud project, no new service account need to be created." +variable "grant_write_permission_on_bkt" { + description = "(Optional) Indicates whether the module is responsible for granting write permission on the logbucket. This permission will be given by default, but if the user wants, this module can skip this step. This is the case when the sink route logs to a log bucket in the same Cloud project, no new service account will be created and this module will need to bypass granting permissions." type = bool - default = false + default = true } From 118e72c7092f763cafd25d80bd718aefb9e4bbe4 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Thu, 28 Jul 2022 16:36:43 -0300 Subject: [PATCH 06/21] Created example of logbucket with project usage --- examples/logbucket/project/README.md | 8 ++++-- examples/logbucket/project/main.tf | 35 +++++++++---------------- examples/logbucket/project/outputs.tf | 25 ++++++++++++++++-- examples/logbucket/project/variables.tf | 2 +- 4 files changed, 42 insertions(+), 28 deletions(-) diff --git a/examples/logbucket/project/README.md b/examples/logbucket/project/README.md index c3bcb7d0..db604367 100644 --- a/examples/logbucket/project/README.md +++ b/examples/logbucket/project/README.md @@ -8,7 +8,7 @@ These examples configures a project-level log sink that feeds a logging log buck | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | parent\_resource\_project | The ID of the project in which the log export will be created. | `string` | n/a | yes | -| project\_id | The ID of the project in which log bucket destination will be created. | `string` | n/a | yes | +| project\_destination\_logbkt\_id | The ID of the project in which log bucket destination will be created. | `string` | n/a | yes | ## Outputs @@ -20,7 +20,11 @@ These examples configures a project-level log sink that feeds a logging log buck | log\_bucket\_project\_same\_project\_example | The project where the log bucket is created for sink and logbucket in same project example. | | log\_sink\_destination\_uri | A fully qualified URI for the log sink. | | log\_sink\_destination\_uri\_same\_project\_example | A fully qualified URI for the log sink for sink and logbucket in same project example. | -| log\_sink\_folder\_id\_same\_project\_example | The folder id where the log sink is created for sink and logbucket in same project example. | | log\_sink\_project\_id | The project id where the log sink is created. | +| log\_sink\_project\_id\_same\_project\_example | The project id where the log sink is created for sink and logbucket in same project example. | +| log\_sink\_resource\_name | The resource name of the log sink that was created. | +| log\_sink\_resource\_name\_same\_project\_example | The resource name of the log sink that was created in same project example. | +| log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. | +| log\_sink\_writer\_identity\_same\_project\_example | The service account in same project example that logging uses to write log entries to the destination. | diff --git a/examples/logbucket/project/main.tf b/examples/logbucket/project/main.tf index c5e0e47c..89f497f5 100644 --- a/examples/logbucket/project/main.tf +++ b/examples/logbucket/project/main.tf @@ -14,23 +14,11 @@ * limitations under the License. */ -resource "random_string" "suffix" { - length = 4 - upper = false - special = false -} - -resource "random_string" "suffix_same_project_example" { - length = 4 - upper = false - special = false -} - module "log_export" { source = "../../../" destination_uri = module.destination.destination_uri filter = "resource.type = gce_instance" - log_sink_name = "logbucket_project_${random_string.suffix.result}" + log_sink_name = "logbucket_other_project" parent_resource_id = var.parent_resource_project parent_resource_type = "project" unique_writer_identity = true @@ -38,8 +26,8 @@ module "log_export" { module "destination" { source = "../../..//modules/logbucket" - project_id = var.project_id - name = "logbucket_project_${random_string.suffix.result}" + project_id = var.project_destination_logbkt_id + name = "logbucket_project_from_${var.parent_resource_project}" location = "global" log_sink_writer_identity = module.log_export.writer_identity } @@ -51,17 +39,18 @@ module "log_export_same_project_example" { source = "../../../" destination_uri = module.destination_same_project_example.destination_uri filter = "resource.type = gce_instance" - log_sink_name = "logbucket_folder_${random_string.suffix_same_project_example.result}" - parent_resource_id = var.project_id + log_sink_name = "logbucket_same_project" + parent_resource_id = var.project_destination_logbkt_id parent_resource_type = "project" unique_writer_identity = true } module "destination_same_project_example" { - source = "../../..//modules/logbucket" - project_id = var.project_id - name = "logbucket_project_${random_string.suffix.result}" - location = "global" - log_sink_writer_identity = module.log_export_same_project_example.writer_identity - sink_and_bucket_in_same_project = true + source = "../../..//modules/logbucket" + project_id = var.project_destination_logbkt_id + name = "logbucket_project_from_${var.project_destination_logbkt_id}" + location = "global" + log_sink_writer_identity = module.log_export_same_project_example.writer_identity + grant_write_permission_on_bkt = false + retention_days = 20 } diff --git a/examples/logbucket/project/outputs.tf b/examples/logbucket/project/outputs.tf index f83be2be..f42761d9 100644 --- a/examples/logbucket/project/outputs.tf +++ b/examples/logbucket/project/outputs.tf @@ -34,6 +34,17 @@ output "log_sink_destination_uri" { value = module.destination.destination_uri } +output "log_sink_resource_name" { + description = "The resource name of the log sink that was created." + value = module.log_export.log_sink_resource_name +} + +output "log_sink_writer_identity" { + description = "The service account that logging uses to write log entries to the destination." + value = module.log_export.writer_identity +} + + #-------------------------------------# # Log Bucket and Sink in same project # #-------------------------------------# @@ -47,8 +58,8 @@ output "log_bucket_name_same_project_example" { value = module.destination_same_project_example.resource_name } -output "log_sink_folder_id_same_project_example" { - description = "The folder id where the log sink is created for sink and logbucket in same project example." +output "log_sink_project_id_same_project_example" { + description = "The project id where the log sink is created for sink and logbucket in same project example." value = module.log_export_same_project_example.parent_resource_id } @@ -56,3 +67,13 @@ output "log_sink_destination_uri_same_project_example" { description = "A fully qualified URI for the log sink for sink and logbucket in same project example." value = module.destination_same_project_example.destination_uri } + +output "log_sink_resource_name_same_project_example" { + description = "The resource name of the log sink that was created in same project example." + value = module.log_export_same_project_example.log_sink_resource_name +} + +output "log_sink_writer_identity_same_project_example" { + description = "The service account in same project example that logging uses to write log entries to the destination." + value = module.log_export_same_project_example.writer_identity +} diff --git a/examples/logbucket/project/variables.tf b/examples/logbucket/project/variables.tf index 8578b52a..16bb70e5 100644 --- a/examples/logbucket/project/variables.tf +++ b/examples/logbucket/project/variables.tf @@ -14,7 +14,7 @@ * limitations under the License. */ -variable "project_id" { +variable "project_destination_logbkt_id" { description = "The ID of the project in which log bucket destination will be created." type = string } From ebc3e4b516a3af866594862e89930a469409a0ae Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Fri, 29 Jul 2022 09:04:37 -0300 Subject: [PATCH 07/21] Changed name of bucket due to colisions during integrated tests --- examples/logbucket/project/main.tf | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/logbucket/project/main.tf b/examples/logbucket/project/main.tf index 89f497f5..6c042fa8 100644 --- a/examples/logbucket/project/main.tf +++ b/examples/logbucket/project/main.tf @@ -14,6 +14,12 @@ * limitations under the License. */ +resource "random_string" "suffix" { + length = 4 + upper = false + special = false +} + module "log_export" { source = "../../../" destination_uri = module.destination.destination_uri @@ -27,7 +33,7 @@ module "log_export" { module "destination" { source = "../../..//modules/logbucket" project_id = var.project_destination_logbkt_id - name = "logbucket_project_from_${var.parent_resource_project}" + name = "logbucket_from_other_project_${random_string.suffix.result}" location = "global" log_sink_writer_identity = module.log_export.writer_identity } @@ -48,7 +54,7 @@ module "log_export_same_project_example" { module "destination_same_project_example" { source = "../../..//modules/logbucket" project_id = var.project_destination_logbkt_id - name = "logbucket_project_from_${var.project_destination_logbkt_id}" + name = "logbucket_from_same_project_${random_string.suffix.result}" location = "global" log_sink_writer_identity = module.log_export_same_project_example.writer_identity grant_write_permission_on_bkt = false From cf81d2fc3361bf986c6e89188ea88cfea73b81c4 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Fri, 29 Jul 2022 09:06:04 -0300 Subject: [PATCH 08/21] Setup a new project for logbucket project testing in same and other projects --- test/setup/iam.tf | 17 ++++++++++++++++- test/setup/main.tf | 33 +++++++++++++++++++++++++++++++-- test/setup/outputs.tf | 4 ++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 49109b3b..ffe3749c 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -87,6 +87,12 @@ resource "google_service_account" "int_test" { display_name = "ci-account" } +resource "google_service_account" "int_test_logbkt" { + project = module.project_destination_logbkt.project_id + account_id = "ci-account" + display_name = "ci-account" +} + resource "google_project_iam_member" "int_test" { for_each = toset(local.log_export_required_roles) @@ -95,6 +101,14 @@ resource "google_project_iam_member" "int_test" { member = "serviceAccount:${google_service_account.int_test.email}" } +resource "google_project_iam_member" "int_test_logbkt" { + for_each = toset(local.log_export_required_roles) + + project = module.project_destination_logbkt.project_id + role = each.value + member = "serviceAccount:${google_service_account.int_test.email}" +} + resource "google_billing_account_iam_member" "int_test" { for_each = toset(local.log_export_billing_account_roles) @@ -139,6 +153,7 @@ resource "null_resource" "wait_permissions" { google_billing_account_iam_member.int_test, google_folder_iam_member.int_test, google_organization_iam_member.int_test, - google_project_iam_member.int_test + google_project_iam_member.int_test, + google_project_iam_member.int_test_logbkt ] } diff --git a/test/setup/main.tf b/test/setup/main.tf index 63d87ba0..31c15eb7 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -41,12 +41,41 @@ module "project" { ] } +module "project_destination_logbkt" { + source = "terraform-google-modules/project-factory/google" + version = "~> 10.0" + + name = "ci-destination-logbkt" + random_project_id = true + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account + + activate_apis = [ + "cloudapis.googleapis.com", + "cloudbuild.googleapis.com", + "cloudfunctions.googleapis.com", + "cloudscheduler.googleapis.com", + "securitycenter.googleapis.com", + "cloudresourcemanager.googleapis.com", + "oslogin.googleapis.com", + "compute.googleapis.com", + "pubsub.googleapis.com", + "storage-component.googleapis.com", + "storage-api.googleapis.com", + "iam.googleapis.com", + "cloudbilling.googleapis.com" + ] +} + resource "null_resource" "wait_apis" { # Adding a pause as a workaround for of the provider issue # https://github.com/terraform-providers/terraform-provider-google/issues/1131 provisioner "local-exec" { command = "echo sleep 120s for APIs to get enabled; sleep 120" } - depends_on = [module.project.project_id] + depends_on = [ + module.project.project_id, + module.project_destination_logbkt.project_id + ] } - diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index 6b45b70b..5a4dd9bc 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -38,3 +38,7 @@ output "parent_resource_billing_account" { output "parent_resource_organization" { value = var.org_id } + +output "project_destination_logbkt_id" { + value = module.project_destination_logbkt.project_id +} From 151ee3e2e2e8da47e34221f8e56aea5b46cd01de Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Fri, 29 Jul 2022 09:06:55 -0300 Subject: [PATCH 09/21] Integrated tests to Logbucket Project example --- build/int.cloudbuild.yaml | 15 +++ .../logbucket_project_test.go | 104 ++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 test/integration/logbucket-project/logbucket_project_test.go diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 3ee402c2..1a881854 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -69,6 +69,21 @@ steps: - go-verify-logbucket-org name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestLogBucketOrgModule --stage teardown --verbose'] +- id: go-apply-logbucket-project + waitFor: + - init-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage apply --verbose'] +- id: go-verify-logbucket-project + waitFor: + - go-apply-logbucket-org + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage verify --verbose'] +- id: go-teardown-logbucket-project + waitFor: + - go-verify-logbucket-project + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage teardown --verbose'] tags: - 'ci' - 'integration' diff --git a/test/integration/logbucket-project/logbucket_project_test.go b/test/integration/logbucket-project/logbucket_project_test.go new file mode 100644 index 00000000..0efaef2f --- /dev/null +++ b/test/integration/logbucket-project/logbucket_project_test.go @@ -0,0 +1,104 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logbucket_project + +import ( + "fmt" + "testing" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/stretchr/testify/assert" +) + +func TestLogBucketProjectModule(t *testing.T) { + + const logApiFdqm = "logging.googleapis.com" + const RETENTION_DAYS = 20 + const SAMEPROJ_LOGSINK_WRITER_IDENTITY string = "" + const ROLE_BUCKET_WRITER string = "roles/logging.bucketWriter" + + insSimpleT := tft.NewTFBlueprintTest(t, + tft.WithTFDir("../../../examples/logbucket/project"), + ) + insSimpleT.DefineVerify(func(assert *assert.Assertions) { + insSimpleT.DefaultVerify(assert) + + //******************************* + // Bucket on other project test * + //******************************* + projectId := insSimpleT.GetStringOutput("log_bucket_project") + logBucketName := insSimpleT.GetStringOutput("log_bucket_name") + logBucketDestinationProjId := insSimpleT.GetStringOutput("log_bucket_project_same_project_example") + logSinkProjectId := insSimpleT.GetStringOutput("log_sink_project_id") + logSinkDestination := insSimpleT.GetStringOutput("log_sink_destination_uri") + logSinkName := insSimpleT.GetStringOutput("log_sink_resource_name") + logSinkWriterIdentity := insSimpleT.GetStringOutput("log_sink_writer_identity") + + logBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", logBucketName, "global", projectId)) + + // assert log bucket name, retention days & location + assert.Equal(logSinkDestination[len(logApiFdqm)+1:], logBucketDetails.Get("name").String(), "log bucket name should match") + assert.Equal(int64(30), logBucketDetails.Get("retentionDays").Int(), "retention days should match") + + logSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", logSinkName, logSinkProjectId)) + + // assert log sink name, destination & filter + assert.Equal(logSinkDestination, logSinkDetails.Get("destination").String(), "log sink destination should match") + assert.Equal("resource.type = gce_instance", logSinkDetails.Get("filter").String(), "log sink filter should match") + assert.Equal(logSinkWriterIdentity, logSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should not be empty") + + + //********************************** + // Bucket on the same project test * + //********************************** + sameProjId := insSimpleT.GetStringOutput("log_bucket_project_same_project_example") + sameProjLogBucketName := insSimpleT.GetStringOutput("log_bucket_name_same_project_example") + sameProjLogSinkProjectId := insSimpleT.GetStringOutput("log_sink_project_id_same_project_example") + sameProjLogSinkDestination := insSimpleT.GetStringOutput("log_sink_destination_uri_same_project_example") + sameProjLogSinkName := insSimpleT.GetStringOutput("log_sink_resource_name_same_project_example") + + assert.NotEqual(sameProjId, "a", "log sink destination should match") + assert.NotEqual(sameProjLogBucketName, "", "log sink destination should match") + assert.NotEqual(sameProjLogSinkProjectId, "", "log sink destination should match") + assert.NotEqual(sameProjLogSinkDestination, "", "log sink destination should match") + + sameProjLogBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", sameProjLogBucketName, "global", sameProjId)) + + // assert log bucket name, retention days & location + assert.Equal(sameProjLogSinkDestination[len(logApiFdqm)+1:], sameProjLogBucketDetails.Get("name").String(), "log bucket name should match") + assert.Equal(int64(RETENTION_DAYS), sameProjLogBucketDetails.Get("retentionDays").Int(), "retention days should match") + + sameProjLogSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", sameProjLogSinkName, sameProjLogSinkProjectId)) + + // assert log sink name, destination & filter + assert.Equal(sameProjLogSinkDestination, sameProjLogSinkDetails.Get("destination").String(), "log sink destination should match") + assert.Equal("resource.type = gce_instance", sameProjLogSinkDetails.Get("filter").String(), "log sink filter should match") + assert.Equal(SAMEPROJ_LOGSINK_WRITER_IDENTITY, sameProjLogSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should not be empty") + + + //*************************** + // Test SAs and Permissions * + //*************************** + projPermissionsDetails := gcloud.Runf(t, fmt.Sprintf("projects get-iam-policy %s", logBucketDestinationProjId)) + listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\""+ ROLE_BUCKET_WRITER +"\").members").Array()) + + // assert sink writer identity service account permission + assert.Contains(listMembers, logSinkWriterIdentity, "log sink writer identity permission should match") + assert.Equal(int(1), len(listMembers), "only one writer identity should have logbucket write permission") + }) + insSimpleT.Test() +} From e7351215f04858d0a60f2e1ca4b304a83b73cabf Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Fri, 29 Jul 2022 20:59:00 -0300 Subject: [PATCH 10/21] Revert mischanged copyright year --- modules/logbucket/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/logbucket/variables.tf b/modules/logbucket/variables.tf index a6380378..5b3e78a4 100644 --- a/modules/logbucket/variables.tf +++ b/modules/logbucket/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From ccf60ea39057944186211081188595866cdda222 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Fri, 29 Jul 2022 21:00:07 -0300 Subject: [PATCH 11/21] Shorten variable names --- examples/logbucket/project/README.md | 12 ++-- examples/logbucket/project/main.tf | 8 +-- examples/logbucket/project/outputs.tf | 24 +++---- .../logbucket_project_test.go | 72 +++++++++---------- 4 files changed, 54 insertions(+), 62 deletions(-) diff --git a/examples/logbucket/project/README.md b/examples/logbucket/project/README.md index db604367..317f3fbb 100644 --- a/examples/logbucket/project/README.md +++ b/examples/logbucket/project/README.md @@ -14,17 +14,17 @@ These examples configures a project-level log sink that feeds a logging log buck | Name | Description | |------|-------------| +| log\_bkt\_name\_same\_proj | The name for the log bucket for sink and logbucket in same project example. | +| log\_bkt\_same\_proj | The project where the log bucket is created for sink and logbucket in same project example. | | log\_bucket\_name | The name for the log bucket. | -| log\_bucket\_name\_same\_project\_example | The name for the log bucket for sink and logbucket in same project example. | | log\_bucket\_project | The project where the log bucket is created. | -| log\_bucket\_project\_same\_project\_example | The project where the log bucket is created for sink and logbucket in same project example. | +| log\_sink\_dest\_uri\_same\_proj | A fully qualified URI for the log sink for sink and logbucket in same project example. | | log\_sink\_destination\_uri | A fully qualified URI for the log sink. | -| log\_sink\_destination\_uri\_same\_project\_example | A fully qualified URI for the log sink for sink and logbucket in same project example. | +| log\_sink\_id\_same\_proj | The project id where the log sink is created for sink and logbucket in same project example. | | log\_sink\_project\_id | The project id where the log sink is created. | -| log\_sink\_project\_id\_same\_project\_example | The project id where the log sink is created for sink and logbucket in same project example. | | log\_sink\_resource\_name | The resource name of the log sink that was created. | -| log\_sink\_resource\_name\_same\_project\_example | The resource name of the log sink that was created in same project example. | +| log\_sink\_resource\_name\_same\_proj | The resource name of the log sink that was created in same project example. | | log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. | -| log\_sink\_writer\_identity\_same\_project\_example | The service account in same project example that logging uses to write log entries to the destination. | +| log\_sink\_writer\_identity\_same\_proj | The service account in same project example that logging uses to write log entries to the destination. | diff --git a/examples/logbucket/project/main.tf b/examples/logbucket/project/main.tf index 6c042fa8..5c1d4e9e 100644 --- a/examples/logbucket/project/main.tf +++ b/examples/logbucket/project/main.tf @@ -41,9 +41,9 @@ module "destination" { #-------------------------------------# # Log Bucket and Sink in same project # #-------------------------------------# -module "log_export_same_project_example" { +module "log_export_same_proj" { source = "../../../" - destination_uri = module.destination_same_project_example.destination_uri + destination_uri = module.dest_same_proj.destination_uri filter = "resource.type = gce_instance" log_sink_name = "logbucket_same_project" parent_resource_id = var.project_destination_logbkt_id @@ -51,12 +51,12 @@ module "log_export_same_project_example" { unique_writer_identity = true } -module "destination_same_project_example" { +module "dest_same_proj" { source = "../../..//modules/logbucket" project_id = var.project_destination_logbkt_id name = "logbucket_from_same_project_${random_string.suffix.result}" location = "global" - log_sink_writer_identity = module.log_export_same_project_example.writer_identity + log_sink_writer_identity = module.log_export_same_proj.writer_identity grant_write_permission_on_bkt = false retention_days = 20 } diff --git a/examples/logbucket/project/outputs.tf b/examples/logbucket/project/outputs.tf index f42761d9..50b292be 100644 --- a/examples/logbucket/project/outputs.tf +++ b/examples/logbucket/project/outputs.tf @@ -48,32 +48,32 @@ output "log_sink_writer_identity" { #-------------------------------------# # Log Bucket and Sink in same project # #-------------------------------------# -output "log_bucket_project_same_project_example" { +output "log_bkt_same_proj" { description = "The project where the log bucket is created for sink and logbucket in same project example." - value = module.destination_same_project_example.project + value = module.dest_same_proj.project } -output "log_bucket_name_same_project_example" { +output "log_bkt_name_same_proj" { description = "The name for the log bucket for sink and logbucket in same project example." - value = module.destination_same_project_example.resource_name + value = module.dest_same_proj.resource_name } -output "log_sink_project_id_same_project_example" { +output "log_sink_id_same_proj" { description = "The project id where the log sink is created for sink and logbucket in same project example." - value = module.log_export_same_project_example.parent_resource_id + value = module.log_export_same_proj.parent_resource_id } -output "log_sink_destination_uri_same_project_example" { +output "log_sink_dest_uri_same_proj" { description = "A fully qualified URI for the log sink for sink and logbucket in same project example." - value = module.destination_same_project_example.destination_uri + value = module.dest_same_proj.destination_uri } -output "log_sink_resource_name_same_project_example" { +output "log_sink_resource_name_same_proj" { description = "The resource name of the log sink that was created in same project example." - value = module.log_export_same_project_example.log_sink_resource_name + value = module.log_export_same_proj.log_sink_resource_name } -output "log_sink_writer_identity_same_project_example" { +output "log_sink_writer_identity_same_proj" { description = "The service account in same project example that logging uses to write log entries to the destination." - value = module.log_export_same_project_example.writer_identity + value = module.log_export_same_proj.writer_identity } diff --git a/test/integration/logbucket-project/logbucket_project_test.go b/test/integration/logbucket-project/logbucket_project_test.go index 0efaef2f..1e8cd2b7 100644 --- a/test/integration/logbucket-project/logbucket_project_test.go +++ b/test/integration/logbucket-project/logbucket_project_test.go @@ -28,77 +28,69 @@ func TestLogBucketProjectModule(t *testing.T) { const logApiFdqm = "logging.googleapis.com" const RETENTION_DAYS = 20 - const SAMEPROJ_LOGSINK_WRITER_IDENTITY string = "" const ROLE_BUCKET_WRITER string = "roles/logging.bucketWriter" - insSimpleT := tft.NewTFBlueprintTest(t, + bpt := tft.NewTFBlueprintTest(t, tft.WithTFDir("../../../examples/logbucket/project"), ) - insSimpleT.DefineVerify(func(assert *assert.Assertions) { - insSimpleT.DefaultVerify(assert) + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) //******************************* // Bucket on other project test * //******************************* - projectId := insSimpleT.GetStringOutput("log_bucket_project") - logBucketName := insSimpleT.GetStringOutput("log_bucket_name") - logBucketDestinationProjId := insSimpleT.GetStringOutput("log_bucket_project_same_project_example") - logSinkProjectId := insSimpleT.GetStringOutput("log_sink_project_id") - logSinkDestination := insSimpleT.GetStringOutput("log_sink_destination_uri") - logSinkName := insSimpleT.GetStringOutput("log_sink_resource_name") - logSinkWriterIdentity := insSimpleT.GetStringOutput("log_sink_writer_identity") + projId := bpt.GetStringOutput("log_bucket_project") + bktName := bpt.GetStringOutput("log_bucket_name") + bktDestProjId := bpt.GetStringOutput("log_bkt_same_proj") + sinkProjId := bpt.GetStringOutput("log_sink_project_id") + sinkDest := bpt.GetStringOutput("log_sink_destination_uri") + sinkName := bpt.GetStringOutput("log_sink_resource_name") + sinkWriterIdentity := bpt.GetStringOutput("log_sink_writer_identity") - logBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", logBucketName, "global", projectId)) + logBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", bktName, "global", projId)) // assert log bucket name, retention days & location - assert.Equal(logSinkDestination[len(logApiFdqm)+1:], logBucketDetails.Get("name").String(), "log bucket name should match") + assert.Equal(sinkDest[len(logApiFdqm)+1:], logBucketDetails.Get("name").String(), "log bucket name should match") assert.Equal(int64(30), logBucketDetails.Get("retentionDays").Int(), "retention days should match") - logSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", logSinkName, logSinkProjectId)) + logSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", sinkName, sinkProjId)) // assert log sink name, destination & filter - assert.Equal(logSinkDestination, logSinkDetails.Get("destination").String(), "log sink destination should match") + assert.Equal(sinkDest, logSinkDetails.Get("destination").String(), "log sink destination should match") assert.Equal("resource.type = gce_instance", logSinkDetails.Get("filter").String(), "log sink filter should match") - assert.Equal(logSinkWriterIdentity, logSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should not be empty") - + assert.Equal(sinkWriterIdentity, logSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should not be empty") //********************************** // Bucket on the same project test * //********************************** - sameProjId := insSimpleT.GetStringOutput("log_bucket_project_same_project_example") - sameProjLogBucketName := insSimpleT.GetStringOutput("log_bucket_name_same_project_example") - sameProjLogSinkProjectId := insSimpleT.GetStringOutput("log_sink_project_id_same_project_example") - sameProjLogSinkDestination := insSimpleT.GetStringOutput("log_sink_destination_uri_same_project_example") - sameProjLogSinkName := insSimpleT.GetStringOutput("log_sink_resource_name_same_project_example") - - assert.NotEqual(sameProjId, "a", "log sink destination should match") - assert.NotEqual(sameProjLogBucketName, "", "log sink destination should match") - assert.NotEqual(sameProjLogSinkProjectId, "", "log sink destination should match") - assert.NotEqual(sameProjLogSinkDestination, "", "log sink destination should match") + sameProjId := bpt.GetStringOutput("log_bkt_same_proj") + sameProjBktName := bpt.GetStringOutput("log_bkt_name_same_proj") + sameProjSinkProjId := bpt.GetStringOutput("log_sink_id_same_proj") + sameProjSinkDest := bpt.GetStringOutput("log_sink_dest_uri_same_proj") + sameProjSinkName := bpt.GetStringOutput("log_sink_resource_name_same_proj") - sameProjLogBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", sameProjLogBucketName, "global", sameProjId)) + sameProjBktDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", sameProjBktName, "global", sameProjId)) // assert log bucket name, retention days & location - assert.Equal(sameProjLogSinkDestination[len(logApiFdqm)+1:], sameProjLogBucketDetails.Get("name").String(), "log bucket name should match") - assert.Equal(int64(RETENTION_DAYS), sameProjLogBucketDetails.Get("retentionDays").Int(), "retention days should match") + assert.Equal(sameProjSinkDest[len(logApiFdqm)+1:], sameProjBktDetails.Get("name").String(), "log bucket name should match") + assert.Equal(int64(RETENTION_DAYS), sameProjBktDetails.Get("retentionDays").Int(), "retention days should match") - sameProjLogSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", sameProjLogSinkName, sameProjLogSinkProjectId)) + sameProjSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", sameProjSinkName, sameProjSinkProjId)) // assert log sink name, destination & filter - assert.Equal(sameProjLogSinkDestination, sameProjLogSinkDetails.Get("destination").String(), "log sink destination should match") - assert.Equal("resource.type = gce_instance", sameProjLogSinkDetails.Get("filter").String(), "log sink filter should match") - assert.Equal(SAMEPROJ_LOGSINK_WRITER_IDENTITY, sameProjLogSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should not be empty") - + assert.Equal(sameProjSinkDest, sameProjSinkDetails.Get("destination").String(), "log sink destination should match") + assert.Equal("resource.type = gce_instance", sameProjSinkDetails.Get("filter").String(), "log sink filter should match") + assert.Empty(sameProjSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity same project should be empty") //*************************** // Test SAs and Permissions * //*************************** - projPermissionsDetails := gcloud.Runf(t, fmt.Sprintf("projects get-iam-policy %s", logBucketDestinationProjId)) - listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\""+ ROLE_BUCKET_WRITER +"\").members").Array()) + projPermissionsDetails := gcloud.Runf(t, fmt.Sprintf("projects get-iam-policy %s", bktDestProjId)) + listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\"" + ROLE_BUCKET_WRITER + "\").members").Array()) // assert sink writer identity service account permission - assert.Contains(listMembers, logSinkWriterIdentity, "log sink writer identity permission should match") - assert.Equal(int(1), len(listMembers), "only one writer identity should have logbucket write permission") + assert.Contains(listMembers, sinkWriterIdentity, "log sink writer identity permission should match") + assert.Len(listMembers, 1, "only one writer identity should have logbucket write permission") }) - insSimpleT.Test() + bpt.Test() } From 9f0126f5566245f8b5b550818ffcbff9d5aa0245 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio <108549791+felipecrescencio-cit@users.noreply.github.com> Date: Tue, 2 Aug 2022 09:11:08 -0300 Subject: [PATCH 12/21] Fix logbucket/project example README.md Co-authored-by: Daniel Andrade --- examples/logbucket/project/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/logbucket/project/README.md b/examples/logbucket/project/README.md index 317f3fbb..6d6ff9de 100644 --- a/examples/logbucket/project/README.md +++ b/examples/logbucket/project/README.md @@ -1,4 +1,4 @@ -# Log Export: Log Bucket destination at Folder level +# Log Export: Log Bucket destination at Project level These examples configures a project-level log sink that feeds a logging log bucket destination with log bucket and log sink in the same project and in separated projects. From 467524753711511036329c27d5609bf6eb271ce2 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio <108549791+felipecrescencio-cit@users.noreply.github.com> Date: Tue, 2 Aug 2022 09:11:54 -0300 Subject: [PATCH 13/21] Fix logbucket/project example README.md Co-authored-by: Daniel Andrade --- examples/logbucket/project/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/logbucket/project/README.md b/examples/logbucket/project/README.md index 6d6ff9de..e6b569df 100644 --- a/examples/logbucket/project/README.md +++ b/examples/logbucket/project/README.md @@ -1,6 +1,6 @@ # Log Export: Log Bucket destination at Project level -These examples configures a project-level log sink that feeds a logging log bucket destination with log bucket and log sink in the same project and in separated projects. +These examples configures a project-level log sink that feeds a logging log bucket destination with log bucket and log sink in the same project or in separated projects. ## Inputs From 46e6e4132133662b717fd02a490460133632adaf Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Tue, 2 Aug 2022 09:14:00 -0300 Subject: [PATCH 14/21] Revert mischanged copyright year --- modules/logbucket/outputs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/logbucket/outputs.tf b/modules/logbucket/outputs.tf index 410cb900..5b48d2eb 100644 --- a/modules/logbucket/outputs.tf +++ b/modules/logbucket/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2019 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 84e7940ebd5b6594bfe3a6a8234e407f7169223a Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Tue, 2 Aug 2022 10:36:52 -0300 Subject: [PATCH 15/21] Inline constants --- .../logbucket-project/logbucket_project_test.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/integration/logbucket-project/logbucket_project_test.go b/test/integration/logbucket-project/logbucket_project_test.go index 1e8cd2b7..7ce1cc52 100644 --- a/test/integration/logbucket-project/logbucket_project_test.go +++ b/test/integration/logbucket-project/logbucket_project_test.go @@ -26,9 +26,8 @@ import ( func TestLogBucketProjectModule(t *testing.T) { - const logApiFdqm = "logging.googleapis.com" - const RETENTION_DAYS = 20 - const ROLE_BUCKET_WRITER string = "roles/logging.bucketWriter" + const logApiFdqm, roleBucketWriter string = "logging.googleapis.com", "roles/logging.bucketWriter" + const retentionDays int64 = 20 bpt := tft.NewTFBlueprintTest(t, tft.WithTFDir("../../../examples/logbucket/project"), @@ -73,7 +72,7 @@ func TestLogBucketProjectModule(t *testing.T) { // assert log bucket name, retention days & location assert.Equal(sameProjSinkDest[len(logApiFdqm)+1:], sameProjBktDetails.Get("name").String(), "log bucket name should match") - assert.Equal(int64(RETENTION_DAYS), sameProjBktDetails.Get("retentionDays").Int(), "retention days should match") + assert.Equal(int64(retentionDays), sameProjBktDetails.Get("retentionDays").Int(), "retention days should match") sameProjSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", sameProjSinkName, sameProjSinkProjId)) @@ -86,7 +85,7 @@ func TestLogBucketProjectModule(t *testing.T) { // Test SAs and Permissions * //*************************** projPermissionsDetails := gcloud.Runf(t, fmt.Sprintf("projects get-iam-policy %s", bktDestProjId)) - listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\"" + ROLE_BUCKET_WRITER + "\").members").Array()) + listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\"" + roleBucketWriter + "\").members").Array()) // assert sink writer identity service account permission assert.Contains(listMembers, sinkWriterIdentity, "log sink writer identity permission should match") From 6114dc2c9b8ee951a2a60317b75641a5fd3962f8 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Tue, 2 Aug 2022 10:42:45 -0300 Subject: [PATCH 16/21] Changed variables creation place to right before usage --- .../logbucket-project/logbucket_project_test.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test/integration/logbucket-project/logbucket_project_test.go b/test/integration/logbucket-project/logbucket_project_test.go index 7ce1cc52..21d01b40 100644 --- a/test/integration/logbucket-project/logbucket_project_test.go +++ b/test/integration/logbucket-project/logbucket_project_test.go @@ -40,11 +40,7 @@ func TestLogBucketProjectModule(t *testing.T) { //******************************* projId := bpt.GetStringOutput("log_bucket_project") bktName := bpt.GetStringOutput("log_bucket_name") - bktDestProjId := bpt.GetStringOutput("log_bkt_same_proj") - sinkProjId := bpt.GetStringOutput("log_sink_project_id") sinkDest := bpt.GetStringOutput("log_sink_destination_uri") - sinkName := bpt.GetStringOutput("log_sink_resource_name") - sinkWriterIdentity := bpt.GetStringOutput("log_sink_writer_identity") logBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", bktName, "global", projId)) @@ -52,6 +48,10 @@ func TestLogBucketProjectModule(t *testing.T) { assert.Equal(sinkDest[len(logApiFdqm)+1:], logBucketDetails.Get("name").String(), "log bucket name should match") assert.Equal(int64(30), logBucketDetails.Get("retentionDays").Int(), "retention days should match") + sinkProjId := bpt.GetStringOutput("log_sink_project_id") + sinkName := bpt.GetStringOutput("log_sink_resource_name") + sinkWriterIdentity := bpt.GetStringOutput("log_sink_writer_identity") + logSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", sinkName, sinkProjId)) // assert log sink name, destination & filter @@ -64,15 +64,16 @@ func TestLogBucketProjectModule(t *testing.T) { //********************************** sameProjId := bpt.GetStringOutput("log_bkt_same_proj") sameProjBktName := bpt.GetStringOutput("log_bkt_name_same_proj") - sameProjSinkProjId := bpt.GetStringOutput("log_sink_id_same_proj") sameProjSinkDest := bpt.GetStringOutput("log_sink_dest_uri_same_proj") - sameProjSinkName := bpt.GetStringOutput("log_sink_resource_name_same_proj") sameProjBktDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", sameProjBktName, "global", sameProjId)) // assert log bucket name, retention days & location assert.Equal(sameProjSinkDest[len(logApiFdqm)+1:], sameProjBktDetails.Get("name").String(), "log bucket name should match") - assert.Equal(int64(retentionDays), sameProjBktDetails.Get("retentionDays").Int(), "retention days should match") + assert.Equal(retentionDays, sameProjBktDetails.Get("retentionDays").Int(), "retention days should match") + + sameProjSinkProjId := bpt.GetStringOutput("log_sink_id_same_proj") + sameProjSinkName := bpt.GetStringOutput("log_sink_resource_name_same_proj") sameProjSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", sameProjSinkName, sameProjSinkProjId)) @@ -84,6 +85,8 @@ func TestLogBucketProjectModule(t *testing.T) { //*************************** // Test SAs and Permissions * //*************************** + bktDestProjId := bpt.GetStringOutput("log_bkt_same_proj") + projPermissionsDetails := gcloud.Runf(t, fmt.Sprintf("projects get-iam-policy %s", bktDestProjId)) listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\"" + roleBucketWriter + "\").members").Array()) From 1b10674bf70c91a4c4b5a6e760f3c47ff7d4e51f Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Tue, 2 Aug 2022 16:00:55 -0300 Subject: [PATCH 17/21] Remove retention_days to keep it default user better understanding example --- examples/logbucket/project/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/logbucket/project/main.tf b/examples/logbucket/project/main.tf index 5c1d4e9e..e936df31 100644 --- a/examples/logbucket/project/main.tf +++ b/examples/logbucket/project/main.tf @@ -58,5 +58,4 @@ module "dest_same_proj" { location = "global" log_sink_writer_identity = module.log_export_same_proj.writer_identity grant_write_permission_on_bkt = false - retention_days = 20 } From 8693453cc932c4fa1d19867f78b1471fb8078c07 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Tue, 2 Aug 2022 16:22:41 -0300 Subject: [PATCH 18/21] -Created constant defaultRetentionDays and kept both examples with this default for user better understanding code and example -Changed assert bucket name to remove substring and compare to a built string fully qualified name -Combined similar asserts to tabular --- .../logbucket_project_test.go | 106 +++++++++--------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/test/integration/logbucket-project/logbucket_project_test.go b/test/integration/logbucket-project/logbucket_project_test.go index 21d01b40..c4993f4a 100644 --- a/test/integration/logbucket-project/logbucket_project_test.go +++ b/test/integration/logbucket-project/logbucket_project_test.go @@ -24,10 +24,11 @@ import ( "github.com/stretchr/testify/assert" ) -func TestLogBucketProjectModule(t *testing.T) { +const ( + defaultRetentionDays int64 = 30 +) - const logApiFdqm, roleBucketWriter string = "logging.googleapis.com", "roles/logging.bucketWriter" - const retentionDays int64 = 20 +func TestLogBucketProjectModule(t *testing.T) { bpt := tft.NewTFBlueprintTest(t, tft.WithTFDir("../../../examples/logbucket/project"), @@ -35,60 +36,57 @@ func TestLogBucketProjectModule(t *testing.T) { bpt.DefineVerify(func(assert *assert.Assertions) { bpt.DefaultVerify(assert) - //******************************* - // Bucket on other project test * - //******************************* - projId := bpt.GetStringOutput("log_bucket_project") - bktName := bpt.GetStringOutput("log_bucket_name") - sinkDest := bpt.GetStringOutput("log_sink_destination_uri") - - logBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", bktName, "global", projId)) - - // assert log bucket name, retention days & location - assert.Equal(sinkDest[len(logApiFdqm)+1:], logBucketDetails.Get("name").String(), "log bucket name should match") - assert.Equal(int64(30), logBucketDetails.Get("retentionDays").Int(), "retention days should match") - - sinkProjId := bpt.GetStringOutput("log_sink_project_id") - sinkName := bpt.GetStringOutput("log_sink_resource_name") - sinkWriterIdentity := bpt.GetStringOutput("log_sink_writer_identity") - - logSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", sinkName, sinkProjId)) - - // assert log sink name, destination & filter - assert.Equal(sinkDest, logSinkDetails.Get("destination").String(), "log sink destination should match") - assert.Equal("resource.type = gce_instance", logSinkDetails.Get("filter").String(), "log sink filter should match") - assert.Equal(sinkWriterIdentity, logSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should not be empty") - - //********************************** - // Bucket on the same project test * - //********************************** - sameProjId := bpt.GetStringOutput("log_bkt_same_proj") - sameProjBktName := bpt.GetStringOutput("log_bkt_name_same_proj") - sameProjSinkDest := bpt.GetStringOutput("log_sink_dest_uri_same_proj") - - sameProjBktDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", sameProjBktName, "global", sameProjId)) - - // assert log bucket name, retention days & location - assert.Equal(sameProjSinkDest[len(logApiFdqm)+1:], sameProjBktDetails.Get("name").String(), "log bucket name should match") - assert.Equal(retentionDays, sameProjBktDetails.Get("retentionDays").Int(), "retention days should match") - - sameProjSinkProjId := bpt.GetStringOutput("log_sink_id_same_proj") - sameProjSinkName := bpt.GetStringOutput("log_sink_resource_name_same_proj") - - sameProjSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", sameProjSinkName, sameProjSinkProjId)) - - // assert log sink name, destination & filter - assert.Equal(sameProjSinkDest, sameProjSinkDetails.Get("destination").String(), "log sink destination should match") - assert.Equal("resource.type = gce_instance", sameProjSinkDetails.Get("filter").String(), "log sink filter should match") - assert.Empty(sameProjSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity same project should be empty") - - //*************************** - // Test SAs and Permissions * - //*************************** + for _, tc := range []struct { + projId string + bktName string + sinkDest string + sinkProjId string + sinkName string + sinkWriterIdentity string + }{ + { + projId: bpt.GetStringOutput("log_bucket_project"), + bktName: bpt.GetStringOutput("log_bucket_name"), + sinkDest: bpt.GetStringOutput("log_sink_destination_uri"), + sinkProjId: bpt.GetStringOutput("log_sink_project_id"), + sinkName: bpt.GetStringOutput("log_sink_resource_name"), + sinkWriterIdentity: bpt.GetStringOutput("log_sink_writer_identity"), + }, + { + projId: bpt.GetStringOutput("log_bkt_same_proj"), + bktName: bpt.GetStringOutput("log_bkt_name_same_proj"), + sinkDest: bpt.GetStringOutput("log_sink_dest_uri_same_proj"), + sinkProjId: bpt.GetStringOutput("log_sink_id_same_proj"), + sinkName: bpt.GetStringOutput("log_sink_resource_name_same_proj"), + sinkWriterIdentity: "", + }, + } { + //************************ + // Assert bucket details * + //************************ + bktFullName := fmt.Sprintf("projects/%s/locations/%s/buckets/%s", tc.projId, "global", tc.bktName) + logBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", tc.bktName, "global", tc.projId)) + + // assert log bucket name, retention days & location + assert.Equal(bktFullName, logBucketDetails.Get("name").String(), "log bucket name should match") + assert.Equal(defaultRetentionDays, logBucketDetails.Get("retentionDays").Int(), "retention days should match") + + logSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", tc.sinkName, tc.sinkProjId)) + + // assert log sink name, destination & filter + assert.Equal(tc.sinkDest, logSinkDetails.Get("destination").String(), "log sink destination should match") + assert.Equal("resource.type = gce_instance", logSinkDetails.Get("filter").String(), "log sink filter should match") + assert.Equal(tc.sinkWriterIdentity, logSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should not be empty") + } + + //***************************** + // Assert SAs and Permissions * + //***************************** bktDestProjId := bpt.GetStringOutput("log_bkt_same_proj") + sinkWriterIdentity := bpt.GetStringOutput("log_sink_writer_identity") projPermissionsDetails := gcloud.Runf(t, fmt.Sprintf("projects get-iam-policy %s", bktDestProjId)) - listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\"" + roleBucketWriter + "\").members").Array()) + listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\"roles/logging.bucketWriter\").members").Array()) // assert sink writer identity service account permission assert.Contains(listMembers, sinkWriterIdentity, "log sink writer identity permission should match") From 3fcace824cffeda1f0df2873aa076d6a88a91c27 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio <108549791+felipecrescencio-cit@users.noreply.github.com> Date: Tue, 2 Aug 2022 17:01:04 -0300 Subject: [PATCH 19/21] Removed because empty is string default value Co-authored-by: Daniel Andrade --- test/integration/logbucket-project/logbucket_project_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/logbucket-project/logbucket_project_test.go b/test/integration/logbucket-project/logbucket_project_test.go index c4993f4a..5e98c9ff 100644 --- a/test/integration/logbucket-project/logbucket_project_test.go +++ b/test/integration/logbucket-project/logbucket_project_test.go @@ -58,7 +58,6 @@ func TestLogBucketProjectModule(t *testing.T) { sinkDest: bpt.GetStringOutput("log_sink_dest_uri_same_proj"), sinkProjId: bpt.GetStringOutput("log_sink_id_same_proj"), sinkName: bpt.GetStringOutput("log_sink_resource_name_same_proj"), - sinkWriterIdentity: "", }, } { //************************ From 997681accd644b57549f8ecf4107f1affc8c1404 Mon Sep 17 00:00:00 2001 From: Felipe Crescencio <108549791+felipecrescencio-cit@users.noreply.github.com> Date: Tue, 2 Aug 2022 17:01:40 -0300 Subject: [PATCH 20/21] Removed sink prefix because there is only one writerIdentity Co-authored-by: Daniel Andrade --- test/integration/logbucket-project/logbucket_project_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/logbucket-project/logbucket_project_test.go b/test/integration/logbucket-project/logbucket_project_test.go index 5e98c9ff..ce4c42bf 100644 --- a/test/integration/logbucket-project/logbucket_project_test.go +++ b/test/integration/logbucket-project/logbucket_project_test.go @@ -42,7 +42,7 @@ func TestLogBucketProjectModule(t *testing.T) { sinkDest string sinkProjId string sinkName string - sinkWriterIdentity string + writerIdentity string }{ { projId: bpt.GetStringOutput("log_bucket_project"), From ae537034f070a049030580276ea67f5b15eb59ec Mon Sep 17 00:00:00 2001 From: Felipe Crescencio Date: Tue, 2 Aug 2022 17:18:55 -0300 Subject: [PATCH 21/21] Fix assert writerIdentity message for examples --- .../logbucket_project_test.go | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/test/integration/logbucket-project/logbucket_project_test.go b/test/integration/logbucket-project/logbucket_project_test.go index ce4c42bf..9247c717 100644 --- a/test/integration/logbucket-project/logbucket_project_test.go +++ b/test/integration/logbucket-project/logbucket_project_test.go @@ -37,27 +37,28 @@ func TestLogBucketProjectModule(t *testing.T) { bpt.DefaultVerify(assert) for _, tc := range []struct { - projId string - bktName string - sinkDest string - sinkProjId string - sinkName string + projId string + bktName string + sinkDest string + sinkProjId string + sinkName string writerIdentity string }{ { - projId: bpt.GetStringOutput("log_bucket_project"), - bktName: bpt.GetStringOutput("log_bucket_name"), - sinkDest: bpt.GetStringOutput("log_sink_destination_uri"), - sinkProjId: bpt.GetStringOutput("log_sink_project_id"), - sinkName: bpt.GetStringOutput("log_sink_resource_name"), - sinkWriterIdentity: bpt.GetStringOutput("log_sink_writer_identity"), + projId: bpt.GetStringOutput("log_bucket_project"), + bktName: bpt.GetStringOutput("log_bucket_name"), + sinkDest: bpt.GetStringOutput("log_sink_destination_uri"), + sinkProjId: bpt.GetStringOutput("log_sink_project_id"), + sinkName: bpt.GetStringOutput("log_sink_resource_name"), + writerIdentity: bpt.GetStringOutput("log_sink_writer_identity"), }, { - projId: bpt.GetStringOutput("log_bkt_same_proj"), - bktName: bpt.GetStringOutput("log_bkt_name_same_proj"), - sinkDest: bpt.GetStringOutput("log_sink_dest_uri_same_proj"), - sinkProjId: bpt.GetStringOutput("log_sink_id_same_proj"), - sinkName: bpt.GetStringOutput("log_sink_resource_name_same_proj"), + projId: bpt.GetStringOutput("log_bkt_same_proj"), + bktName: bpt.GetStringOutput("log_bkt_name_same_proj"), + sinkDest: bpt.GetStringOutput("log_sink_dest_uri_same_proj"), + sinkProjId: bpt.GetStringOutput("log_sink_id_same_proj"), + sinkName: bpt.GetStringOutput("log_sink_resource_name_same_proj"), + // writerIdentity: As sink and bucket are in same project no service account is needed and writerIdentity is empty }, } { //************************ @@ -75,7 +76,7 @@ func TestLogBucketProjectModule(t *testing.T) { // assert log sink name, destination & filter assert.Equal(tc.sinkDest, logSinkDetails.Get("destination").String(), "log sink destination should match") assert.Equal("resource.type = gce_instance", logSinkDetails.Get("filter").String(), "log sink filter should match") - assert.Equal(tc.sinkWriterIdentity, logSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should not be empty") + assert.Equal(tc.writerIdentity, logSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should match") } //*****************************