fix:logbucket sink in same project

build/int.cloudbuild.yaml

@@ -69,6 +69,21 @@ steps:
     - go-verify-logbucket-org
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestLogBucketOrgModule --stage teardown --verbose']
+- id: go-apply-logbucket-project
+  waitFor:
+    - init-all
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage apply --verbose']
+- id: go-verify-logbucket-project
+  waitFor:
+    - go-apply-logbucket-org
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage verify --verbose']
+- id: go-teardown-logbucket-project
+  waitFor:
+    - go-verify-logbucket-project
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestLogBucketProjectModule --stage teardown --verbose']
 tags:
 - 'ci'
 - 'integration'

examples/logbucket/project/README.md

@@ -0,0 +1,30 @@
+# Log Export: Log Bucket destination at Project level
+
+These examples configures a project-level log sink that feeds a logging log bucket destination with log bucket and log sink in the same project or in separated projects.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| parent\_resource\_project | The ID of the project in which the log export will be created. | `string` | n/a | yes |
+| project\_destination\_logbkt\_id | The ID of the project in which log bucket destination will be created. | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| log\_bkt\_name\_same\_proj | The name for the log bucket for sink and logbucket in same project example. |
+| log\_bkt\_same\_proj | The project where the log bucket is created for sink and logbucket in same project example. |
+| log\_bucket\_name | The name for the log bucket. |
+| log\_bucket\_project | The project where the log bucket is created. |
+| log\_sink\_dest\_uri\_same\_proj | A fully qualified URI for the log sink for sink and logbucket in same project example. |
+| log\_sink\_destination\_uri | A fully qualified URI for the log sink. |
+| log\_sink\_id\_same\_proj | The project id where the log sink is created for sink and logbucket in same project example. |
+| log\_sink\_project\_id | The project id where the log sink is created. |
+| log\_sink\_resource\_name | The resource name of the log sink that was created. |
+| log\_sink\_resource\_name\_same\_proj | The resource name of the log sink that was created in same project example. |
+| log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. |
+| log\_sink\_writer\_identity\_same\_proj | The service account in same project example that logging uses to write log entries to the destination. |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/logbucket/project/main.tf

@@ -0,0 +1,61 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "random_string" "suffix" {
+  length  = 4
+  upper   = false
+  special = false
+}
+
+module "log_export" {
+  source                 = "../../../"
+  destination_uri        = module.destination.destination_uri
+  filter                 = "resource.type = gce_instance"
+  log_sink_name          = "logbucket_other_project"
+  parent_resource_id     = var.parent_resource_project
+  parent_resource_type   = "project"
+  unique_writer_identity = true
+}
+
+module "destination" {
+  source                   = "../../..//modules/logbucket"
+  project_id               = var.project_destination_logbkt_id
+  name                     = "logbucket_from_other_project_${random_string.suffix.result}"
+  location                 = "global"
+  log_sink_writer_identity = module.log_export.writer_identity
+}
+
+#-------------------------------------#
+# Log Bucket and Sink in same project #
+#-------------------------------------#
+module "log_export_same_proj" {
+  source                 = "../../../"
+  destination_uri        = module.dest_same_proj.destination_uri
+  filter                 = "resource.type = gce_instance"
+  log_sink_name          = "logbucket_same_project"
+  parent_resource_id     = var.project_destination_logbkt_id
+  parent_resource_type   = "project"
+  unique_writer_identity = true
+}
+
+module "dest_same_proj" {
+  source                        = "../../..//modules/logbucket"
+  project_id                    = var.project_destination_logbkt_id
+  name                          = "logbucket_from_same_project_${random_string.suffix.result}"
+  location                      = "global"
+  log_sink_writer_identity      = module.log_export_same_proj.writer_identity
+  grant_write_permission_on_bkt = false
+}

examples/logbucket/project/outputs.tf

@@ -0,0 +1,79 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "log_bucket_project" {
+  description = "The project where the log bucket is created."
+  value       = module.destination.project
+}
+
+output "log_bucket_name" {
+  description = "The name for the log bucket."
+  value       = module.destination.resource_name
+}
+
+output "log_sink_project_id" {
+  description = "The project id where the log sink is created."
+  value       = module.log_export.parent_resource_id
+}
+
+output "log_sink_destination_uri" {
+  description = "A fully qualified URI for the log sink."
+  value       = module.destination.destination_uri
+}
+
+output "log_sink_resource_name" {
+  description = "The resource name of the log sink that was created."
+  value       = module.log_export.log_sink_resource_name
+}
+
+output "log_sink_writer_identity" {
+  description = "The service account that logging uses to write log entries to the destination."
+  value       = module.log_export.writer_identity
+}
+
+
+#-------------------------------------#
+# Log Bucket and Sink in same project #
+#-------------------------------------#
+output "log_bkt_same_proj" {
+  description = "The project where the log bucket is created for sink and logbucket in same project example."
+  value       = module.dest_same_proj.project
+}
+
+output "log_bkt_name_same_proj" {
+  description = "The name for the log bucket for sink and logbucket in same project example."
+  value       = module.dest_same_proj.resource_name
+}
+
+output "log_sink_id_same_proj" {
+  description = "The project id where the log sink is created for sink and logbucket in same project example."
+  value       = module.log_export_same_proj.parent_resource_id
+}
+
+output "log_sink_dest_uri_same_proj" {
+  description = "A fully qualified URI for the log sink for sink and logbucket in same project example."
+  value       = module.dest_same_proj.destination_uri
+}
+
+output "log_sink_resource_name_same_proj" {
+  description = "The resource name of the log sink that was created in same project example."
+  value       = module.log_export_same_proj.log_sink_resource_name
+}
+
+output "log_sink_writer_identity_same_proj" {
+  description = "The service account in same project example that logging uses to write log entries to the destination."
+  value       = module.log_export_same_proj.writer_identity
+}

examples/logbucket/project/variables.tf

@@ -0,0 +1,25 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_destination_logbkt_id" {
+  description = "The ID of the project in which log bucket destination will be created."
+  type        = string
+}
+
+variable "parent_resource_project" {
+  description = "The ID of the project in which the log export will be created."
+  type        = string
+}

examples/logbucket/project/versions.tf

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+terraform {
+  required_version = ">= 0.13"
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.0"
+    }
+    random = {
+      source = "hashicorp/random"
+    }
+  }
+}

modules/logbucket/README.md

@@ -37,6 +37,7 @@ module "destination" {
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| grant\_write\_permission\_on\_bkt | (Optional) Indicates whether the module is responsible for granting write permission on the logbucket. This permission will be given by default, but if the user wants, this module can skip this step. This is the case when the sink route logs to a log bucket in the same Cloud project, no new service account will be created and this module will need to bypass granting permissions. | `bool` | `true` | no |
 | location | The location of the log bucket. | `string` | `"global"` | no |
 | log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. (This is available as an output coming from the root module). | `string` | n/a | yes |
 | name | The name of the log bucket to be created and used for log entries matching the filter. | `string` | n/a | yes |

modules/logbucket/main.tf

@@ -43,6 +43,8 @@ resource "google_logging_project_bucket_config" "bucket" {
 # Service account IAM membership #
 #--------------------------------#
 resource "google_project_iam_member" "logbucket_sink_member" {
+  count = var.grant_write_permission_on_bkt ? 1 : 0
+
   project = google_logging_project_bucket_config.bucket.project
   role    = "roles/logging.bucketWriter"
   member  = var.log_sink_writer_identity

modules/logbucket/variables.tf

@@ -40,3 +40,9 @@ variable "retention_days" {
   type        = number
   default     = 30
 }
+
+variable "grant_write_permission_on_bkt" {
+  description = "(Optional) Indicates whether the module is responsible for granting write permission on the logbucket. This permission will be given by default, but if the user wants, this module can skip this step. This is the case when the sink route logs to a log bucket in the same Cloud project, no new service account will be created and this module will need to bypass granting permissions."
+  type        = bool
+  default     = true
+}

test/integration/logbucket-project/logbucket_project_test.go

@@ -0,0 +1,96 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logbucket_project
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	defaultRetentionDays int64 = 30
+)
+
+func TestLogBucketProjectModule(t *testing.T) {
+
+	bpt := tft.NewTFBlueprintTest(t,
+		tft.WithTFDir("../../../examples/logbucket/project"),
+	)
+	bpt.DefineVerify(func(assert *assert.Assertions) {
+		bpt.DefaultVerify(assert)
+
+		for _, tc := range []struct {
+			projId         string
+			bktName        string
+			sinkDest       string
+			sinkProjId     string
+			sinkName       string
+			writerIdentity string
+		}{
+			{
+				projId:         bpt.GetStringOutput("log_bucket_project"),
+				bktName:        bpt.GetStringOutput("log_bucket_name"),
+				sinkDest:       bpt.GetStringOutput("log_sink_destination_uri"),
+				sinkProjId:     bpt.GetStringOutput("log_sink_project_id"),
+				sinkName:       bpt.GetStringOutput("log_sink_resource_name"),
+				writerIdentity: bpt.GetStringOutput("log_sink_writer_identity"),
+			},
+			{
+				projId:     bpt.GetStringOutput("log_bkt_same_proj"),
+				bktName:    bpt.GetStringOutput("log_bkt_name_same_proj"),
+				sinkDest:   bpt.GetStringOutput("log_sink_dest_uri_same_proj"),
+				sinkProjId: bpt.GetStringOutput("log_sink_id_same_proj"),
+				sinkName:   bpt.GetStringOutput("log_sink_resource_name_same_proj"),
+				// writerIdentity: As sink and bucket are in same project no service account is needed and writerIdentity is empty
+			},
+		} {
+			//************************
+			// Assert bucket details *
+			//************************
+			bktFullName := fmt.Sprintf("projects/%s/locations/%s/buckets/%s", tc.projId, "global", tc.bktName)
+			logBucketDetails := gcloud.Runf(t, fmt.Sprintf("logging buckets describe %s --location=%s --project=%s", tc.bktName, "global", tc.projId))
+
+			// assert log bucket name, retention days & location
+			assert.Equal(bktFullName, logBucketDetails.Get("name").String(), "log bucket name should match")
+			assert.Equal(defaultRetentionDays, logBucketDetails.Get("retentionDays").Int(), "retention days should match")
+
+			logSinkDetails := gcloud.Runf(t, fmt.Sprintf("logging sinks describe %s --project=%s", tc.sinkName, tc.sinkProjId))
+
+			// assert log sink name, destination & filter
+			assert.Equal(tc.sinkDest, logSinkDetails.Get("destination").String(), "log sink destination should match")
+			assert.Equal("resource.type = gce_instance", logSinkDetails.Get("filter").String(), "log sink filter should match")
+			assert.Equal(tc.writerIdentity, logSinkDetails.Get("writerIdentity").String(), "log sink writerIdentity should match")
+		}
+
+		//*****************************
+		// Assert SAs and Permissions *
+		//*****************************
+		bktDestProjId := bpt.GetStringOutput("log_bkt_same_proj")
+		sinkWriterIdentity := bpt.GetStringOutput("log_sink_writer_identity")
+
+		projPermissionsDetails := gcloud.Runf(t, fmt.Sprintf("projects get-iam-policy %s", bktDestProjId))
+		listMembers := utils.GetResultStrSlice(projPermissionsDetails.Get("bindings.#(role==\"roles/logging.bucketWriter\").members").Array())
+
+		// assert sink writer identity service account permission
+		assert.Contains(listMembers, sinkWriterIdentity, "log sink writer identity permission should match")
+		assert.Len(listMembers, 1, "only one writer identity should have logbucket write permission")
+	})
+	bpt.Test()
+}