From 90bc8b2cee6f609831a18eade2d820cbb741647b Mon Sep 17 00:00:00 2001 From: Sumeet Date: Fri, 25 Jun 2021 00:33:51 +0530 Subject: [PATCH 1/6] added labels for gcs destination --- modules/storage/README.md | 1 + modules/storage/main.tf | 1 + modules/storage/variables.tf | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/modules/storage/README.md b/modules/storage/README.md index 2516fa68..d1ae62ad 100644 --- a/modules/storage/README.md +++ b/modules/storage/README.md @@ -42,6 +42,7 @@ so that all dependencies are met. | log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. (This is available as an output coming from the root module). | `string` | n/a | yes | | project\_id | The ID of the project in which the storage bucket will be created. | `string` | n/a | yes | | retention\_policy | Configuration of the bucket's data retention policy for how long objects in the bucket should be retained. | 
object({
    is_locked             = bool
    retention_period_days = number
  })
| `null` | no | +| storage\_bucket\_labels | Labels to apply to the storage bucket. | `map(string)` | `{}` | no | | storage\_bucket\_name | The name of the storage bucket to be created and used for log entries matching the filter. | `string` | n/a | yes | | storage\_class | The storage class of the storage bucket. | `string` | `"STANDARD"` | no | | uniform\_bucket\_level\_access | Enables Uniform bucket-level access to a bucket. | `bool` | `true` | no | diff --git a/modules/storage/main.tf b/modules/storage/main.tf index e3ca1f31..cbcaa7bb 100644 --- a/modules/storage/main.tf +++ b/modules/storage/main.tf @@ -38,6 +38,7 @@ resource "google_storage_bucket" "bucket" { location = var.location force_destroy = var.force_destroy uniform_bucket_level_access = var.uniform_bucket_level_access + labels = var.storage_bucket_labels versioning { enabled = var.versioning diff --git a/modules/storage/variables.tf b/modules/storage/variables.tf index cdf806bb..c6ae2b53 100644 --- a/modules/storage/variables.tf +++ b/modules/storage/variables.tf @@ -41,6 +41,12 @@ variable "storage_class" { default = "STANDARD" } +variable "storage_bucket_labels" { + description = "Labels to apply to the storage bucket." + type = map(string) + default = {} +} + variable "uniform_bucket_level_access" { description = "Enables Uniform bucket-level access to a bucket." type = bool From 04a5c412ad6b75014f6201d9adf6368b21e95965 Mon Sep 17 00:00:00 2001 From: Sumeet Date: Fri, 25 Jun 2021 00:44:01 +0530 Subject: [PATCH 2/6] formatted --- modules/storage/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/storage/main.tf b/modules/storage/main.tf index cbcaa7bb..8c6cd5f7 100644 --- a/modules/storage/main.tf +++ b/modules/storage/main.tf @@ -38,7 +38,7 @@ resource "google_storage_bucket" "bucket" { location = var.location force_destroy = var.force_destroy uniform_bucket_level_access = var.uniform_bucket_level_access - labels = var.storage_bucket_labels + labels = var.storage_bucket_labels versioning { enabled = var.versioning From 1e5be2a289570fc618f6eab431a455c2370162a8 Mon Sep 17 00:00:00 2001 From: Sumeet Date: Fri, 25 Jun 2021 01:57:12 +0530 Subject: [PATCH 3/6] 1. Added lifecycle rules instead of just expiration days in GCS destination, updated relevant examples. 2. Added opinion based service account creation for subscriber, not always a service account would be required --- examples/storage/project/main.tf | 13 +++++++++++-- modules/pubsub/README.md | 1 + modules/pubsub/main.tf | 6 +++--- modules/pubsub/variables.tf | 6 ++++++ modules/storage/README.md | 23 ++++++++++++++++++++++- modules/storage/main.tf | 13 +++++++++---- modules/storage/variables.tf | 22 ++++++++++++++++++---- 7 files changed, 70 insertions(+), 14 deletions(-) diff --git a/examples/storage/project/main.tf b/examples/storage/project/main.tf index b1a2456b..e8d206c2 100644 --- a/examples/storage/project/main.tf +++ b/examples/storage/project/main.tf @@ -34,6 +34,15 @@ module "destination" { source = "../../..//modules/storage" project_id = var.project_id storage_bucket_name = "storage_project_${random_string.suffix.result}" - expiration_days = 365 log_sink_writer_identity = module.log_export.writer_identity -} + + lifecycle_rules = [{ + action = { + type = "Delete" + } + condition = { + age = 365 + with_state = "ANY" + } + }] +} \ No newline at end of file diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index df16d08c..495a2c30 100755 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -39,6 +39,7 @@ so that all dependencies are met. |------|-------------|------|---------|:--------:| | create\_push\_subscriber | Whether to add a push configuration to the subcription. If 'true', a push subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic. | `bool` | `false` | no | | create\_subscriber | Whether to create a subscription to the topic that was created and used for log entries matching the filter. If 'true', a pull subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic. | `bool` | `false` | no | +| create\_subscriber\_sa | Whether to create a Service Account for the subscription. If 'true', a service account is created and granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic. | `bool` | `false` | no | | log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. (This is available as an output coming from the root module). | `string` | n/a | yes | | project\_id | The ID of the project in which the pubsub topic will be created. | `string` | n/a | yes | | push\_endpoint | The URL locating the endpoint to which messages should be pushed. | `string` | `""` | no | diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf index 89215d98..cc13f5bb 100755 --- a/modules/pubsub/main.tf +++ b/modules/pubsub/main.tf @@ -67,14 +67,14 @@ resource "google_pubsub_topic_iam_member" "pubsub_sink_member" { # Pub/Sub topic subscription (for integrations) # #-----------------------------------------------# resource "google_service_account" "pubsub_subscriber" { - count = var.create_subscriber ? 1 : 0 + count = var.create_subscriber_sa ? 1 : 0 account_id = local.subscriber_id display_name = "${local.topic_name} Topic Subscriber" project = var.project_id } resource "google_pubsub_subscription_iam_member" "pubsub_subscriber_role" { - count = var.create_subscriber ? 1 : 0 + count = var.create_subscriber_sa ? 1 : 0 role = "roles/pubsub.subscriber" project = var.project_id subscription = local.pubsub_subscription @@ -82,7 +82,7 @@ resource "google_pubsub_subscription_iam_member" "pubsub_subscriber_role" { } resource "google_pubsub_topic_iam_member" "pubsub_viewer_role" { - count = var.create_subscriber ? 1 : 0 + count = var.create_subscriber_sa ? 1 : 0 role = "roles/pubsub.viewer" project = var.project_id topic = local.topic_name diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf index 86c5e10a..bd3d22c6 100755 --- a/modules/pubsub/variables.tf +++ b/modules/pubsub/variables.tf @@ -20,6 +20,12 @@ variable "create_subscriber" { default = false } +variable "create_subscriber_sa" { + description = "Whether to create a Service Account for the subscription. If 'true', a service account is created and granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic." + type = bool + default = false +} + variable "subscriber_id" { description = "The ID to give the pubsub pull subscriber service account (optional)." type = string diff --git a/modules/storage/README.md b/modules/storage/README.md index d1ae62ad..aa4e489a 100644 --- a/modules/storage/README.md +++ b/modules/storage/README.md @@ -24,6 +24,27 @@ module "destination" { project_id = "sample-project" storage_bucket_name = "sample_storage_bucket" log_sink_writer_identity = "${module.log_export.writer_identity}" + lifecycle_rules = [ + { + action = { + type = "Delete" + } + condition = { + age = 365 + with_state = "ANY" + } + }, + { + action = { + type = "SetStorageClass" + storage_class = "COLDLINE" + } + condition = { + age = 180 + with_state = "ANY" + } + } + ] } ``` @@ -36,8 +57,8 @@ so that all dependencies are met. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| expiration\_days | Object expiration time. If unset logs will never be deleted. | `number` | `null` | no | | force\_destroy | When deleting a bucket, this boolean option will delete all contained objects. | `bool` | `false` | no | +| lifecycle\_rules | List of lifecycle rules to configure. Format is the same as described in provider documentation https://www.terraform.io/docs/providers/google/r/storage_bucket.html#lifecycle_rule except condition.matches\_storage\_class should be a comma delimited string. | 
set(object({
    # Object with keys:
    # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass.
    # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule.
    action = map(string)

    # Object with keys:
    # - age - (Optional) Minimum age of an object in days to satisfy this condition.
    # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition.
    # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY".
    # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, STANDARD, DURABLE_REDUCED_AVAILABILITY.
    # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition.
    # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true.
    condition = map(string)
  }))
| `[]` | no | | location | The location of the storage bucket. | `string` | `"US"` | no | | log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. (This is available as an output coming from the root module). | `string` | n/a | yes | | project\_id | The ID of the project in which the storage bucket will be created. | `string` | n/a | yes | diff --git a/modules/storage/main.tf b/modules/storage/main.tf index 8c6cd5f7..f7c4088e 100644 --- a/modules/storage/main.tf +++ b/modules/storage/main.tf @@ -45,14 +45,19 @@ resource "google_storage_bucket" "bucket" { } dynamic "lifecycle_rule" { - for_each = var.expiration_days == null ? [] : [var.expiration_days] + for_each = var.lifecycle_rules content { action { - type = "Delete" + type = lifecycle_rule.value.action.type + storage_class = lookup(lifecycle_rule.value.action, "storage_class", null) } condition { - age = var.expiration_days - with_state = "ANY" + age = lookup(lifecycle_rule.value.condition, "age", null) + created_before = lookup(lifecycle_rule.value.condition, "created_before", null) + with_state = lookup(lifecycle_rule.value.condition, "with_state", lookup(lifecycle_rule.value.condition, "is_live", false) ? "LIVE" : null) + matches_storage_class = contains(keys(lifecycle_rule.value.condition), "matches_storage_class") ? split(",", lifecycle_rule.value.condition["matches_storage_class"]) : null + num_newer_versions = lookup(lifecycle_rule.value.condition, "num_newer_versions", null) + days_since_custom_time = lookup(lifecycle_rule.value.condition, "days_since_custom_time", null) } } } diff --git a/modules/storage/variables.tf b/modules/storage/variables.tf index c6ae2b53..dc26ba76 100644 --- a/modules/storage/variables.tf +++ b/modules/storage/variables.tf @@ -53,10 +53,24 @@ variable "uniform_bucket_level_access" { default = true } -variable "expiration_days" { - description = "Object expiration time. If unset logs will never be deleted." - type = number - default = null +variable "lifecycle_rules" { + type = set(object({ + # Object with keys: + # - type - The type of the action of this Lifecycle Rule. Supported values: Delete and SetStorageClass. + # - storage_class - (Required if action type is SetStorageClass) The target Storage Class of objects affected by this Lifecycle Rule. + action = map(string) + + # Object with keys: + # - age - (Optional) Minimum age of an object in days to satisfy this condition. + # - created_before - (Optional) Creation date of an object in RFC 3339 (e.g. 2017-06-13) to satisfy this condition. + # - with_state - (Optional) Match to live and/or archived objects. Supported values include: "LIVE", "ARCHIVED", "ANY". + # - matches_storage_class - (Optional) Comma delimited string for storage class of objects to satisfy this condition. Supported values include: MULTI_REGIONAL, REGIONAL, NEARLINE, COLDLINE, STANDARD, DURABLE_REDUCED_AVAILABILITY. + # - num_newer_versions - (Optional) Relevant only for versioned objects. The number of newer versions of an object to satisfy this condition. + # - days_since_custom_time - (Optional) The number of days from the Custom-Time metadata attribute after which this condition becomes true. + condition = map(string) + })) + description = "List of lifecycle rules to configure. Format is the same as described in provider documentation https://www.terraform.io/docs/providers/google/r/storage_bucket.html#lifecycle_rule except condition.matches_storage_class should be a comma delimited string." + default = [] } variable "force_destroy" { From 6f3500cabe27b7e160d71059fea570980fcf0936 Mon Sep 17 00:00:00 2001 From: Sumeet Date: Fri, 25 Jun 2021 02:37:30 +0530 Subject: [PATCH 4/6] added new line --- examples/storage/project/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/storage/project/main.tf b/examples/storage/project/main.tf index e8d206c2..00312cda 100644 --- a/examples/storage/project/main.tf +++ b/examples/storage/project/main.tf @@ -45,4 +45,5 @@ module "destination" { with_state = "ANY" } }] -} \ No newline at end of file +} + From 95b36ffa55c42c36b8235d4db2e99517c5744463 Mon Sep 17 00:00:00 2001 From: Sumeet Date: Mon, 28 Jun 2021 22:16:56 +0530 Subject: [PATCH 5/6] added upgrade document --- docs/upgrading_to_v6.0.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 docs/upgrading_to_v6.0.md diff --git a/docs/upgrading_to_v6.0.md b/docs/upgrading_to_v6.0.md new file mode 100644 index 00000000..3a883cd6 --- /dev/null +++ b/docs/upgrading_to_v6.0.md @@ -0,0 +1,30 @@ +# Upgrading to Log Export v6.0 + +The v6.0 release of Log Export is a backwards incompatible release and features few additional features for log retenions and bucket policy lifecycles. + +Breaking changes have only been made to the storage module. +Other modules can safely update the version without needing any changes. + +## Migration Instructions + +NOTE: Users should prefer to let Terraform update their resources to the newer defaults. +To preserve the existing defaults, see below: + +```diff +module "gcs" { + source = "terraform-google-modules/log-export/google//modules/storage" +- version = "v5.0" ++ version = "v6.0" + +- expiration_days = 365 ++ lifecycle_rules = [{ ++ action = { ++ type = "Delete" ++ } ++ condition = { ++ age = 365 ++ with_state = "ANY" ++ } ++ }] +} +``` From b8a72e5625115cfc359fdf286382c7e9090f3529 Mon Sep 17 00:00:00 2001 From: Sumeet Date: Fri, 2 Jul 2021 22:11:17 +0530 Subject: [PATCH 6/6] reverted pubsub subscription service account flag modifications --- modules/pubsub/README.md | 1 - modules/pubsub/main.tf | 6 +++--- modules/pubsub/variables.tf | 6 ------ 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index 495a2c30..df16d08c 100755 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -39,7 +39,6 @@ so that all dependencies are met. |------|-------------|------|---------|:--------:| | create\_push\_subscriber | Whether to add a push configuration to the subcription. If 'true', a push subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic. | `bool` | `false` | no | | create\_subscriber | Whether to create a subscription to the topic that was created and used for log entries matching the filter. If 'true', a pull subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic. | `bool` | `false` | no | -| create\_subscriber\_sa | Whether to create a Service Account for the subscription. If 'true', a service account is created and granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic. | `bool` | `false` | no | | log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. (This is available as an output coming from the root module). | `string` | n/a | yes | | project\_id | The ID of the project in which the pubsub topic will be created. | `string` | n/a | yes | | push\_endpoint | The URL locating the endpoint to which messages should be pushed. | `string` | `""` | no | diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf index cc13f5bb..89215d98 100755 --- a/modules/pubsub/main.tf +++ b/modules/pubsub/main.tf @@ -67,14 +67,14 @@ resource "google_pubsub_topic_iam_member" "pubsub_sink_member" { # Pub/Sub topic subscription (for integrations) # #-----------------------------------------------# resource "google_service_account" "pubsub_subscriber" { - count = var.create_subscriber_sa ? 1 : 0 + count = var.create_subscriber ? 1 : 0 account_id = local.subscriber_id display_name = "${local.topic_name} Topic Subscriber" project = var.project_id } resource "google_pubsub_subscription_iam_member" "pubsub_subscriber_role" { - count = var.create_subscriber_sa ? 1 : 0 + count = var.create_subscriber ? 1 : 0 role = "roles/pubsub.subscriber" project = var.project_id subscription = local.pubsub_subscription @@ -82,7 +82,7 @@ resource "google_pubsub_subscription_iam_member" "pubsub_subscriber_role" { } resource "google_pubsub_topic_iam_member" "pubsub_viewer_role" { - count = var.create_subscriber_sa ? 1 : 0 + count = var.create_subscriber ? 1 : 0 role = "roles/pubsub.viewer" project = var.project_id topic = local.topic_name diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf index bd3d22c6..86c5e10a 100755 --- a/modules/pubsub/variables.tf +++ b/modules/pubsub/variables.tf @@ -20,12 +20,6 @@ variable "create_subscriber" { default = false } -variable "create_subscriber_sa" { - description = "Whether to create a Service Account for the subscription. If 'true', a service account is created and granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic." - type = bool - default = false -} - variable "subscriber_id" { description = "The ID to give the pubsub pull subscriber service account (optional)." type = string