Merge pull request #22 from anyscale/brent/logging-sink

feat: Add logging sink module

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 default_stages: [commit]
 repos:
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.88.2
+    rev: v1.92.0
     hooks:
       - id: terraform_fmt
       - id: terraform_validate
@@ -22,7 +22,7 @@ repos:
         args:
           - --args=provider google -v "~> 5.0"
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer

.tflint.hcl

@@ -1,12 +1,12 @@
 plugin "google" {
   enabled = true
-  version = "0.27.1"
+  version = "0.30.0"
   source  = "github.com/terraform-linters/tflint-ruleset-google"
 }
 
 config {
-  module     = true
-  force      = false
+  call_module_type = "local"
+  force            = false
 }
 
 rule "terraform_required_providers" {

CHANGELOG.md

@@ -1,3 +1,16 @@
+## 0.13.0 (Released)
+FEATURES:
+- Optional new sub-module to manage the `_Default` Logging Sink - `syslog` logs are automatically enabled for new Projects and this can lead to unexpected storage costs in Projects with many active Anyscale clusters and nodes.
+
+BUG FIXES:
+
+BREAKING CHANGES:
+
+OTHER:
+- tflint updates
+- pre-commit updates
+- trivy rule update on cloudstorage module
+
 ## 0.12.1 (Released)
 FEATURES:
 

README.md

@@ -94,8 +94,8 @@ None
 
 | Name | Version |
 |------|---------|
-| <a name="provider_google"></a> [google](#provider\_google) | 4.84.0 |
-| <a name="provider_random"></a> [random](#provider\_random) | 3.6.0 |
+| <a name="provider_google"></a> [google](#provider\_google) | 5.38.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | 3.6.2 |
 
 ## Modules
 
@@ -105,6 +105,7 @@ None
 | <a name="module_google_anyscale_cloudstorage"></a> [google\_anyscale\_cloudstorage](#module\_google\_anyscale\_cloudstorage) | ./modules/google-anyscale-cloudstorage | n/a |
 | <a name="module_google_anyscale_filestore"></a> [google\_anyscale\_filestore](#module\_google\_anyscale\_filestore) | ./modules/google-anyscale-filestore | n/a |
 | <a name="module_google_anyscale_iam"></a> [google\_anyscale\_iam](#module\_google\_anyscale\_iam) | ./modules/google-anyscale-iam | n/a |
+| <a name="module_google_anyscale_loggingsink"></a> [google\_anyscale\_loggingsink](#module\_google\_anyscale\_loggingsink) | ./modules/google-anyscale-loggingsink | n/a |
 | <a name="module_google_anyscale_memorystore"></a> [google\_anyscale\_memorystore](#module\_google\_anyscale\_memorystore) | ./modules/google-anyscale-memorystore | n/a |
 | <a name="module_google_anyscale_project"></a> [google\_anyscale\_project](#module\_google\_anyscale\_project) | ./modules/google-anyscale-project | n/a |
 | <a name="module_google_anyscale_vpc"></a> [google\_anyscale\_vpc](#module\_google\_anyscale\_vpc) | ./modules/google-anyscale-vpc | n/a |
@@ -187,6 +188,7 @@ None
 | <a name="input_enable_anyscale_filestore"></a> [enable\_anyscale\_filestore](#input\_enable\_anyscale\_filestore) | (Optional) Determines if the Anyscale Filestore is created.<br><br>ex:<pre>enable_anyscale_filestore = true</pre> | `bool` | `true` | no |
 | <a name="input_enable_anyscale_gcs"></a> [enable\_anyscale\_gcs](#input\_enable\_anyscale\_gcs) | (Optional) Determines if the Anyscale Cloud Storage bucket is created.<br><br>ex:<pre>enable_anyscale_gcs = true</pre> | `bool` | `true` | no |
 | <a name="input_enable_anyscale_iam"></a> [enable\_anyscale\_iam](#input\_enable\_anyscale\_iam) | (Optional) Determines if the Anyscale IAM resources are created.<br><br>ex:<pre>enable_anyscale_iam = true</pre> | `bool` | `true` | no |
+| <a name="input_enable_anyscale_loggingsink"></a> [enable\_anyscale\_loggingsink](#input\_enable\_anyscale\_loggingsink) | (Optional) Determines if the Anyscale Logging Sink is executed.<br><br>This sub-module will disable sending syslog events to the `_Default` Log Sink.<br><br>ex:<pre>enable_anyscale_loggingsink = true</pre> | `bool` | `true` | no |
 | <a name="input_enable_anyscale_memorystore"></a> [enable\_anyscale\_memorystore](#input\_enable\_anyscale\_memorystore) | (Optional) Determines if the Anyscale Memorystore is created.<br><br>ex:<pre>enable_anyscale_memorystore = true</pre> | `bool` | `false` | no |
 | <a name="input_enable_anyscale_vpc_firewall"></a> [enable\_anyscale\_vpc\_firewall](#input\_enable\_anyscale\_vpc\_firewall) | (Optional) Determines if the Anyscale VPC Firewall is created.<br><br>ex:<pre>enable_anyscale_vpc_firewall = true</pre> | `bool` | `true` | no |
 | <a name="input_enable_cloud_logging_monitoring"></a> [enable\_cloud\_logging\_monitoring](#input\_enable\_cloud\_logging\_monitoring) | (Optional) Determines if the Google Cloud Logging and Monitoring APIs are enabled.<br><br>If this is set to `true`, the following APIs will be enabled:<br>  - logging.googleapis.com<br>  - monitoring.googleapis.com<br><br>Additionally, the Anyscale Cluster Role will be granted access to the following roles:<br>  - logging.logWriter<br>  - monitoring.metricWriter<br>  - monitoring.viewer<br><br>ex:<pre>enable_cloud_logging_monitoring = true</pre> | `bool` | `false` | no |

examples/anyscale-v2-commonname/main.tf

@@ -44,5 +44,7 @@ module "google_anyscale_v2_commonname" {
   anyscale_bucket_location = "US"
 
   # Enable Cloud Logging on GCP - this will enable sending logs and metrics to GCP Logs and Monitoring
-  enable_cloud_logging_monitoring = "true"
+  enable_cloud_logging_monitoring = true
+
+  enable_anyscale_loggingsink = false
 }

main.tf

@@ -337,3 +337,18 @@ module "google_anyscale_memorystore" {
 
   memorystore_vpc_name = local.memorystore_vpc_name
 }
+
+# ------------------------------
+# Google LoggingSink Module
+# ------------------------------
+locals {
+  execute_loggingsink_submodule = var.enable_anyscale_loggingsink ? true : false
+}
+module "google_anyscale_loggingsink" {
+  source              = "./modules/google-anyscale-loggingsink"
+  module_enabled      = local.execute_loggingsink_submodule
+  anyscale_project_id = coalesce(var.existing_project_id, module.google_anyscale_project.project_id)
+  depends_on = [
+    module.google_anyscale_cloudapis
+  ]
+}

modules/google-anyscale-cloudstorage/main.tf

@@ -115,6 +115,7 @@ locals {
 
   bucket_iam_binding_enabled = var.module_enabled && length(local.bucket_binding_roles) > 0 && length(var.bucket_iam_binding_members) > 0 ? true : false
 }
+#trivy:ignore:avd-gcp-0007
 resource "google_storage_bucket_iam_binding" "anyscale_bucket" {
   for_each = local.bucket_iam_binding_enabled ? local.bucket_binding_roles : []
   bucket   = google_storage_bucket.anyscale_bucket[0].name

modules/google-anyscale-loggingsink/README.md

@@ -0,0 +1,54 @@
+[![Build Status][badge-build]][build-status]
+[![Terraform Version][badge-terraform]](https://github.com/hashicorp/terraform/releases)
+[![Google Provider Version][badge-tf-google]](https://github.com/terraform-providers/terraform-provider-google/releases)
+# google-anyscale-loggingsink
+
+This sub-module modifies the `_Default` logging sink to disable syslog logging. The sub-module should be used from the [root module](../../README.md).
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | ~> 5.0 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | ~> 3.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_google"></a> [google](#provider\_google) | 5.38.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [google_logging_project_sink.default](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/logging_project_sink) | resource |
+| [google_client_config.current](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/client_config) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_anyscale_project_id"></a> [anyscale\_project\_id](#input\_anyscale\_project\_id) | (Optional) The ID of the project to create the resource in.<br><br>If not provided, the provider project is used.<br><br>ex:<pre>anyscale_project_id = "my-project"</pre> | `string` | `null` | no |
+| <a name="input_module_enabled"></a> [module\_enabled](#input\_module\_enabled) | (Optional) Determines whether to create the resources inside this module.<br><br>ex:<pre>module_enabled = true</pre> | `bool` | `true` | no |
+| <a name="input_sink_destination"></a> [sink\_destination](#input\_sink\_destination) | (Optional) The destination for the sink.<br><br>ex:<pre>sink_destination = "pubsub.googleapis.com/projects/my-project/topics/my-topic"</pre> | `string` | `null` | no |
+| <a name="input_sink_exclusion_filter"></a> [sink\_exclusion\_filter](#input\_sink\_exclusion\_filter) | (Optional) The filter for the sink.<br><br>ex:<pre>sink_filter = "resource.type=gce_instance AND logName=projects/example/logs/syslog"</pre> | `string` | `null` | no |
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+
+<!-- References -->
+[Terraform]: https://www.terraform.io
+[Issues]: https://github.com/anyscale/sa-terraform-google-cloudfoundation-modules/issues
+[badge-build]: https://github.com/anyscale/sa-terraform-google-cloudfoundation-modules/workflows/CI/CD%20Pipeline/badge.svg
+[badge-terraform]: https://img.shields.io/badge/terraform-1.x%20-623CE4.svg?logo=terraform
+[badge-tf-google]: https://img.shields.io/badge/GCP-5.+-F8991D.svg?logo=terraform
+[build-status]: https://github.com/anyscale/sa-terraform-google-cloudfoundation-modules/actions

modules/google-anyscale-loggingsink/data.tf

@@ -0,0 +1 @@
+data "google_client_config" "current" {}

modules/google-anyscale-loggingsink/main.tf

@@ -0,0 +1,24 @@
+locals {
+  project_id = coalesce(var.anyscale_project_id, data.google_client_config.current.project)
+
+  sink_destination      = var.sink_destination != null ? var.sink_destination : "logging.googleapis.com/projects/${local.project_id}/locations/global/buckets/_Default"
+  sink_exclusion_filter = var.sink_exclusion_filter != null ? var.sink_exclusion_filter : "resource.type=gce_instance AND logName=\"projects/${local.project_id}/logs/syslog\" "
+}
+
+# Update the _Default Logging Sink to disable syslog being sent to the default bucket
+resource "google_logging_project_sink" "default" {
+  count = var.module_enabled ? 1 : 0
+
+  name        = "_Default"
+  destination = local.sink_destination
+
+  exclusions {
+    name        = "anyscale"
+    description = "Exclude syslog files from Anyscale clusters"
+    filter      = local.sink_exclusion_filter
+  }
+  unique_writer_identity = true
+
+  project = var.anyscale_project_id
+
+}

modules/google-anyscale-loggingsink/variables.tf

@@ -0,0 +1,64 @@
+# ------------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+# These variables must be set when using this module.
+# ------------------------------------------------------------------------------
+
+
+# ------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These variables have defaults, but may be overridden.
+# ------------------------------------------------------------------------------
+variable "module_enabled" {
+  description = <<-EOF
+    (Optional) Determines whether to create the resources inside this module.
+
+    ex:
+    ```
+    module_enabled = true
+    ```
+  EOF
+  type        = bool
+  default     = true
+}
+
+variable "anyscale_project_id" {
+  description = <<-EOF
+    (Optional) The ID of the project to create the resource in.
+
+    If not provided, the provider project is used.
+
+    ex:
+    ```
+    anyscale_project_id = "my-project"
+    ```
+  EOF
+  type        = string
+  default     = null
+}
+
+
+variable "sink_destination" {
+  description = <<-EOF
+    (Optional) The destination for the sink.
+
+    ex:
+    ```
+    sink_destination = "pubsub.googleapis.com/projects/my-project/topics/my-topic"
+    ```
+  EOF
+  type        = string
+  default     = null
+}
+
+variable "sink_exclusion_filter" {
+  description = <<-EOF
+    (Optional) The filter for the sink.
+
+    ex:
+    ```
+    sink_filter = "resource.type=gce_instance AND logName=projects/example/logs/syslog"
+    ```
+  EOF
+  type        = string
+  default     = null
+}

modules/google-anyscale-loggingsink/versions.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 5.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.0"
+    }
+  }
+}

variables.tf

@@ -1537,3 +1537,21 @@ variable "anyscale_memorystore_labels" {
   type        = map(string)
   default     = {}
 }
+
+# --------------------------------------------
+# Anyscale LoggingSink Variables
+# --------------------------------------------
+variable "enable_anyscale_loggingsink" {
+  description = <<-EOT
+    (Optional) Determines if the Anyscale Logging Sink is executed.
+
+    This sub-module will disable sending syslog events to the `_Default` Log Sink.
+
+    ex:
+    ```
+    enable_anyscale_loggingsink = true
+    ```
+  EOT
+  type        = bool
+  default     = true
+}