add sample bigquery configuration, make sure config schema is up to d…

…ate, use bigquery sink in default cloud run setup (#566)
silverton-io · May 5, 2023 · 2ef04cb · 2ef04cb
1 parent c3a2fee
commit 2ef04cb
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 55 deletions.
diff --git a/deploy/terraform/gcp/cloud_run/config.yml.tftpl b/deploy/terraform/gcp/cloud_run/config.yml.tftpl
@@ -85,9 +85,10 @@ registry:
 
 sinks:
   - name: primary
-    type: pubsub
+    type: bigquery
     deliveryRequired: true
     project: ${project}
+    dataset: ${dataset}
     defaultOutput: ${defaultOutput}
     deadletterOutput: ${deadletterOutput}
 

diff --git a/deploy/terraform/gcp/cloud_run/main.tf b/deploy/terraform/gcp/cloud_run/main.tf
@@ -10,6 +10,13 @@ data "google_project" "project" {}
 #   }
 # }
 
+terraform {
+  backend "gcs" {
+    bucket = "silverton-tfstate"
+    prefix = "buz/production/"
+  }
+}
+
 module "template_files" {
   source = "hashicorp/dir/template"
 
@@ -37,14 +44,6 @@ resource "google_storage_bucket_object" "schemas" {
   source   = each.value.source_path
 }
 
-resource "google_pubsub_topic" "default_output" {
-  name = local.default_output
-}
-
-resource "google_pubsub_topic" "deadletter_output" {
-  name = local.deadletter_output
-}
-
 resource "google_secret_manager_secret" "buz_config" {
   secret_id = local.config
 
@@ -65,6 +64,7 @@ resource "google_secret_manager_secret_version" "buz_config" {
   secret = google_secret_manager_secret.buz_config.id
   secret_data = templatefile("${path.module}/config.yml.tftpl", {
     project          = var.gcp_project,
+    dataset          = var.bigquery_dataset_name,
     system           = var.system,
     env              = var.env,
     port             = var.buz_service_container_port
@@ -174,8 +174,6 @@ resource "google_cloud_run_service" "buz" {
   depends_on = [
     google_project_service.project_services,
     google_storage_bucket.schemas,
-    google_pubsub_topic.deadletter_output,
-    google_pubsub_topic.default_output,
     null_resource.pull_and_push_image,
   ]
 }
@@ -232,43 +230,3 @@ resource "google_bigquery_dataset" "buz" {
   description   = "A dataset for Buz events"
   location      = var.bigquery_location
 }
-
-resource "google_bigquery_table" "default" {
-  table_id   = local.default_table
-  dataset_id = google_bigquery_dataset.buz.dataset_id
-  schema     = file("schema.json")
-  depends_on = [
-    google_bigquery_dataset.buz
-  ]
-}
-
-resource "google_bigquery_table" "deadletter" {
-  table_id   = local.deadletter_table
-  dataset_id = google_bigquery_dataset.buz.dataset_id
-  schema     = file("schema.json")
-  depends_on = [
-    google_bigquery_dataset.buz
-  ]
-}
-
-resource "google_pubsub_subscription" "default" {
-  name  = local.default_subscription
-  topic = google_pubsub_topic.default_output.name
-  bigquery_config {
-    table            = "${google_bigquery_table.default.project}.${google_bigquery_table.default.dataset_id}.${google_bigquery_table.default.table_id}"
-    use_topic_schema = true
-    write_metadata   = true
-  }
-  depends_on = [google_project_iam_member.bigquery_viewer, google_project_iam_member.bigquery_editor, google_bigquery_table.default]
-}
-
-resource "google_pubsub_subscription" "deadletter" {
-  name  = local.deadletter_subscription
-  topic = google_pubsub_topic.deadletter_output.name
-  bigquery_config {
-    table            = "${google_bigquery_table.deadletter.project}.${google_bigquery_table.deadletter.dataset_id}.${google_bigquery_table.deadletter.table_id}"
-    use_topic_schema = true
-    write_metadata   = true
-  }
-  depends_on = [google_project_iam_member.bigquery_viewer, google_project_iam_member.bigquery_editor, google_bigquery_table.deadletter]
-}
diff --git a/examples/devel/buz/simple.conf.yml b/examples/devel/buz/simple.conf.yml
@@ -192,6 +192,13 @@ sinks:
   #   apiKey: abdf41b4-f6e8-46a7-bb80-d349181af78c
   #   defaultOutput: main # This is tied to splunk HEC so it's moot
   #   deadletterOutput: main # This is tied to splunk HEC so it's moot
+  # - name: bigquery
+  #   type: bigquery
+  #   deliveryRequired: true
+  #   project: $YOUR_PROJECT
+  #   dataset: buz
+  #   defaultOutput: events
+  #   deadletterOutput: invalid_events
 
 squawkBox:
   enabled: true

diff --git a/schemas/io.silverton/buz/internal/config/app/v1.0.json b/schemas/io.silverton/buz/internal/config/app/v1.0.json
@@ -819,7 +819,8 @@
                             "nats",
                             "amplitude",
                             "eventbridge",
-                            "splunk"
+                            "splunk",
+                            "bigquery"
                         ]
                     },
                     "deliveryRequired": {
@@ -840,6 +841,11 @@
                         "default": "buz-gcp-project",
                         "description": "The name of the GCP project"
                     },
+                    "dataset": {
+                        "type": "string",
+                        "default": "buz-gcp-dataset",
+                        "description": "The name of the BigQuery dataset"
+                    },
                     "url": {
                         "type": "string",
                         "description": "The destination url",
@@ -1151,12 +1157,23 @@
                     {
                         "properties": {
                             "type": {
-                                "const": "amplitude"
+                                "const": "splunk"
+                            }
+                        },
+                        "required": [
+                            "url",
+                            "apiKey"
+                        ]
+                    },
+                    {
+                        "properties": {
+                            "type": {
+                                "const": "bigquery"
                             }
                         },
                         "required": [
-                            "apiKey",
-                            "region"
+                            "project",
+                            "dataset"
                         ]
                     }
                 ]