Merge 6ac5569 into 832900c

ocadotechnology · Apr 23, 2019 · cfd9278 · cfd9278
2 parents 832900c + 6ac5569
commit cfd9278
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 50 deletions.
diff --git a/terraform/sli/SLI_0_days_views.tf b/terraform/sli/SLI_0_days_views.tf
@@ -1,14 +1,15 @@
 resource "google_bigquery_table" "census_data_0_days_ago_view" {
   project = "${local.SLI_views_destination_project}"
-  dataset_id = "${var.SLI_backup_creation_latency_views_dataset}"
+  dataset_id = "${google_bigquery_dataset.SLI_backup_creation_latency_views_dataset}"
   table_id = "census_data_0_days_ago"
+  description = "All tables and partitions seen by GCP Census 0 days ago"
 
   view {
     query = <<EOF
             #legacySQL
               -- Shows all tables and partitions seen by census now
             SELECT * FROM (
-              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime
+              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
               FROM (
                 SELECT
                   projectId, datasetId, tableId, creationTime, lastModifiedTime, 'None' AS partitionId,
@@ -21,7 +22,7 @@ resource "google_bigquery_table" "census_data_0_days_ago_view" {
               )
               WHERE rownum = 1
             ), (
-              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime
+              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
               FROM (
                 SELECT
                   projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime,
@@ -36,14 +37,13 @@ resource "google_bigquery_table" "census_data_0_days_ago_view" {
         EOF
     use_legacy_sql = true
   }
-
-  depends_on = ["google_bigquery_dataset.SLI_backup_creation_latency_views_dataset"]
 }
 
 resource "google_bigquery_table" "SLI_0_days_view" {
   project = "${local.SLI_views_destination_project}"
-  dataset_id = "${var.SLI_backup_creation_latency_views_dataset}"
+  dataset_id = "${google_bigquery_dataset.SLI_backup_creation_latency_views_dataset}"
   table_id = "SLI_0_days"
+  description = "All tables and partitions which backups potentially violate 0 days latency"
 
   view {
     query = <<EOF
@@ -58,12 +58,12 @@ resource "google_bigquery_table" "SLI_0_days_view" {
             IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) as backup_created,
             IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) as backup_last_modified
           FROM
-            [${local.SLI_views_destination_project}.${var.SLI_backup_creation_latency_views_dataset}.census_data_0_days_ago] AS census
+            [${google_bigquery_table.census_data_0_days_ago_view.id}] as census
           LEFT JOIN (
             SELECT
               backup_created, backup_last_modified, source_project_id, source_dataset_id, source_table_id, source_partition_id
             FROM
-              [${local.datastore_export_project}.${var.datastore_export_views_dataset}.last_available_backup_for_every_table_entity]
+              [${google_bigquery_table.last_available_backup_for_every_table_entity_view.id}]
           ) AS last_backups
           ON
             census.projectId=last_backups.source_project_id AND
@@ -74,11 +74,10 @@ resource "google_bigquery_table" "SLI_0_days_view" {
             projectId != "${var.bbq_project}"
             AND projectId != "${var.bbq_restoration_project}"
             AND partitionId != "__UNPARTITIONED__"
+            AND census.numRows != 0
             AND IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) < CURRENT_TIMESTAMP()
             AND IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) < lastModifiedTime
         EOF
     use_legacy_sql = true
   }
-
-  depends_on = ["google_bigquery_table.census_data_0_days_ago_view", "google_bigquery_table.last_available_backup_for_every_table_entity_view"]
 }
diff --git a/terraform/sli/SLI_3_days_views.tf b/terraform/sli/SLI_3_days_views.tf
@@ -1,14 +1,15 @@
 resource "google_bigquery_table" "census_data_3_days_ago_view" {
   project = "${local.SLI_views_destination_project}"
-  dataset_id = "${var.SLI_backup_creation_latency_views_dataset}"
+  dataset_id = "${google_bigquery_dataset.SLI_backup_creation_latency_views_dataset}"
   table_id = "census_data_3_days_ago"
+  description = "All tables and partitions seen by GCP Census 3 days ago"
 
   view {
     query = <<EOF
             #legacySQL
               -- Shows all tables and partitions seen by census 3 days ago
             SELECT * FROM (
-              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime
+              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
               FROM (
                 SELECT
                   projectId, datasetId, tableId, creationTime, lastModifiedTime, 'None' AS partitionId,
@@ -21,7 +22,7 @@ resource "google_bigquery_table" "census_data_3_days_ago_view" {
               )
               WHERE rownum = 1
             ), (
-              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime
+              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
               FROM (
                 SELECT
                   projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime,
@@ -36,15 +37,14 @@ resource "google_bigquery_table" "census_data_3_days_ago_view" {
         EOF
     use_legacy_sql = true
   }
-
-  depends_on = ["google_bigquery_dataset.SLI_backup_creation_latency_views_dataset"]
 }
 
 
 resource "google_bigquery_table" "SLI_3_days_view" {
   project = "${local.SLI_views_destination_project}"
-  dataset_id = "${var.SLI_backup_creation_latency_views_dataset}"
+  dataset_id = "${google_bigquery_dataset.SLI_backup_creation_latency_views_dataset}"
   table_id = "SLI_3_days"
+  description = "All tables and partitions which backups potentially violate 3 days latency"
 
   view {
     query = <<EOF
@@ -59,12 +59,12 @@ resource "google_bigquery_table" "SLI_3_days_view" {
             IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) as backup_created,
             IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) as backup_last_modified
           FROM
-            [${local.SLI_views_destination_project}.${var.SLI_backup_creation_latency_views_dataset}.census_data_3_days_ago] AS census
+            [${google_bigquery_table.census_data_3_days_ago_view.id}] as census
           LEFT JOIN (
             SELECT
               backup_created, backup_last_modified, source_project_id, source_dataset_id, source_table_id, source_partition_id
             FROM
-              [${local.datastore_export_project}.${var.datastore_export_views_dataset}.last_available_backup_for_every_table_entity]
+              [${google_bigquery_table.last_available_backup_for_every_table_entity_view.id}]
           ) AS last_backups
           ON
             census.projectId=last_backups.source_project_id AND
@@ -75,11 +75,10 @@ resource "google_bigquery_table" "SLI_3_days_view" {
             projectId != "${var.bbq_project}"
             AND projectId != "${var.bbq_restoration_project}"
             AND partitionId != "__UNPARTITIONED__"
+            AND census.numRows != 0
             AND IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) < TIMESTAMP(DATE_ADD(CURRENT_TIMESTAMP(), -3 , "DAY"))
             AND IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) < lastModifiedTime
         EOF
     use_legacy_sql = true
   }
-
-  depends_on = ["google_bigquery_table.census_data_3_days_ago_view", "google_bigquery_table.last_available_backup_for_every_table_entity_view"]
 }
diff --git a/terraform/sli/SLI_4_days_views.tf b/terraform/sli/SLI_4_days_views.tf
@@ -1,14 +1,15 @@
 resource "google_bigquery_table" "census_data_4_days_ago_view" {
   project = "${local.SLI_views_destination_project}"
-  dataset_id = "${var.SLI_backup_creation_latency_views_dataset}"
+  dataset_id = "${google_bigquery_dataset.SLI_backup_creation_latency_views_dataset}"
   table_id = "census_data_4_days_ago"
+  description = "All tables and partitions seen by GCP Census 4 days ago"
 
   view {
     query = <<EOF
             #legacySQL
               -- Shows all tables and partitions seen by census 4 days ago
             SELECT * FROM (
-              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime
+              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
               FROM (
                 SELECT
                   projectId, datasetId, tableId, creationTime, lastModifiedTime, 'None' AS partitionId,
@@ -21,7 +22,7 @@ resource "google_bigquery_table" "census_data_4_days_ago_view" {
               )
               WHERE rownum = 1
             ), (
-              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime
+              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
               FROM (
                 SELECT
                   projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime,
@@ -36,15 +37,14 @@ resource "google_bigquery_table" "census_data_4_days_ago_view" {
         EOF
     use_legacy_sql = true
   }
-
-  depends_on = ["google_bigquery_dataset.SLI_backup_creation_latency_views_dataset"]
 }
 
 
 resource "google_bigquery_table" "SLI_4_days_view" {
   project = "${local.SLI_views_destination_project}"
-  dataset_id = "${var.SLI_backup_creation_latency_views_dataset}"
+  dataset_id = "${google_bigquery_dataset.SLI_backup_creation_latency_views_dataset}"
   table_id = "SLI_4_days"
+  description = "All tables and partitions which backups potentially violate 4 days latency"
 
   view {
     query = <<EOF
@@ -59,12 +59,12 @@ resource "google_bigquery_table" "SLI_4_days_view" {
             IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) as backup_created,
             IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) as backup_last_modified
           FROM
-            [${local.SLI_views_destination_project}.${var.SLI_backup_creation_latency_views_dataset}.census_data_4_days_ago] AS census
+            [${google_bigquery_table.census_data_4_days_ago_view.id}] as census
           LEFT JOIN (
             SELECT
               backup_created, backup_last_modified, source_project_id, source_dataset_id, source_table_id, source_partition_id
             FROM
-              [${local.datastore_export_project}.${var.datastore_export_views_dataset}.last_available_backup_for_every_table_entity]
+              [${google_bigquery_table.last_available_backup_for_every_table_entity_view.id}]
           ) AS last_backups
           ON
             census.projectId=last_backups.source_project_id AND
@@ -75,11 +75,10 @@ resource "google_bigquery_table" "SLI_4_days_view" {
             projectId != "${var.bbq_project}"
             AND projectId != "${var.bbq_restoration_project}"
             AND partitionId != "__UNPARTITIONED__"
+            AND census.numRows != 0
             AND IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) < TIMESTAMP(DATE_ADD(CURRENT_TIMESTAMP(), -4 , "DAY"))
             AND IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) < lastModifiedTime
         EOF
     use_legacy_sql = true
   }
-
-  depends_on = ["google_bigquery_table.census_data_4_days_ago_view", "google_bigquery_table.last_available_backup_for_every_table_entity_view"]
 }
diff --git a/terraform/sli/SLI_5_days_views.tf b/terraform/sli/SLI_5_days_views.tf
@@ -1,14 +1,16 @@
 resource "google_bigquery_table" "census_data_5_days_ago_view" {
   project = "${local.SLI_views_destination_project}"
-  dataset_id = "${var.SLI_backup_creation_latency_views_dataset}"
+  dataset_id = "${google_bigquery_dataset.SLI_backup_creation_latency_views_dataset}"
   table_id = "census_data_5_days_ago"
+  description = "All tables and partitions seen by GCP Census 5 days ago"
+
 
   view {
     query = <<EOF
             #legacySQL
               -- Shows all tables and partitions seen by census 5 days ago
             SELECT * FROM (
-              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime
+              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
               FROM (
                 SELECT
                   projectId, datasetId, tableId, creationTime, lastModifiedTime, 'None' AS partitionId,
@@ -21,7 +23,7 @@ resource "google_bigquery_table" "census_data_5_days_ago_view" {
               )
               WHERE rownum = 1
             ), (
-              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime
+              SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
               FROM (
                 SELECT
                   projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime,
@@ -36,15 +38,14 @@ resource "google_bigquery_table" "census_data_5_days_ago_view" {
         EOF
     use_legacy_sql = true
   }
-
-  depends_on = ["google_bigquery_dataset.SLI_backup_creation_latency_views_dataset"]
 }
 
 
 resource "google_bigquery_table" "SLI_5_days_view" {
   project = "${local.SLI_views_destination_project}"
-  dataset_id = "${var.SLI_backup_creation_latency_views_dataset}"
+  dataset_id = "${google_bigquery_dataset.SLI_backup_creation_latency_views_dataset}"
   table_id = "SLI_5_days"
+  description = "All tables and partitions which backups potentially violate 5 days latency"
 
   view {
     query = <<EOF
@@ -59,12 +60,12 @@ resource "google_bigquery_table" "SLI_5_days_view" {
             IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) as backup_created,
             IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) as backup_last_modified
           FROM
-            [${local.SLI_views_destination_project}.${var.SLI_backup_creation_latency_views_dataset}.census_data_5_days_ago] AS census
+            [${google_bigquery_table.census_data_5_days_ago_view.id}] as census
           LEFT JOIN (
             SELECT
               backup_created, backup_last_modified, source_project_id, source_dataset_id, source_table_id, source_partition_id
             FROM
-              [${local.datastore_export_project}.${var.datastore_export_views_dataset}.last_available_backup_for_every_table_entity]
+              [${google_bigquery_table.last_available_backup_for_every_table_entity_view.id}]
           ) AS last_backups
           ON
             census.projectId=last_backups.source_project_id AND
@@ -75,11 +76,10 @@ resource "google_bigquery_table" "SLI_5_days_view" {
             projectId != "${var.bbq_project}"
             AND projectId != "${var.bbq_restoration_project}"
             AND partitionId != "__UNPARTITIONED__"
+            AND numRows != 0
             AND IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) < TIMESTAMP(DATE_ADD(CURRENT_TIMESTAMP(), -5 , "DAY"))
             AND IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) < lastModifiedTime
         EOF
     use_legacy_sql = true
   }
-
-  depends_on = ["google_bigquery_table.census_data_5_days_ago_view", "google_bigquery_table.last_available_backup_for_every_table_entity_view"]
 }
diff --git a/terraform/sli/SLI_7_days_views.tf b/terraform/sli/SLI_7_days_views.tf
@@ -1,14 +1,15 @@
 resource "google_bigquery_table" "census_data_7_days_ago_view" {
   project = "${local.SLI_views_destination_project}"
-  dataset_id = "${var.SLI_backup_creation_latency_views_dataset}"
+  dataset_id = "${google_bigquery_dataset.SLI_backup_creation_latency_views_dataset.id}"
   table_id = "census_data_7_days_ago"
+  description = "All tables and partitions seen by GCP Census 7 days ago"
 
   view {
     query = <<EOF
           #legacySQL
             -- Shows all tables and partitions seen by census 7 days ago
           SELECT * FROM (
-            SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime
+            SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
             FROM (
               SELECT
                 projectId, datasetId, tableId, creationTime, lastModifiedTime, 'None' AS partitionId,
@@ -21,7 +22,7 @@ resource "google_bigquery_table" "census_data_7_days_ago_view" {
             )
             WHERE rownum = 1
           ), (
-            SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime
+            SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
             FROM (
               SELECT
                 projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime,
@@ -36,15 +37,14 @@ resource "google_bigquery_table" "census_data_7_days_ago_view" {
         EOF
     use_legacy_sql = true
   }
-
-  depends_on = ["google_bigquery_dataset.SLI_backup_creation_latency_views_dataset"]
 }
 
 
 resource "google_bigquery_table" "SLI_7_days_view" {
   project = "${local.SLI_views_destination_project}"
-  dataset_id = "${var.SLI_backup_creation_latency_views_dataset}"
+  dataset_id = "${google_bigquery_dataset.SLI_backup_creation_latency_views_dataset.id}"
   table_id = "SLI_7_days"
+  description = "All tables and partitions which backups potentially violate 7 days latency"
 
   view {
     query = <<EOF
@@ -59,12 +59,12 @@ resource "google_bigquery_table" "SLI_7_days_view" {
               IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) as backup_created,
               IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) as backup_last_modified
             FROM
-              [${local.SLI_views_destination_project}:${var.SLI_backup_creation_latency_views_dataset}.census_data_7_days_ago] AS census
+            [${google_bigquery_table.census_data_7_days_ago_view.id}] as census
             LEFT JOIN (
               SELECT
                 backup_created, backup_last_modified, source_project_id, source_dataset_id, source_table_id, source_partition_id
               FROM
-                [${local.datastore_export_project}:${var.datastore_export_views_dataset}.last_available_backup_for_every_table_entity]
+                [${google_bigquery_table.last_available_backup_for_every_table_entity_view.id}]
             ) AS last_backups
             ON
               census.projectId=last_backups.source_project_id AND
@@ -75,11 +75,10 @@ resource "google_bigquery_table" "SLI_7_days_view" {
               projectId != "${var.bbq_project}"
               AND projectId != "${var.bbq_restoration_project}"
               AND partitionId != "__UNPARTITIONED__"
+              AND numRows != 0
               AND IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) < TIMESTAMP(DATE_ADD(CURRENT_TIMESTAMP(), -7 , "DAY"))
               AND IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) < lastModifiedTime
         EOF
     use_legacy_sql = true
   }
-
-  depends_on = ["google_bigquery_table.census_data_7_days_ago_view", "google_bigquery_table.last_available_backup_for_every_table_entity_view"]
 }