Skip to content
This repository has been archived by the owner on Aug 25, 2023. It is now read-only.

Commit

Permalink
Merge 38b2b57 into d4258d6
Browse files Browse the repository at this point in the history
  • Loading branch information
@radkomateusz
radkomateusz committed Jul 23, 2019
2 parents d4258d6 + 38b2b57 commit 5302dc8
Show file tree
Hide file tree
Showing 6 changed files with 88 additions and 173 deletions.
2 changes: 1 addition & 1 deletion src/slo/sli_handlers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def create_slo_recalculation_tasks():
Tasks.create(method='POST',
url='/sli/latency_for_x_days',
params={'x_days': x_days})
for x_days in [3, 4, 5, 7]
for x_days in [4, 5, 7]
]
tasks.append(
Tasks.create(
Expand Down
10 changes: 5 additions & 5 deletions terraform/sli/SLI_0_days_views.tf
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
resource "google_bigquery_table" "census_data_0_days_ago_view" {
resource "google_bigquery_table" "data_modified_0_days_ago_view" {
project = local.SLI_views_destination_project
dataset_id = google_bigquery_dataset.SLI_backup_creation_latency_views_dataset.dataset_id
table_id = "census_data_0_days_ago"
description = "All tables and partitions seen by GCP Census 0 days ago"
table_id = "data_modified_0_days_ago"
description = "All tables which have modifications before 0 days ago"

view {
query = <<EOF
#legacySQL
-- Shows all tables and partitions seen by census now
-- Shows all tables which have modifications before 0 days ago
SELECT * FROM (
SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
FROM (
Expand Down Expand Up @@ -60,7 +60,7 @@ resource "google_bigquery_table" "SLI_0_days_view" {
IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) as backup_created,
IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) as backup_last_modified
FROM
[${google_bigquery_table.census_data_0_days_ago_view.id}] as census
[${google_bigquery_table.data_modified_0_days_ago_view.id}] as census
LEFT JOIN (
SELECT
backup_created, backup_last_modified, source_project_id, source_dataset_id, source_table_id, source_partition_id
Expand Down
85 changes: 0 additions & 85 deletions terraform/sli/SLI_3_days_views.tf
View file

This file was deleted.

20 changes: 10 additions & 10 deletions terraform/sli/SLI_4_days_views.tf
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
resource "google_bigquery_table" "census_data_4_days_ago_view" {
resource "google_bigquery_table" "data_modified_3_days_ago_view" {
project = local.SLI_views_destination_project
dataset_id = google_bigquery_dataset.SLI_backup_creation_latency_views_dataset.dataset_id
table_id = "census_data_4_days_ago"
description = "All tables and partitions seen by GCP Census 4 days ago"
table_id = "data_modified_3_days_ago"
description = "All tables which have modifications before 3 days ago"

view {
query = <<EOF
#legacySQL
-- Shows all tables and partitions seen by census 4 days ago
-- Shows all tables which have modifications before 3 days ago
SELECT * FROM (
SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
FROM (
Expand All @@ -17,8 +17,8 @@ resource "google_bigquery_table" "census_data_4_days_ago_view" {
FROM
[${var.gcp_census_project}.bigquery.table_metadata_v1_0]
WHERE
_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(NOW(), -7, "DAY")) AND TIMESTAMP(DATE_ADD(NOW(), -2, "DAY"))
AND lastModifiedTime <= TIMESTAMP(DATE_ADD(NOW(), -4, "DAY"))
_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(NOW(), -6, "DAY")) AND TIMESTAMP(DATE_ADD(NOW(), -1, "DAY"))
AND lastModifiedTime <= TIMESTAMP(DATE_ADD(NOW(), -3, "DAY"))
AND timePartitioning.type IS NULL AND type='TABLE'
)
WHERE rownum = 1
Expand All @@ -31,8 +31,8 @@ resource "google_bigquery_table" "census_data_4_days_ago_view" {
FROM
[${var.gcp_census_project}.bigquery.partition_metadata_v1_0]
WHERE
_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(NOW(), -7, "DAY")) AND TIMESTAMP(DATE_ADD(NOW(), -2, "DAY"))
AND lastModifiedTime <= TIMESTAMP(DATE_ADD(NOW(), -4, "DAY"))
_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(NOW(), -6, "DAY")) AND TIMESTAMP(DATE_ADD(NOW(), -1, "DAY"))
AND lastModifiedTime <= TIMESTAMP(DATE_ADD(NOW(), -3, "DAY"))
)
WHERE rownum = 1
)
Expand Down Expand Up @@ -60,7 +60,7 @@ resource "google_bigquery_table" "SLI_4_days_view" {
IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) as backup_created,
IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) as backup_last_modified
FROM
[${google_bigquery_table.census_data_4_days_ago_view.id}] as census
[${google_bigquery_table.data_modified_3_days_ago_view.id}] as census
LEFT JOIN (
SELECT
backup_created, backup_last_modified, source_project_id, source_dataset_id, source_table_id, source_partition_id
Expand All @@ -77,7 +77,7 @@ resource "google_bigquery_table" "SLI_4_days_view" {
AND projectId != "${var.bbq_restoration_project}"
AND partitionId != "__UNPARTITIONED__"
AND census.numRows != 0
AND IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) < TIMESTAMP(DATE_ADD(CURRENT_TIMESTAMP(), -4 , "DAY"))
AND IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) < TIMESTAMP(DATE_ADD(CURRENT_TIMESTAMP(), -3 , "DAY"))
AND IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) < lastModifiedTime
EOF
use_legacy_sql = true
Expand Down
26 changes: 13 additions & 13 deletions terraform/sli/SLI_5_days_views.tf
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
resource "google_bigquery_table" "census_data_5_days_ago_view" {
resource "google_bigquery_table" "data_modified_4_days_ago_view" {
project = local.SLI_views_destination_project
dataset_id = google_bigquery_dataset.SLI_backup_creation_latency_views_dataset.dataset_id
table_id = "census_data_5_days_ago"
description = "All tables and partitions seen by GCP Census 5 days ago"
table_id = "data_modified_4_days_ago"
description = "All tables which have modifications before 4 days ago"

view {
query = <<EOF
#legacySQL
-- Shows all tables and partitions seen by census 5 days ago
-- Shows all tables which have modifications before 4 days ago
SELECT * FROM (
SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, numRows
FROM (
Expand All @@ -16,10 +16,10 @@ resource "google_bigquery_table" "census_data_5_days_ago_view" {
ROW_NUMBER() OVER (PARTITION BY projectId, datasetId, tableId ORDER BY snapshotTime DESC) AS rownum
FROM
[${var.gcp_census_project}.bigquery.table_metadata_v1_0]
WHERE
_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(NOW(), -8, "DAY")) AND TIMESTAMP(DATE_ADD(NOW(), -3, "DAY"))
AND lastModifiedTime <= TIMESTAMP(DATE_ADD(NOW(), -5, "DAY"))
AND timePartitioning.type IS NULL AND type='TABLE'
WHERE
_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(NOW(), -7, "DAY")) AND TIMESTAMP(DATE_ADD(NOW(), -2, "DAY"))
AND lastModifiedTime <= TIMESTAMP(DATE_ADD(NOW(), -4, "DAY"))
AND timePartitioning.type IS NULL AND type='TABLE'
)
WHERE rownum = 1
), (
Expand All @@ -31,8 +31,8 @@ resource "google_bigquery_table" "census_data_5_days_ago_view" {
FROM
[${var.gcp_census_project}.bigquery.partition_metadata_v1_0]
WHERE
_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(NOW(), -8, "DAY")) AND TIMESTAMP(DATE_ADD(NOW(), -3, "DAY"))
AND lastModifiedTime <= TIMESTAMP(DATE_ADD(NOW(), -5, "DAY"))
_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(NOW(), -7, "DAY")) AND TIMESTAMP(DATE_ADD(NOW(), -2, "DAY"))
AND lastModifiedTime <= TIMESTAMP(DATE_ADD(NOW(), -4, "DAY"))
)
WHERE rownum = 1
)
Expand Down Expand Up @@ -60,7 +60,7 @@ resource "google_bigquery_table" "SLI_5_days_view" {
IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) as backup_created,
IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) as backup_last_modified
FROM
[${google_bigquery_table.census_data_5_days_ago_view.id}] as census
[${google_bigquery_table.data_modified_4_days_ago_view.id}] as census
LEFT JOIN (
SELECT
backup_created, backup_last_modified, source_project_id, source_dataset_id, source_table_id, source_partition_id
Expand All @@ -76,8 +76,8 @@ resource "google_bigquery_table" "SLI_5_days_view" {
projectId != "${var.bbq_project}"
AND projectId != "${var.bbq_restoration_project}"
AND partitionId != "__UNPARTITIONED__"
AND numRows != 0
AND IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) < TIMESTAMP(DATE_ADD(CURRENT_TIMESTAMP(), -5 , "DAY"))
AND census.numRows != 0
AND IFNULL(last_backups.backup_created, MSEC_TO_TIMESTAMP(0)) < TIMESTAMP(DATE_ADD(CURRENT_TIMESTAMP(), -4 , "DAY"))
AND IFNULL(last_backups.backup_last_modified, MSEC_TO_TIMESTAMP(0)) < lastModifiedTime
EOF
use_legacy_sql = true
Expand Down
Loading

0 comments on commit 5302dc8

Please sign in to comment.