Merge pull request #24 from ocadotechnology/views_change

Views now deduplicate data from 2 days instead of 3

bq_schemas/bigquery_views/partition_metadata_v1_0.sql

@@ -1,11 +1,11 @@
 #standardSQL
--- This view aggregates all partition metadata from last 3 days and deduplicates it based on partition reference.
--- Deleted table/partition can be returned by this query up to 3 days.
+-- This view aggregates all partition metadata from last 2 days and deduplicates it based on partition reference.
+-- Deleted table/partition can be returned by this query up to 2 days.
 SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type FROM (
 SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type,
 row_number() OVER (PARTITION BY projectId, datasetId, tableId, partitionId ORDER BY snapshotTime DESC) AS rownum
 FROM `$PROJECT_ID.bigquery.partition_metadata_v1_0`
 WHERE
-_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(CURRENT_DATE(), INTERVAL -2 DAY))
+_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(CURRENT_DATE(), INTERVAL -1 DAY))
   AND CURRENT_TIMESTAMP())
 WHERE rownum = 1

bq_schemas/bigquery_views/table_metadata_v1_0.sql

@@ -1,11 +1,11 @@
 #standardSQL
--- This view aggregates all table metadata from last 3 days and deduplicates it based on table reference.
--- Deleted table can be returned by this query up to 3 days.
+-- This view aggregates all table metadata from last 2 days and deduplicates it based on table reference.
+-- Deleted table can be returned by this query up to 2 days.
 SELECT projectId, datasetId, tableId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type FROM (
 SELECT projectId, datasetId, tableId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type,
 row_number() OVER (PARTITION BY projectId, datasetId, tableId ORDER BY snapshotTime DESC) AS rownum
 FROM `$PROJECT_ID.bigquery.table_metadata_v1_0`
 WHERE
-_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(CURRENT_DATE(), INTERVAL -2 DAY))
+_PARTITIONTIME BETWEEN TIMESTAMP(DATE_ADD(CURRENT_DATE(), INTERVAL -1 DAY))
   AND CURRENT_TIMESTAMP())
 WHERE rownum = 1

bq_schemas/bigquery_views_legacy_sql/partition_metadata_v1_0.sql

@@ -1,10 +1,10 @@
--- This view aggregates all partition metadata from last 3 days and deduplicates it based on partition reference.
--- Deleted table/partition can be returned by this query up to 3 days.
+-- This view aggregates all partition metadata from last 2 days and deduplicates it based on partition reference.
+-- Deleted table/partition can be returned by this query up to 2 days.
 SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type FROM (
 SELECT projectId, datasetId, tableId, partitionId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type,
 row_number() OVER (PARTITION BY projectId, datasetId, tableId, partitionId ORDER BY snapshotTime DESC) AS rownum,
 FROM [bigquery.partition_metadata_v1_0]
 WHERE
-_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 48 * 60 * 60 * 1000000))
+_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 24 * 60 * 60 * 1000000))
   AND TIMESTAMP(UTC_USEC_TO_DAY(CURRENT_TIMESTAMP())))
 WHERE rownum = 1

bq_schemas/bigquery_views_legacy_sql/table_metadata_v0_1.sql

@@ -1,11 +1,11 @@
--- This view aggregates all data from last 3 days and deduplicates it based on table reference.
--- Deleted table can be returned by this query up to 3 days.
+-- This view aggregates all data from last 2 days and deduplicates it based on table reference.
+-- Deleted table can be returned by this query up to 2 days.
 SELECT * FROM (
 SELECT projectId, datasetId, tableId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type,  timePartitioning.type AS partitioningType,
 row_number() OVER (PARTITION BY projectId, datasetId, tableId ORDER BY snapshotTime DESC) AS rownum,
 COUNT(partition.partitionId) WITHIN RECORD AS partitionCount
 FROM [bigquery.table_metadata_v0_1]
 WHERE
-_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 48 * 60 * 60 * 1000000))
+_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 24 * 60 * 60 * 1000000))
   AND TIMESTAMP(UTC_USEC_TO_DAY(CURRENT_TIMESTAMP())))
 WHERE rownum = 1

bq_schemas/bigquery_views_legacy_sql/table_metadata_v1_0.sql

@@ -1,10 +1,10 @@
--- This view aggregates all table metadata from last 3 days and deduplicates it based on table reference.
--- Deleted table can be returned by this query up to 3 days.
+-- This view aggregates all table metadata from last 2 days and deduplicates it based on table reference.
+-- Deleted table can be returned by this query up to 2 days.
 SELECT projectId, datasetId, tableId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type FROM (
 SELECT projectId, datasetId, tableId, creationTime, lastModifiedTime, location, numBytes, numLongTermBytes, numRows, type,
 row_number() OVER (PARTITION BY projectId, datasetId, tableId ORDER BY snapshotTime DESC) AS rownum,
 FROM [bigquery.table_metadata_v1_0]
 WHERE
-_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 48 * 60 * 60 * 1000000))
+_PARTITIONTIME BETWEEN TIMESTAMP(UTC_USEC_TO_DAY(NOW() - 24 * 60 * 60 * 1000000))
   AND TIMESTAMP(UTC_USEC_TO_DAY(CURRENT_TIMESTAMP())))
 WHERE rownum = 1