Add invalidations for incomplete aggregates

scripts/test_update_from_tag.sh

@@ -175,14 +175,14 @@ docker_run_vol ${CONTAINER_UPDATED} ${UPDATE_VOLUME}:/var/lib/postgresql/data ${
 echo "Executing ALTER EXTENSION timescaledb UPDATE"
 docker_pgcmd ${CONTAINER_UPDATED} "ALTER EXTENSION timescaledb UPDATE"
 
-docker_exec ${CONTAINER_UPDATED} "pg_dump -h localhost -U postgres -Fc single > /tmp/single.sql"
-docker cp ${CONTAINER_UPDATED}:/tmp/single.sql ${TEST_TMPDIR}/single.sql
-
 echo "Executing setup script on clean"
 docker_pgscript ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/setup.${TEST_VERSION}.sql
 
 echo "Testing updated vs clean"
-docker_pgdiff ${CONTAINER_UPDATED} ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/test-rerun.sql
+docker_pgdiff ${CONTAINER_UPDATED} ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/test-rerun.${TEST_VERSION}.sql
+
+docker_exec ${CONTAINER_UPDATED} "pg_dump -h localhost -U postgres -Fc single > /tmp/single.sql"
+docker cp ${CONTAINER_UPDATED}:/tmp/single.sql ${TEST_TMPDIR}/single.sql
 
 echo "Restoring database on clean version"
 docker cp ${TEST_TMPDIR}/single.sql ${CONTAINER_CLEAN_RESTORE}:/tmp/single.sql

scripts/test_updates.sh

@@ -30,7 +30,7 @@ do
             docker rmi -f ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
             ;;
         d)
-            echo "Keeping temporary directory"
+            echo "Keeping temporary directory ${TEST_TMPDIR}"
             KEEP_TEMP_DIRS=true
             TEST_UPDATE_FROM_TAGS_EXTRA_ARGS="-d"
             ;;

sql/updates/latest-dev.sql

@@ -303,6 +303,15 @@ SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job', 'WHERE
 GRANT SELECT ON _timescaledb_config.bgw_job TO PUBLIC;
 GRANT SELECT ON _timescaledb_config.bgw_job_id_seq TO PUBLIC;
 
+-- Add entry to materialization invalidation log to indicate that [watermark, +infinity) is invalid
+INSERT INTO _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
+SELECT materialization_id, BIGINT '-9223372036854775808', watermark, 9223372036854775807
+FROM _timescaledb_catalog.continuous_aggs_completed_threshold;
+-- Also handle continuous aggs that have never been run
+INSERT INTO _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
+SELECT (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg EXCEPT SELECT materialization_id FROM _timescaledb_catalog.continuous_aggs_completed_threshold), 
+-9223372036854775808, -9223372036854775808, 9223372036854775807;
+
 -- drop completed_threshold table, which is no longer used
 ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_completed_threshold;
 DROP TABLE IF EXISTS _timescaledb_catalog.continuous_aggs_completed_threshold;

test/sql/updates/setup.continuous_aggs.v2.sql

@@ -263,3 +263,50 @@ BEGIN
 END $$;
 
 REFRESH MATERIALIZED VIEW mat_ignoreinval;
+
+-- test new data beyond the invalidation threshold is properly handled --
+CREATE TABLE inval_test (time TIMESTAMPTZ, location TEXT, temperature DOUBLE PRECISION);
+SELECT create_hypertable('inval_test', 'time', chunk_time_interval => INTERVAL '1 week');
+
+INSERT INTO inval_test
+SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1 day'), 'POR', generate_series(40.5, 50.0, 0.5);
+INSERT INTO inval_test
+SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1 day'), 'NYC', generate_series(31.0, 50.0, 1.0);
+
+DO LANGUAGE PLPGSQL $$
+DECLARE
+  ts_version TEXT;
+BEGIN
+  SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
+
+  IF ts_version < '2.0.0' THEN
+    CREATE VIEW mat_inval
+    WITH ( timescaledb.continuous, timescaledb.materialized_only=true,
+           timescaledb.refresh_lag='-20 days',
+           timescaledb.refresh_interval='12 hours',
+           timescaledb.max_interval_per_job='100000 days' )
+    AS 
+      SELECT time_bucket('10 minute', time) as bucket, location, min(temperature) as min_temp,
+        max(temperature) as max_temp, avg(temperature) as avg_temp
+      FROM inval_test
+      GROUP BY bucket, location;
+
+  ELSE
+    CREATE MATERIALIZED VIEW mat_inval
+    WITH ( timescaledb.continuous, timescaledb.materialized_only=true )
+    AS 
+      SELECT time_bucket('10 minute', time) as bucket, location, min(temperature) as min_temp, 
+        max(temperature) as max_temp, avg(temperature) as avg_temp
+      FROM inval_test
+      GROUP BY bucket, location WITH NO DATA;
+
+    PERFORM add_continuous_aggregate_policy('mat_inval', NULL, '-20 days'::interval, '12 hours');
+  END IF;
+END $$;
+
+REFRESH MATERIALIZED VIEW mat_inval;
+
+INSERT INTO inval_test
+SELECT generate_series('2118-12-01 00:00'::timestamp, '2118-12-20 00:00'::timestamp, '1 day'), 'POR', generate_series(135.25, 140.0, 0.25);
+INSERT INTO inval_test
+SELECT generate_series('2118-12-01 00:00'::timestamp, '2118-12-20 00:00'::timestamp, '1 day'), 'NYC', generate_series(131.0, 150.0, 1.0);

test/sql/updates/test-rerun.v2.sql

@@ -0,0 +1,5 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\ir test-rerun.v1.sql

test/sql/updates/test-rerun.v3.sql

@@ -0,0 +1,5 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\ir test-rerun.v1.sql

test/sql/updates/test-rerun.v4.sql

@@ -0,0 +1,5 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\ir test-rerun.v1.sql

test/sql/updates/test-rerun.v5.sql

@@ -0,0 +1,5 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\ir test-rerun.v1.sql

test/sql/updates/test-rerun.v6.sql

@@ -0,0 +1,9 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\ir test-rerun.v1.sql
+
+SELECT count(*) FROM mat_inval;
+REFRESH MATERIALIZED VIEW mat_inval;
+SELECT count(*) FROM mat_inval;