Add invalidations for incomplete aggregates

As part of the 2.0 continous aggregate changes, we are removing the
continuous_aggs_completed_threshold table.  However, this may result
in currently running aggregates being considered complete even if
their completed threshold hadn't reached the invalidation threshold.
This change fixes this by adding an entry to the invalidation log
for any such aggregates.

Fixes #2314

scripts/test_update_from_tag.sh

@@ -175,14 +175,14 @@ docker_run_vol ${CONTAINER_UPDATED} ${UPDATE_VOLUME}:/var/lib/postgresql/data ${
 echo "Executing ALTER EXTENSION timescaledb UPDATE"
 docker_pgcmd ${CONTAINER_UPDATED} "ALTER EXTENSION timescaledb UPDATE"
 
-docker_exec ${CONTAINER_UPDATED} "pg_dump -h localhost -U postgres -Fc single > /tmp/single.sql"
-docker cp ${CONTAINER_UPDATED}:/tmp/single.sql ${TEST_TMPDIR}/single.sql
-
 echo "Executing setup script on clean"
 docker_pgscript ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/setup.${TEST_VERSION}.sql
 
 echo "Testing updated vs clean"
-docker_pgdiff ${CONTAINER_UPDATED} ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/test-rerun.sql
+docker_pgdiff ${CONTAINER_UPDATED} ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/test-rerun.${TEST_VERSION}.sql
+
+docker_exec ${CONTAINER_UPDATED} "pg_dump -h localhost -U postgres -Fc single > /tmp/single.sql"
+docker cp ${CONTAINER_UPDATED}:/tmp/single.sql ${TEST_TMPDIR}/single.sql
 
 echo "Restoring database on clean version"
 docker cp ${TEST_TMPDIR}/single.sql ${CONTAINER_CLEAN_RESTORE}:/tmp/single.sql

scripts/test_updates.sh

@@ -30,7 +30,7 @@ do
             docker rmi -f ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
             ;;
         d)
-            echo "Keeping temporary directory"
+            echo "Keeping temporary directory ${TEST_TMPDIR}"
             KEEP_TEMP_DIRS=true
             TEST_UPDATE_FROM_TAGS_EXTRA_ARGS="-d"
             ;;

sql/updates/latest-dev.sql

@@ -303,6 +303,11 @@ SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job', 'WHERE
 GRANT SELECT ON _timescaledb_config.bgw_job TO PUBLIC;
 GRANT SELECT ON _timescaledb_config.bgw_job_id_seq TO PUBLIC;
 
+-- set up future invalidation for current caggs
+INSERT INTO _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
+SELECT materialization_id, BIGINT '-9223372036854775808', watermark, 9223372036854775807
+FROM _timescaledb_catalog.continuous_aggs_completed_threshold;
+
 -- drop completed_threshold table, which is no longer used
 ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_completed_threshold;
 DROP TABLE IF EXISTS _timescaledb_catalog.continuous_aggs_completed_threshold;

test/sql/updates/setup.continuous_aggs.v2.sql

@@ -263,3 +263,49 @@ BEGIN
 END $$;
 
 REFRESH MATERIALIZED VIEW mat_ignoreinval;
+
+-- test proper invalidations after update
+CREATE TABLE inval_test (time TIMESTAMPTZ, location TEXT, temperature DOUBLE PRECISION);
+SELECT create_hypertable('inval_test', 'time', chunk_time_interval => INTERVAL '1 week');
+
+INSERT INTO inval_test
+SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1 day'), 'POR', generate_series(40.5, 50.0, 0.5);
+INSERT INTO inval_test
+SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1 day'), 'NYC', generate_series(31.0, 50.0, 1.0);
+
+DO LANGUAGE PLPGSQL $$
+DECLARE
+  ts_version TEXT;
+BEGIN
+  SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
+
+  IF ts_version < '2.0.0' THEN
+    CREATE VIEW mat_inval
+    WITH ( timescaledb.continuous, timescaledb.materialized_only=true,
+           timescaledb.refresh_lag='-20 days',
+           timescaledb.refresh_interval='12 hours',
+           timescaledb.max_interval_per_job='100000 days' )
+    AS 
+      SELECT time_bucket('10 minute', time) as bucket, location, min(temperature) as min_temp,
+        max(temperature) as max_temp, avg(temperature) as avg_temp
+      FROM inval_test
+      GROUP BY bucket, location;
+  ELSE
+    CREATE MATERIALIZED VIEW mat_inval
+    WITH ( timescaledb.continuous, timescaledb.materialized_only=true )
+    AS 
+      SELECT time_bucket('10 minute', time) as bucket, location, min(temperature) as min_temp, 
+        max(temperature) as max_temp, avg(temperature) as avg_temp
+      FROM inval_test
+      GROUP BY bucket, location WITH NO DATA;
+
+    PERFORM add_continuous_aggregate_policy('mat_inval', NULL, '-20 days'::interval, '12 hours');
+  END IF;
+END $$;
+
+REFRESH MATERIALIZED VIEW mat_inval;
+
+INSERT INTO inval_test
+SELECT generate_series('2118-12-01 00:00'::timestamp, '2118-12-20 00:00'::timestamp, '1 day'), 'POR', generate_series(135.25, 140.0, 0.25);
+INSERT INTO inval_test
+SELECT generate_series('2118-12-01 00:00'::timestamp, '2118-12-20 00:00'::timestamp, '1 day'), 'NYC', generate_series(131.0, 150.0, 1.0);

test/sql/updates/test-rerun.v2.sql

@@ -0,0 +1,5 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\ir test-rerun.v1.sql

test/sql/updates/test-rerun.v3.sql

@@ -0,0 +1,5 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\ir test-rerun.v1.sql

test/sql/updates/test-rerun.v4.sql

@@ -0,0 +1,5 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\ir test-rerun.v1.sql

test/sql/updates/test-rerun.v5.sql

@@ -0,0 +1,5 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\ir test-rerun.v1.sql

test/sql/updates/test-rerun.v6.sql

@@ -0,0 +1,9 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+\ir test-rerun.v1.sql
+
+SELECT count(*) FROM mat_inval;
+REFRESH MATERIALIZED VIEW mat_inval;
+SELECT count(*) FROM mat_inval;