Skip to content

Commit

Permalink
Fix TRUNCATE on hierarchical caggs
Browse files Browse the repository at this point in the history 
When truncating a cagg that had another cagg defined on
top of it, the nested cagg would not get invalidated accordingly.
That was because we were not adding a corresponding entry in
the hypertable invalidation log for the materialization hypertable
of the base cagg.
This commit adds an invalidation entry in the table so that
subsequent refreshes see and properly process this invalidation.

Co-authored-by: name <fabriziomello@gmail.com>
  • Loading branch information
@konskov @fabriziomello
konskov and fabriziomello committed Nov 23, 2022
1 parent 35fa891 commit 40297f1
Show file tree
Hide file tree
Showing 8 changed files with 289 additions and 60 deletions.
11 changes: 11 additions & 0 deletions src/process_utility.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1082,6 +1082,17 @@ process_truncate(ProcessUtilityArgs *args)
TS_TIME_NOBEGIN,
TS_TIME_NOEND);

/* Additionally, this cagg's materialization hypertable could be the
* underlying hypertable for other caggs defined on top of it, in that case
* we must update the hypertable invalidation log */
ContinuousAggHypertableStatus agg_status;

agg_status = ts_continuous_agg_hypertable_status(mat_ht->fd.id);
if (agg_status & HypertableIsRawTable)
ts_cm_functions->continuous_agg_invalidate_raw_ht(mat_ht,
TS_TIME_NOBEGIN,
TS_TIME_NOEND);

/* mark list as changed because we'll add the materialization hypertable */
list_changed = true;
}
Expand Down
54 changes: 44 additions & 10 deletions tsl/test/expected/cagg_on_cagg_integer.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,25 +267,59 @@ SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
10 | 2
(2 rows)

-- TRUNCATE tests
TRUNCATE :CAGG_NAME_2TH_LEVEL;
-- This full refresh will remove all the data from the 3TH level cagg
CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL);
-- Should return no rows
SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;
bucket | temperature
--------+-------------
(0 rows)

SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
bucket | temperature
--------+-------------
(0 rows)

-- If we have all the data in the bottom levels caggs we can rebuild
CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL);
CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL);
-- Now we have all the data
SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;
bucket | temperature
--------+-------------
0 | 17
5 | 20
10 | 2
(3 rows)

SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
bucket | temperature
--------+-------------
0 | 37
10 | 2
(2 rows)

-- DROP tests
\set ON_ERROR_STOP 0
-- should error because it depends of other CAGGs
DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL;
psql:include/cagg_on_cagg_common.sql:164: ERROR: cannot drop view conditions_summary_1_1 because other objects depend on it
psql:include/cagg_on_cagg_common.sql:178: ERROR: cannot drop view conditions_summary_1_1 because other objects depend on it
DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL;
psql:include/cagg_on_cagg_common.sql:165: ERROR: cannot drop view conditions_summary_2_5 because other objects depend on it
psql:include/cagg_on_cagg_common.sql:179: ERROR: cannot drop view conditions_summary_2_5 because other objects depend on it
CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL);
psql:include/cagg_on_cagg_common.sql:166: NOTICE: continuous aggregate "conditions_summary_1_1" is already up-to-date
psql:include/cagg_on_cagg_common.sql:180: NOTICE: continuous aggregate "conditions_summary_1_1" is already up-to-date
CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL);
psql:include/cagg_on_cagg_common.sql:167: NOTICE: continuous aggregate "conditions_summary_2_5" is already up-to-date
psql:include/cagg_on_cagg_common.sql:181: NOTICE: continuous aggregate "conditions_summary_2_5" is already up-to-date
\set ON_ERROR_STOP 1
-- DROP the 3TH level CAGG don't affect others
DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL;
psql:include/cagg_on_cagg_common.sql:171: NOTICE: drop cascades to table _timescaledb_internal._hyper_4_4_chunk
psql:include/cagg_on_cagg_common.sql:185: NOTICE: drop cascades to table _timescaledb_internal._hyper_4_4_chunk
\set ON_ERROR_STOP 0
-- should error because it was dropped
SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
psql:include/cagg_on_cagg_common.sql:174: ERROR: relation "conditions_summary_3_10" does not exist at character 15
psql:include/cagg_on_cagg_common.sql:188: ERROR: relation "conditions_summary_3_10" does not exist at character 15
\set ON_ERROR_STOP 1
-- should work because dropping the top level CAGG
-- don't affect the down level CAGGs
Expand All @@ -308,11 +342,11 @@ SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;

-- DROP the 2TH level CAGG don't affect others
DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL;
psql:include/cagg_on_cagg_common.sql:185: NOTICE: drop cascades to table _timescaledb_internal._hyper_3_3_chunk
psql:include/cagg_on_cagg_common.sql:199: NOTICE: drop cascades to table _timescaledb_internal._hyper_3_3_chunk
\set ON_ERROR_STOP 0
-- should error because it was dropped
SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;
psql:include/cagg_on_cagg_common.sql:188: ERROR: relation "conditions_summary_2_5" does not exist at character 15
psql:include/cagg_on_cagg_common.sql:202: ERROR: relation "conditions_summary_2_5" does not exist at character 15
\set ON_ERROR_STOP 1
-- should work because dropping the top level CAGG
-- don't affect the down level CAGGs
Expand All @@ -327,9 +361,9 @@ SELECT * FROM :CAGG_NAME_1ST_LEVEL ORDER BY bucket;

-- DROP the first CAGG should work
DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL;
psql:include/cagg_on_cagg_common.sql:195: NOTICE: drop cascades to table _timescaledb_internal._hyper_2_2_chunk
psql:include/cagg_on_cagg_common.sql:209: NOTICE: drop cascades to table _timescaledb_internal._hyper_2_2_chunk
\set ON_ERROR_STOP 0
-- should error because it was dropped
SELECT * FROM :CAGG_NAME_1ST_LEVEL ORDER BY bucket;
psql:include/cagg_on_cagg_common.sql:198: ERROR: relation "conditions_summary_1_1" does not exist at character 15
psql:include/cagg_on_cagg_common.sql:212: ERROR: relation "conditions_summary_1_1" does not exist at character 15
\set ON_ERROR_STOP 1
54 changes: 44 additions & 10 deletions tsl/test/expected/cagg_on_cagg_integer_dist_ht.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -301,25 +301,59 @@ SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
10 | 2
(2 rows)

-- TRUNCATE tests
TRUNCATE :CAGG_NAME_2TH_LEVEL;
-- This full refresh will remove all the data from the 3TH level cagg
CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL);
-- Should return no rows
SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;
bucket | temperature
--------+-------------
(0 rows)

SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
bucket | temperature
--------+-------------
(0 rows)

-- If we have all the data in the bottom levels caggs we can rebuild
CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL);
CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL);
-- Now we have all the data
SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;
bucket | temperature
--------+-------------
0 | 17
5 | 20
10 | 2
(3 rows)

SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
bucket | temperature
--------+-------------
0 | 37
10 | 2
(2 rows)

-- DROP tests
\set ON_ERROR_STOP 0
-- should error because it depends of other CAGGs
DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL;
psql:include/cagg_on_cagg_common.sql:164: ERROR: cannot drop view conditions_summary_1_1 because other objects depend on it
psql:include/cagg_on_cagg_common.sql:178: ERROR: cannot drop view conditions_summary_1_1 because other objects depend on it
DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL;
psql:include/cagg_on_cagg_common.sql:165: ERROR: cannot drop view conditions_summary_2_5 because other objects depend on it
psql:include/cagg_on_cagg_common.sql:179: ERROR: cannot drop view conditions_summary_2_5 because other objects depend on it
CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL);
psql:include/cagg_on_cagg_common.sql:166: NOTICE: continuous aggregate "conditions_summary_1_1" is already up-to-date
psql:include/cagg_on_cagg_common.sql:180: NOTICE: continuous aggregate "conditions_summary_1_1" is already up-to-date
CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL);
psql:include/cagg_on_cagg_common.sql:167: NOTICE: continuous aggregate "conditions_summary_2_5" is already up-to-date
psql:include/cagg_on_cagg_common.sql:181: NOTICE: continuous aggregate "conditions_summary_2_5" is already up-to-date
\set ON_ERROR_STOP 1
-- DROP the 3TH level CAGG don't affect others
DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL;
psql:include/cagg_on_cagg_common.sql:171: NOTICE: drop cascades to table _timescaledb_internal._hyper_4_4_chunk
psql:include/cagg_on_cagg_common.sql:185: NOTICE: drop cascades to table _timescaledb_internal._hyper_4_4_chunk
\set ON_ERROR_STOP 0
-- should error because it was dropped
SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
psql:include/cagg_on_cagg_common.sql:174: ERROR: relation "conditions_summary_3_10" does not exist at character 15
psql:include/cagg_on_cagg_common.sql:188: ERROR: relation "conditions_summary_3_10" does not exist at character 15
\set ON_ERROR_STOP 1
-- should work because dropping the top level CAGG
-- don't affect the down level CAGGs
Expand All @@ -342,11 +376,11 @@ SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;

-- DROP the 2TH level CAGG don't affect others
DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL;
psql:include/cagg_on_cagg_common.sql:185: NOTICE: drop cascades to table _timescaledb_internal._hyper_3_3_chunk
psql:include/cagg_on_cagg_common.sql:199: NOTICE: drop cascades to table _timescaledb_internal._hyper_3_3_chunk
\set ON_ERROR_STOP 0
-- should error because it was dropped
SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;
psql:include/cagg_on_cagg_common.sql:188: ERROR: relation "conditions_summary_2_5" does not exist at character 15
psql:include/cagg_on_cagg_common.sql:202: ERROR: relation "conditions_summary_2_5" does not exist at character 15
\set ON_ERROR_STOP 1
-- should work because dropping the top level CAGG
-- don't affect the down level CAGGs
Expand All @@ -361,11 +395,11 @@ SELECT * FROM :CAGG_NAME_1ST_LEVEL ORDER BY bucket;

-- DROP the first CAGG should work
DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL;
psql:include/cagg_on_cagg_common.sql:195: NOTICE: drop cascades to table _timescaledb_internal._hyper_2_2_chunk
psql:include/cagg_on_cagg_common.sql:209: NOTICE: drop cascades to table _timescaledb_internal._hyper_2_2_chunk
\set ON_ERROR_STOP 0
-- should error because it was dropped
SELECT * FROM :CAGG_NAME_1ST_LEVEL ORDER BY bucket;
psql:include/cagg_on_cagg_common.sql:198: ERROR: relation "conditions_summary_1_1" does not exist at character 15
psql:include/cagg_on_cagg_common.sql:212: ERROR: relation "conditions_summary_1_1" does not exist at character 15
\set ON_ERROR_STOP 1
-- cleanup
\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER;
Expand Down
54 changes: 44 additions & 10 deletions tsl/test/expected/cagg_on_cagg_timestamp.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,25 +264,59 @@ SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
Mon Jan 03 00:00:00 2022 | 2
(2 rows)

-- TRUNCATE tests
TRUNCATE :CAGG_NAME_2TH_LEVEL;
-- This full refresh will remove all the data from the 3TH level cagg
CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL);
-- Should return no rows
SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;
bucket | temperature
--------+-------------
(0 rows)

SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
bucket | temperature
--------+-------------
(0 rows)

-- If we have all the data in the bottom levels caggs we can rebuild
CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL);
CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL);
-- Now we have all the data
SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;
bucket | temperature
--------------------------+-------------
Sat Jan 01 00:00:00 2022 | 17
Sun Jan 02 00:00:00 2022 | 20
Mon Jan 03 00:00:00 2022 | 2
(3 rows)

SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
bucket | temperature
--------------------------+-------------
Mon Dec 27 00:00:00 2021 | 37
Mon Jan 03 00:00:00 2022 | 2
(2 rows)

-- DROP tests
\set ON_ERROR_STOP 0
-- should error because it depends of other CAGGs
DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL;
psql:include/cagg_on_cagg_common.sql:164: ERROR: cannot drop view conditions_summary_1_hourly because other objects depend on it
psql:include/cagg_on_cagg_common.sql:178: ERROR: cannot drop view conditions_summary_1_hourly because other objects depend on it
DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL;
psql:include/cagg_on_cagg_common.sql:165: ERROR: cannot drop view conditions_summary_2_daily because other objects depend on it
psql:include/cagg_on_cagg_common.sql:179: ERROR: cannot drop view conditions_summary_2_daily because other objects depend on it
CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL);
psql:include/cagg_on_cagg_common.sql:166: NOTICE: continuous aggregate "conditions_summary_1_hourly" is already up-to-date
psql:include/cagg_on_cagg_common.sql:180: NOTICE: continuous aggregate "conditions_summary_1_hourly" is already up-to-date
CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL);
psql:include/cagg_on_cagg_common.sql:167: NOTICE: continuous aggregate "conditions_summary_2_daily" is already up-to-date
psql:include/cagg_on_cagg_common.sql:181: NOTICE: continuous aggregate "conditions_summary_2_daily" is already up-to-date
\set ON_ERROR_STOP 1
-- DROP the 3TH level CAGG don't affect others
DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL;
psql:include/cagg_on_cagg_common.sql:171: NOTICE: drop cascades to table _timescaledb_internal._hyper_4_4_chunk
psql:include/cagg_on_cagg_common.sql:185: NOTICE: drop cascades to table _timescaledb_internal._hyper_4_4_chunk
\set ON_ERROR_STOP 0
-- should error because it was dropped
SELECT * FROM :CAGG_NAME_3TH_LEVEL ORDER BY bucket;
psql:include/cagg_on_cagg_common.sql:174: ERROR: relation "conditions_summary_3_weekly" does not exist at character 15
psql:include/cagg_on_cagg_common.sql:188: ERROR: relation "conditions_summary_3_weekly" does not exist at character 15
\set ON_ERROR_STOP 1
-- should work because dropping the top level CAGG
-- don't affect the down level CAGGs
Expand All @@ -305,11 +339,11 @@ SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;

-- DROP the 2TH level CAGG don't affect others
DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL;
psql:include/cagg_on_cagg_common.sql:185: NOTICE: drop cascades to table _timescaledb_internal._hyper_3_3_chunk
psql:include/cagg_on_cagg_common.sql:199: NOTICE: drop cascades to table _timescaledb_internal._hyper_3_3_chunk
\set ON_ERROR_STOP 0
-- should error because it was dropped
SELECT * FROM :CAGG_NAME_2TH_LEVEL ORDER BY bucket;
psql:include/cagg_on_cagg_common.sql:188: ERROR: relation "conditions_summary_2_daily" does not exist at character 15
psql:include/cagg_on_cagg_common.sql:202: ERROR: relation "conditions_summary_2_daily" does not exist at character 15
\set ON_ERROR_STOP 1
-- should work because dropping the top level CAGG
-- don't affect the down level CAGGs
Expand All @@ -324,9 +358,9 @@ SELECT * FROM :CAGG_NAME_1ST_LEVEL ORDER BY bucket;

-- DROP the first CAGG should work
DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL;
psql:include/cagg_on_cagg_common.sql:195: NOTICE: drop cascades to table _timescaledb_internal._hyper_2_2_chunk
psql:include/cagg_on_cagg_common.sql:209: NOTICE: drop cascades to table _timescaledb_internal._hyper_2_2_chunk
\set ON_ERROR_STOP 0
-- should error because it was dropped
SELECT * FROM :CAGG_NAME_1ST_LEVEL ORDER BY bucket;
psql:include/cagg_on_cagg_common.sql:198: ERROR: relation "conditions_summary_1_hourly" does not exist at character 15
psql:include/cagg_on_cagg_common.sql:212: ERROR: relation "conditions_summary_1_hourly" does not exist at character 15
\set ON_ERROR_STOP 1

0 comments on commit 40297f1

Please sign in to comment.