diff --git a/CHANGELOG.md b/CHANGELOG.md index 770657b03d1..b1a32e26679 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,11 @@ argument or resolve the type ambiguity by casting to the intended type. * #4696 Report warning when enabling compression on hypertable * #4745 Fix FK constraint violation error while insert into hypertable which references partitioned table * #4756 Improve compression job IO performance -* #4807 Fix segmentation fault during INSERT into compressed hypertable. +* #4807 Fix segmentation fault during INSERT into compressed hypertable * #4840 Fix performance regressions in the copy code -* #4823 Fix a crash that could occur when using nested user-defined functions with hypertables. +* #4823 Fix a crash that could occur when using nested user-defined functions with hypertables +* #4898 Fix cagg migration failure when trying to resume +* #4955 Fix cagg migration for hypertables using timestamp without timezone **Thanks** * @jflambert for reporting a crash with nested user-defined functions. diff --git a/sql/cagg_migrate.sql b/sql/cagg_migrate.sql index f3db668ef23..34bc1c6f59a 100644 --- a/sql/cagg_migrate.sql +++ b/sql/cagg_migrate.sql @@ -367,6 +367,8 @@ BEGIN CASE _plan_step.config->>'window_start_type' WHEN 'timestamp with time zone' THEN CALL @extschema@.refresh_continuous_aggregate(_cagg, (_plan_step.config->>'window_start')::timestamptz, NULL); + WHEN 'timestamp without time zone' THEN + CALL @extschema@.refresh_continuous_aggregate(_cagg, (_plan_step.config->>'window_start')::timestamp, NULL); WHEN 'bigint' THEN CALL @extschema@.refresh_continuous_aggregate(_cagg, (_plan_step.config->>'window_start')::bigint, NULL); WHEN 'integer' THEN diff --git a/tsl/test/expected/cagg_migrate_integer.out b/tsl/test/expected/cagg_migrate_integer.out index e2bc4caec5f..81a947befbe 100644 --- a/tsl/test/expected/cagg_migrate_integer.out +++ b/tsl/test/expected/cagg_migrate_integer.out @@ -3,6 +3,7 @@ -- LICENSE-TIMESCALE for a copy of the license. \set IS_DISTRIBUTED FALSE \set IS_TIME_DIMENSION FALSE +\set TIME_DIMENSION_DATATYPE INTEGER \ir include/cagg_migrate_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and @@ -14,13 +15,8 @@ \echo 'Running local hypertable tests' Running local hypertable tests \endif -\if :IS_TIME_DIMENSION - \set TIME_DATATYPE TIMESTAMPTZ -\else - \set TIME_DATATYPE INTEGER -\endif CREATE TABLE conditions ( - "time" :TIME_DATATYPE NOT NULL, + "time" :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC ); \if :IS_DISTRIBUTED @@ -48,17 +44,16 @@ CREATE TABLE conditions ( 0.25; \else CREATE OR REPLACE FUNCTION integer_now() - RETURNS integer LANGUAGE SQL STABLE AS + RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$; \if :IS_DISTRIBUTED - CALL distributed_exec ( - $DIST$ - CREATE OR REPLACE FUNCTION integer_now() RETURNS integer LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$; - $DIST$ - ); + SELECT + 'CREATE OR REPLACE FUNCTION integer_now() RETURNS '||:'TIME_DIMENSION_DATATYPE'||' LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$;' AS "STMT" + \gset + CALL distributed_exec (:'STMT'); \endif SELECT set_integer_now_func('conditions', 'integer_now'); set_integer_now_func @@ -74,11 +69,11 @@ CREATE TABLE conditions ( \set ON_ERROR_STOP 0 -- should fail relation does not exist CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:70: ERROR: relation "conditions_summary_daily" does not exist at character 19 +psql:include/cagg_migrate_common.sql:63: ERROR: relation "conditions_summary_daily" does not exist at character 19 CREATE TABLE conditions_summary_daily(); -- should fail continuous agg does not exist CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:73: ERROR: continuous aggregate "public.conditions_summary_daily" does not exist +psql:include/cagg_migrate_common.sql:66: ERROR: continuous aggregate "public.conditions_summary_daily" does not exist \set ON_ERROR_STOP 1 DROP TABLE conditions_summary_daily; CREATE MATERIALIZED VIEW conditions_summary_daily_new @@ -101,7 +96,7 @@ WITH NO DATA; \set ON_ERROR_STOP 0 -- should fail because we don't need to migrate finalized caggs CALL cagg_migrate('conditions_summary_daily_new'); -psql:include/cagg_migrate_common.sql:98: ERROR: continuous aggregate "public.conditions_summary_daily_new" does not require any migration +psql:include/cagg_migrate_common.sql:91: ERROR: continuous aggregate "public.conditions_summary_daily_new" does not require any migration \set ON_ERROR_STOP 1 -- older continuous aggregate to be migrated CREATE MATERIALIZED VIEW conditions_summary_daily @@ -120,7 +115,7 @@ FROM conditions GROUP BY bucket; -psql:include/cagg_migrate_common.sql:117: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +psql:include/cagg_migrate_common.sql:110: NOTICE: refreshing continuous aggregate "conditions_summary_daily" SELECT ca.raw_hypertable_id AS "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", @@ -138,7 +133,7 @@ WHERE \set ON_ERROR_STOP 0 -- should fail because the new cagg with suffix '_new' already exists CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:136: ERROR: continuous aggregate "public.conditions_summary_daily_new" already exists +psql:include/cagg_migrate_common.sql:129: ERROR: continuous aggregate "public.conditions_summary_daily_new" already exists \set ON_ERROR_STOP 1 -- remove the new cagg to execute the migration DROP MATERIALIZED VIEW conditions_summary_daily_new; @@ -182,8 +177,8 @@ SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalo -- should resume the execution CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:156: WARNING: resuming the migration of the continuous aggregate "public.conditions_summary_daily" -psql:include/cagg_migrate_common.sql:156: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:149: WARNING: resuming the migration of the continuous aggregate "public.conditions_summary_daily" +psql:include/cagg_migrate_common.sql:149: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; mat_hypertable_id | step_id | status | type | config -------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------- @@ -210,9 +205,9 @@ SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalo \set ON_ERROR_STOP 0 -- should error because plan already exists CALL _timescaledb_internal.cagg_migrate_create_plan(:'CAGG_DATA', 'conditions_summary_daily_new'); -psql:include/cagg_migrate_common.sql:161: ERROR: plan already exists for materialized hypertable 3 +psql:include/cagg_migrate_common.sql:154: ERROR: plan already exists for materialized hypertable 3 CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:162: ERROR: plan already exists for continuous aggregate public.conditions_summary_daily +psql:include/cagg_migrate_common.sql:155: ERROR: plan already exists for continuous aggregate public.conditions_summary_daily \set ON_ERROR_STOP 1 -- policies for test ALTER MATERIALIZED VIEW conditions_summary_daily SET (timescaledb.compress=true); @@ -254,11 +249,11 @@ AND job_id >= 1000; -- execute the migration DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:185: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:178: NOTICE: drop cascades to 10 other objects TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:186: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:179: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:187: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:180: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date SELECT ca.raw_hypertable_id AS "NEW_RAW_HYPERTABLE_ID", h.schema_name AS "NEW_MAT_SCHEMA_NAME", @@ -389,9 +384,9 @@ JOIN _timescaledb_catalog.continuous_agg ON mat_hypertable_id = hypertable_id ORDER BY bgw_job.id; -- test migration overriding the new cagg and keeping the old DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:235: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:228: NOTICE: drop cascades to 10 other objects TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:236: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:229: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" -- check policies before the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone @@ -402,7 +397,7 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d (3 rows) CALL cagg_migrate('conditions_summary_daily', override => TRUE); -psql:include/cagg_migrate_common.sql:239: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:232: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -- cagg with the new format because it was overriden \d+ conditions_summary_daily View "public.conditions_summary_daily" @@ -463,7 +458,7 @@ UNION ALL \set ON_ERROR_STOP 0 -- should fail because the cagg was overriden SELECT * FROM conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:246: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +psql:include/cagg_migrate_common.sql:239: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 \set ON_ERROR_STOP 1 -- check policies after the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -491,9 +486,9 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d -- test migration overriding the new cagg and removing the old TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:256: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:249: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily; -psql:include/cagg_migrate_common.sql:257: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:250: NOTICE: drop cascades to 10 other objects ALTER MATERIALIZED VIEW conditions_summary_daily_old RENAME TO conditions_summary_daily; -- check policies before the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -505,11 +500,11 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d (3 rows) CALL cagg_migrate('conditions_summary_daily', override => TRUE, drop_old => TRUE); -psql:include/cagg_migrate_common.sql:261: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -psql:include/cagg_migrate_common.sql:261: NOTICE: drop cascades to 10 other objects -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1002 not found, skipping -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1001 not found, skipping -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1000 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:254: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1002 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1001 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1000 not found, skipping -- cagg with the new format because it was overriden \d+ conditions_summary_daily View "public.conditions_summary_daily" @@ -541,10 +536,10 @@ UNION ALL \set ON_ERROR_STOP 0 -- should fail because the cagg was overriden SELECT * FROM conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:266: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +psql:include/cagg_migrate_common.sql:259: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 -- should fail because the old cagg was removed SELECT * FROM conditions_summary_daily_old; -psql:include/cagg_migrate_common.sql:268: ERROR: relation "conditions_summary_daily_old" does not exist at character 15 +psql:include/cagg_migrate_common.sql:261: ERROR: relation "conditions_summary_daily_old" does not exist at character 15 \set ON_ERROR_STOP 1 -- check policies after the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -569,9 +564,9 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d -- permissions test TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:278: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:271: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily; -psql:include/cagg_migrate_common.sql:279: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:272: NOTICE: drop cascades to 10 other objects GRANT ALL ON TABLE conditions TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE MATERIALIZED VIEW conditions_summary_daily @@ -590,11 +585,11 @@ FROM conditions GROUP BY bucket; -psql:include/cagg_migrate_common.sql:298: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +psql:include/cagg_migrate_common.sql:291: NOTICE: refreshing continuous aggregate "conditions_summary_daily" \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan' catalog table CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:302: ERROR: permission denied for table continuous_agg_migrate_plan +psql:include/cagg_migrate_common.sql:295: ERROR: permission denied for table continuous_agg_migrate_plan \set ON_ERROR_STOP 1 RESET ROLE; GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan TO :ROLE_DEFAULT_PERM_USER; @@ -602,7 +597,7 @@ SET ROLE :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step' catalog table CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:312: ERROR: permission denied for table continuous_agg_migrate_plan_step +psql:include/cagg_migrate_common.sql:305: ERROR: permission denied for table continuous_agg_migrate_plan_step \set ON_ERROR_STOP 1 RESET ROLE; GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step TO :ROLE_DEFAULT_PERM_USER; @@ -610,14 +605,14 @@ SET ROLE :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step_step_id_seq' catalog sequence CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:322: ERROR: permission denied for sequence continuous_agg_migrate_plan_step_step_id_seq +psql:include/cagg_migrate_common.sql:315: ERROR: permission denied for sequence continuous_agg_migrate_plan_step_step_id_seq \set ON_ERROR_STOP 1 RESET ROLE; GRANT USAGE ON SEQUENCE _timescaledb_catalog.continuous_agg_migrate_plan_step_step_id_seq TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; -- all necessary permissions granted CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:331: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:324: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -- check migrated data. should return 0 (zero) rows SELECT * FROM conditions_summary_daily EXCEPT @@ -657,14 +652,14 @@ RESET ROLE; -- execute transaction control statements. Transaction control statements are only -- allowed if CALL is executed in its own transaction.` TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:348: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:341: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:349: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:342: NOTICE: drop cascades to 10 other objects \set ON_ERROR_STOP 0 BEGIN; -- should fail with `invalid transaction termination` CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:354: ERROR: invalid transaction termination +psql:include/cagg_migrate_common.sql:347: ERROR: invalid transaction termination ROLLBACK; \set ON_ERROR_STOP 1 CREATE FUNCTION execute_migration() RETURNS void AS @@ -680,6 +675,6 @@ LANGUAGE plpgsql; BEGIN; -- should fail with `invalid transaction termination` SELECT execute_migration(); -psql:include/cagg_migrate_common.sql:371: ERROR: invalid transaction termination +psql:include/cagg_migrate_common.sql:364: ERROR: invalid transaction termination ROLLBACK; \set ON_ERROR_STOP 1 diff --git a/tsl/test/expected/cagg_migrate_integer_dist_ht.out b/tsl/test/expected/cagg_migrate_integer_dist_ht.out index 03b77f3ac0c..28b3532f336 100644 --- a/tsl/test/expected/cagg_migrate_integer_dist_ht.out +++ b/tsl/test/expected/cagg_migrate_integer_dist_ht.out @@ -38,6 +38,7 @@ FROM ( GRANT USAGE ON FOREIGN SERVER :DATA_NODE_1, :DATA_NODE_2, :DATA_NODE_3 TO PUBLIC; \set IS_DISTRIBUTED TRUE \set IS_TIME_DIMENSION FALSE +\set TIME_DIMENSION_DATATYPE INTEGER \ir include/cagg_migrate_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and @@ -49,13 +50,8 @@ Running distributed hypertable tests \else \echo 'Running local hypertable tests' \endif -\if :IS_TIME_DIMENSION - \set TIME_DATATYPE TIMESTAMPTZ -\else - \set TIME_DATATYPE INTEGER -\endif CREATE TABLE conditions ( - "time" :TIME_DATATYPE NOT NULL, + "time" :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC ); \if :IS_DISTRIBUTED @@ -83,17 +79,16 @@ CREATE TABLE conditions ( 0.25; \else CREATE OR REPLACE FUNCTION integer_now() - RETURNS integer LANGUAGE SQL STABLE AS + RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$; \if :IS_DISTRIBUTED - CALL distributed_exec ( - $DIST$ - CREATE OR REPLACE FUNCTION integer_now() RETURNS integer LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$; - $DIST$ - ); + SELECT + 'CREATE OR REPLACE FUNCTION integer_now() RETURNS '||:'TIME_DIMENSION_DATATYPE'||' LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$;' AS "STMT" + \gset + CALL distributed_exec (:'STMT'); \endif SELECT set_integer_now_func('conditions', 'integer_now'); set_integer_now_func @@ -109,11 +104,11 @@ CREATE TABLE conditions ( \set ON_ERROR_STOP 0 -- should fail relation does not exist CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:70: ERROR: relation "conditions_summary_daily" does not exist at character 19 +psql:include/cagg_migrate_common.sql:63: ERROR: relation "conditions_summary_daily" does not exist at character 19 CREATE TABLE conditions_summary_daily(); -- should fail continuous agg does not exist CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:73: ERROR: continuous aggregate "public.conditions_summary_daily" does not exist +psql:include/cagg_migrate_common.sql:66: ERROR: continuous aggregate "public.conditions_summary_daily" does not exist \set ON_ERROR_STOP 1 DROP TABLE conditions_summary_daily; CREATE MATERIALIZED VIEW conditions_summary_daily_new @@ -136,7 +131,7 @@ WITH NO DATA; \set ON_ERROR_STOP 0 -- should fail because we don't need to migrate finalized caggs CALL cagg_migrate('conditions_summary_daily_new'); -psql:include/cagg_migrate_common.sql:98: ERROR: continuous aggregate "public.conditions_summary_daily_new" does not require any migration +psql:include/cagg_migrate_common.sql:91: ERROR: continuous aggregate "public.conditions_summary_daily_new" does not require any migration \set ON_ERROR_STOP 1 -- older continuous aggregate to be migrated CREATE MATERIALIZED VIEW conditions_summary_daily @@ -155,7 +150,7 @@ FROM conditions GROUP BY bucket; -psql:include/cagg_migrate_common.sql:117: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +psql:include/cagg_migrate_common.sql:110: NOTICE: refreshing continuous aggregate "conditions_summary_daily" SELECT ca.raw_hypertable_id AS "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", @@ -173,7 +168,7 @@ WHERE \set ON_ERROR_STOP 0 -- should fail because the new cagg with suffix '_new' already exists CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:136: ERROR: continuous aggregate "public.conditions_summary_daily_new" already exists +psql:include/cagg_migrate_common.sql:129: ERROR: continuous aggregate "public.conditions_summary_daily_new" already exists \set ON_ERROR_STOP 1 -- remove the new cagg to execute the migration DROP MATERIALIZED VIEW conditions_summary_daily_new; @@ -217,8 +212,8 @@ SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalo -- should resume the execution CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:156: WARNING: resuming the migration of the continuous aggregate "public.conditions_summary_daily" -psql:include/cagg_migrate_common.sql:156: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:149: WARNING: resuming the migration of the continuous aggregate "public.conditions_summary_daily" +psql:include/cagg_migrate_common.sql:149: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; mat_hypertable_id | step_id | status | type | config -------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------- @@ -245,9 +240,9 @@ SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalo \set ON_ERROR_STOP 0 -- should error because plan already exists CALL _timescaledb_internal.cagg_migrate_create_plan(:'CAGG_DATA', 'conditions_summary_daily_new'); -psql:include/cagg_migrate_common.sql:161: ERROR: plan already exists for materialized hypertable 3 +psql:include/cagg_migrate_common.sql:154: ERROR: plan already exists for materialized hypertable 3 CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:162: ERROR: plan already exists for continuous aggregate public.conditions_summary_daily +psql:include/cagg_migrate_common.sql:155: ERROR: plan already exists for continuous aggregate public.conditions_summary_daily \set ON_ERROR_STOP 1 -- policies for test ALTER MATERIALIZED VIEW conditions_summary_daily SET (timescaledb.compress=true); @@ -289,11 +284,11 @@ AND job_id >= 1000; -- execute the migration DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:185: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:178: NOTICE: drop cascades to 10 other objects TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:186: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:179: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:187: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:180: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date SELECT ca.raw_hypertable_id AS "NEW_RAW_HYPERTABLE_ID", h.schema_name AS "NEW_MAT_SCHEMA_NAME", @@ -424,9 +419,9 @@ JOIN _timescaledb_catalog.continuous_agg ON mat_hypertable_id = hypertable_id ORDER BY bgw_job.id; -- test migration overriding the new cagg and keeping the old DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:235: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:228: NOTICE: drop cascades to 10 other objects TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:236: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:229: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" -- check policies before the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone @@ -437,7 +432,7 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d (3 rows) CALL cagg_migrate('conditions_summary_daily', override => TRUE); -psql:include/cagg_migrate_common.sql:239: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:232: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -- cagg with the new format because it was overriden \d+ conditions_summary_daily View "public.conditions_summary_daily" @@ -498,7 +493,7 @@ UNION ALL \set ON_ERROR_STOP 0 -- should fail because the cagg was overriden SELECT * FROM conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:246: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +psql:include/cagg_migrate_common.sql:239: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 \set ON_ERROR_STOP 1 -- check policies after the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -526,9 +521,9 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d -- test migration overriding the new cagg and removing the old TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:256: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:249: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily; -psql:include/cagg_migrate_common.sql:257: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:250: NOTICE: drop cascades to 10 other objects ALTER MATERIALIZED VIEW conditions_summary_daily_old RENAME TO conditions_summary_daily; -- check policies before the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -540,11 +535,11 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d (3 rows) CALL cagg_migrate('conditions_summary_daily', override => TRUE, drop_old => TRUE); -psql:include/cagg_migrate_common.sql:261: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -psql:include/cagg_migrate_common.sql:261: NOTICE: drop cascades to 10 other objects -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1002 not found, skipping -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1001 not found, skipping -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1000 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:254: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1002 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1001 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1000 not found, skipping -- cagg with the new format because it was overriden \d+ conditions_summary_daily View "public.conditions_summary_daily" @@ -576,10 +571,10 @@ UNION ALL \set ON_ERROR_STOP 0 -- should fail because the cagg was overriden SELECT * FROM conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:266: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +psql:include/cagg_migrate_common.sql:259: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 -- should fail because the old cagg was removed SELECT * FROM conditions_summary_daily_old; -psql:include/cagg_migrate_common.sql:268: ERROR: relation "conditions_summary_daily_old" does not exist at character 15 +psql:include/cagg_migrate_common.sql:261: ERROR: relation "conditions_summary_daily_old" does not exist at character 15 \set ON_ERROR_STOP 1 -- check policies after the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -604,9 +599,9 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d -- permissions test TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:278: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:271: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily; -psql:include/cagg_migrate_common.sql:279: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:272: NOTICE: drop cascades to 10 other objects GRANT ALL ON TABLE conditions TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE MATERIALIZED VIEW conditions_summary_daily @@ -625,11 +620,11 @@ FROM conditions GROUP BY bucket; -psql:include/cagg_migrate_common.sql:298: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +psql:include/cagg_migrate_common.sql:291: NOTICE: refreshing continuous aggregate "conditions_summary_daily" \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan' catalog table CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:302: ERROR: permission denied for table continuous_agg_migrate_plan +psql:include/cagg_migrate_common.sql:295: ERROR: permission denied for table continuous_agg_migrate_plan \set ON_ERROR_STOP 1 RESET ROLE; GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan TO :ROLE_DEFAULT_PERM_USER; @@ -637,7 +632,7 @@ SET ROLE :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step' catalog table CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:312: ERROR: permission denied for table continuous_agg_migrate_plan_step +psql:include/cagg_migrate_common.sql:305: ERROR: permission denied for table continuous_agg_migrate_plan_step \set ON_ERROR_STOP 1 RESET ROLE; GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step TO :ROLE_DEFAULT_PERM_USER; @@ -645,14 +640,14 @@ SET ROLE :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step_step_id_seq' catalog sequence CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:322: ERROR: permission denied for sequence continuous_agg_migrate_plan_step_step_id_seq +psql:include/cagg_migrate_common.sql:315: ERROR: permission denied for sequence continuous_agg_migrate_plan_step_step_id_seq \set ON_ERROR_STOP 1 RESET ROLE; GRANT USAGE ON SEQUENCE _timescaledb_catalog.continuous_agg_migrate_plan_step_step_id_seq TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; -- all necessary permissions granted CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:331: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:324: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -- check migrated data. should return 0 (zero) rows SELECT * FROM conditions_summary_daily EXCEPT @@ -692,14 +687,14 @@ RESET ROLE; -- execute transaction control statements. Transaction control statements are only -- allowed if CALL is executed in its own transaction.` TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:348: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:341: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:349: NOTICE: drop cascades to 10 other objects +psql:include/cagg_migrate_common.sql:342: NOTICE: drop cascades to 10 other objects \set ON_ERROR_STOP 0 BEGIN; -- should fail with `invalid transaction termination` CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:354: ERROR: invalid transaction termination +psql:include/cagg_migrate_common.sql:347: ERROR: invalid transaction termination ROLLBACK; \set ON_ERROR_STOP 1 CREATE FUNCTION execute_migration() RETURNS void AS @@ -715,7 +710,7 @@ LANGUAGE plpgsql; BEGIN; -- should fail with `invalid transaction termination` SELECT execute_migration(); -psql:include/cagg_migrate_common.sql:371: ERROR: invalid transaction termination +psql:include/cagg_migrate_common.sql:364: ERROR: invalid transaction termination ROLLBACK; \set ON_ERROR_STOP 1 -- cleanup diff --git a/tsl/test/expected/cagg_migrate_timestamp.out b/tsl/test/expected/cagg_migrate_timestamp.out index 06ddb6d2512..acf03ae0a94 100644 --- a/tsl/test/expected/cagg_migrate_timestamp.out +++ b/tsl/test/expected/cagg_migrate_timestamp.out @@ -3,6 +3,7 @@ -- LICENSE-TIMESCALE for a copy of the license. \set IS_DISTRIBUTED FALSE \set IS_TIME_DIMENSION TRUE +\set TIME_DIMENSION_DATATYPE TIMESTAMP \ir include/cagg_migrate_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and @@ -14,13 +15,8 @@ \echo 'Running local hypertable tests' Running local hypertable tests \endif -\if :IS_TIME_DIMENSION - \set TIME_DATATYPE TIMESTAMPTZ -\else - \set TIME_DATATYPE INTEGER -\endif CREATE TABLE conditions ( - "time" :TIME_DATATYPE NOT NULL, + "time" :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC ); \if :IS_DISTRIBUTED @@ -32,6 +28,7 @@ CREATE TABLE conditions ( \else \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); +psql:include/cagg_migrate_common.sql:26: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices table_name ------------ conditions @@ -48,17 +45,16 @@ CREATE TABLE conditions ( 0.25; \else CREATE OR REPLACE FUNCTION integer_now() - RETURNS integer LANGUAGE SQL STABLE AS + RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$; \if :IS_DISTRIBUTED - CALL distributed_exec ( - $DIST$ - CREATE OR REPLACE FUNCTION integer_now() RETURNS integer LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$; - $DIST$ - ); + SELECT + 'CREATE OR REPLACE FUNCTION integer_now() RETURNS '||:'TIME_DIMENSION_DATATYPE'||' LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$;' AS "STMT" + \gset + CALL distributed_exec (:'STMT'); \endif SELECT set_integer_now_func('conditions', 'integer_now'); INSERT INTO conditions ("time", temperature) @@ -69,11 +65,11 @@ CREATE TABLE conditions ( \set ON_ERROR_STOP 0 -- should fail relation does not exist CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:70: ERROR: relation "conditions_summary_daily" does not exist at character 19 +psql:include/cagg_migrate_common.sql:63: ERROR: relation "conditions_summary_daily" does not exist at character 19 CREATE TABLE conditions_summary_daily(); -- should fail continuous agg does not exist CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:73: ERROR: continuous aggregate "public.conditions_summary_daily" does not exist +psql:include/cagg_migrate_common.sql:66: ERROR: continuous aggregate "public.conditions_summary_daily" does not exist \set ON_ERROR_STOP 1 DROP TABLE conditions_summary_daily; CREATE MATERIALIZED VIEW conditions_summary_daily_new @@ -96,7 +92,7 @@ WITH NO DATA; \set ON_ERROR_STOP 0 -- should fail because we don't need to migrate finalized caggs CALL cagg_migrate('conditions_summary_daily_new'); -psql:include/cagg_migrate_common.sql:98: ERROR: continuous aggregate "public.conditions_summary_daily_new" does not require any migration +psql:include/cagg_migrate_common.sql:91: ERROR: continuous aggregate "public.conditions_summary_daily_new" does not require any migration \set ON_ERROR_STOP 1 -- older continuous aggregate to be migrated CREATE MATERIALIZED VIEW conditions_summary_daily @@ -115,7 +111,7 @@ FROM conditions GROUP BY bucket; -psql:include/cagg_migrate_common.sql:117: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +psql:include/cagg_migrate_common.sql:110: NOTICE: refreshing continuous aggregate "conditions_summary_daily" SELECT ca.raw_hypertable_id AS "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", @@ -133,7 +129,7 @@ WHERE \set ON_ERROR_STOP 0 -- should fail because the new cagg with suffix '_new' already exists CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:136: ERROR: continuous aggregate "public.conditions_summary_daily_new" already exists +psql:include/cagg_migrate_common.sql:129: ERROR: continuous aggregate "public.conditions_summary_daily_new" already exists \set ON_ERROR_STOP 1 -- remove the new cagg to execute the migration DROP MATERIALIZED VIEW conditions_summary_daily_new; @@ -153,18 +149,18 @@ SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg_migrate_plan; (1 row) SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; - mat_hypertable_id | step_id | status | type | config --------------------+---------+-------------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + mat_hypertable_id | step_id | status | type | config +-------------------+---------+-------------+------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} 3 | 2 | NOT STARTED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} 3 | 3 | NOT STARTED | DISABLE POLICIES | {"policies": null} - 3 | 4 | NOT STARTED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} - 3 | 5 | NOT STARTED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 6 | NOT STARTED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 7 | NOT STARTED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 8 | NOT STARTED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 9 | NOT STARTED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 10 | NOT STARTED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 4 | NOT STARTED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp without time zone"} + 3 | 5 | NOT STARTED | COPY DATA | {"end_ts": "Fri Mar 11 00:00:00 2022", "start_ts": "Fri Dec 31 00:00:00 2021", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 6 | NOT STARTED | COPY DATA | {"end_ts": "Fri May 20 00:00:00 2022", "start_ts": "Fri Mar 11 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 7 | NOT STARTED | COPY DATA | {"end_ts": "Fri Jul 29 00:00:00 2022", "start_ts": "Fri May 20 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 8 | NOT STARTED | COPY DATA | {"end_ts": "Fri Oct 07 00:00:00 2022", "start_ts": "Fri Jul 29 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 9 | NOT STARTED | COPY DATA | {"end_ts": "Fri Dec 16 00:00:00 2022", "start_ts": "Fri Oct 07 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 10 | NOT STARTED | COPY DATA | {"end_ts": "Fri Feb 24 00:00:00 2023", "start_ts": "Fri Dec 16 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} 3 | 11 | NOT STARTED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} 3 | 12 | NOT STARTED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} 3 | 13 | NOT STARTED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} @@ -173,21 +169,21 @@ SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalo -- should resume the execution CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:156: WARNING: resuming the migration of the continuous aggregate "public.conditions_summary_daily" -psql:include/cagg_migrate_common.sql:156: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:149: WARNING: resuming the migration of the continuous aggregate "public.conditions_summary_daily" +psql:include/cagg_migrate_common.sql:149: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; - mat_hypertable_id | step_id | status | type | config --------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} 3 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} 3 | 3 | FINISHED | DISABLE POLICIES | {"policies": null} - 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} - 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp without time zone"} + 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 00:00:00 2022", "start_ts": "Fri Dec 31 00:00:00 2021", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 00:00:00 2022", "start_ts": "Fri Mar 11 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 00:00:00 2022", "start_ts": "Fri May 20 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 00:00:00 2022", "start_ts": "Fri Jul 29 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 00:00:00 2022", "start_ts": "Fri Oct 07 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 00:00:00 2023", "start_ts": "Fri Dec 16 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} 3 | 11 | FINISHED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} 3 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} 3 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} @@ -197,9 +193,9 @@ SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalo \set ON_ERROR_STOP 0 -- should error because plan already exists CALL _timescaledb_internal.cagg_migrate_create_plan(:'CAGG_DATA', 'conditions_summary_daily_new'); -psql:include/cagg_migrate_common.sql:161: ERROR: plan already exists for materialized hypertable 3 +psql:include/cagg_migrate_common.sql:154: ERROR: plan already exists for materialized hypertable 3 CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:162: ERROR: plan already exists for continuous aggregate public.conditions_summary_daily +psql:include/cagg_migrate_common.sql:155: ERROR: plan already exists for continuous aggregate public.conditions_summary_daily \set ON_ERROR_STOP 1 -- policies for test ALTER MATERIALIZED VIEW conditions_summary_daily SET (timescaledb.compress=true); @@ -241,11 +237,11 @@ AND job_id >= 1000; -- execute the migration DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:185: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:178: NOTICE: drop cascades to 6 other objects TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:186: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:179: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:187: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:180: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date SELECT ca.raw_hypertable_id AS "NEW_RAW_HYPERTABLE_ID", h.schema_name AS "NEW_MAT_SCHEMA_NAME", @@ -261,14 +257,14 @@ WHERE user_view_name = 'conditions_summary_daily_new' \gset \d+ conditions_summary_daily_new - View "public.conditions_summary_daily_new" - Column | Type | Collation | Nullable | Default | Storage | Description ---------+--------------------------+-----------+----------+---------+---------+------------- - bucket | timestamp with time zone | | | | plain | - min | numeric | | | | main | - max | numeric | | | | main | - avg | numeric | | | | main | - sum | numeric | | | | main | + View "public.conditions_summary_daily_new" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+-----------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp without time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | View definition: SELECT _materialized_hypertable_6.bucket, _materialized_hypertable_6.min, @@ -276,7 +272,7 @@ View definition: _materialized_hypertable_6.avg, _materialized_hypertable_6.sum FROM _timescaledb_internal._materialized_hypertable_6 - WHERE _materialized_hypertable_6.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp with time zone) + WHERE _materialized_hypertable_6.bucket < COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, min(conditions.temperature) AS min, @@ -284,7 +280,7 @@ UNION ALL avg(conditions.temperature) AS avg, sum(conditions.temperature) AS sum FROM conditions - WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp with time zone) + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); SELECT * @@ -300,18 +296,18 @@ AND job_id >= 1000; (3 rows) SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; - mat_hypertable_id | step_id | status | type | config --------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} 3 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} 3 | 3 | FINISHED | DISABLE POLICIES | {"policies": [1002, 1000]} - 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} - 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp without time zone"} + 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 00:00:00 2022", "start_ts": "Fri Dec 31 00:00:00 2021", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 00:00:00 2022", "start_ts": "Fri Mar 11 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 00:00:00 2022", "start_ts": "Fri May 20 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 00:00:00 2022", "start_ts": "Fri Jul 29 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 00:00:00 2022", "start_ts": "Fri Oct 07 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 00:00:00 2023", "start_ts": "Fri Dec 16 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} 3 | 11 | FINISHED | COPY POLICIES | {"policies": [1002, 1001, 1000], "cagg_name_new": "conditions_summary_daily_new"} 3 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} 3 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} @@ -364,9 +360,9 @@ JOIN _timescaledb_catalog.continuous_agg ON mat_hypertable_id = hypertable_id ORDER BY bgw_job.id; -- test migration overriding the new cagg and keeping the old DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:235: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:228: NOTICE: drop cascades to 6 other objects TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:236: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:229: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" -- check policies before the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone @@ -377,17 +373,17 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d (3 rows) CALL cagg_migrate('conditions_summary_daily', override => TRUE); -psql:include/cagg_migrate_common.sql:239: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:232: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -- cagg with the new format because it was overriden \d+ conditions_summary_daily - View "public.conditions_summary_daily" - Column | Type | Collation | Nullable | Default | Storage | Description ---------+--------------------------+-----------+----------+---------+---------+------------- - bucket | timestamp with time zone | | | | plain | - min | numeric | | | | main | - max | numeric | | | | main | - avg | numeric | | | | main | - sum | numeric | | | | main | + View "public.conditions_summary_daily" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+-----------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp without time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | View definition: SELECT _materialized_hypertable_8.bucket, _materialized_hypertable_8.min, @@ -395,7 +391,7 @@ View definition: _materialized_hypertable_8.avg, _materialized_hypertable_8.sum FROM _timescaledb_internal._materialized_hypertable_8 - WHERE _materialized_hypertable_8.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp with time zone) + WHERE _materialized_hypertable_8.bucket < COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, min(conditions.temperature) AS min, @@ -403,19 +399,19 @@ UNION ALL avg(conditions.temperature) AS avg, sum(conditions.temperature) AS sum FROM conditions - WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp with time zone) + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); -- cagg with the old format because it was overriden \d+ conditions_summary_daily_old - View "public.conditions_summary_daily_old" - Column | Type | Collation | Nullable | Default | Storage | Description ---------+--------------------------+-----------+----------+---------+---------+------------- - bucket | timestamp with time zone | | | | plain | - min | numeric | | | | main | - max | numeric | | | | main | - avg | numeric | | | | main | - sum | numeric | | | | main | + View "public.conditions_summary_daily_old" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+-----------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp without time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | View definition: SELECT _materialized_hypertable_3.bucket, _timescaledb_internal.finalize_agg('pg_catalog.min(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_2_2, NULL::numeric) AS min, @@ -423,7 +419,7 @@ View definition: _timescaledb_internal.finalize_agg('pg_catalog.avg(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_4_4, NULL::numeric) AS avg, _timescaledb_internal.finalize_agg('pg_catalog.sum(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_5_5, NULL::numeric) AS sum FROM _timescaledb_internal._materialized_hypertable_3 - WHERE _materialized_hypertable_3.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp with time zone) + WHERE _materialized_hypertable_3.bucket < COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp without time zone) GROUP BY _materialized_hypertable_3.bucket UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, @@ -432,13 +428,13 @@ UNION ALL avg(conditions.temperature) AS avg, sum(conditions.temperature) AS sum FROM conditions - WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp with time zone) + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); \set ON_ERROR_STOP 0 -- should fail because the cagg was overriden SELECT * FROM conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:246: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +psql:include/cagg_migrate_common.sql:239: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 \set ON_ERROR_STOP 1 -- check policies after the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -466,9 +462,9 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d -- test migration overriding the new cagg and removing the old TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:256: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:249: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily; -psql:include/cagg_migrate_common.sql:257: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:250: NOTICE: drop cascades to 6 other objects ALTER MATERIALIZED VIEW conditions_summary_daily_old RENAME TO conditions_summary_daily; -- check policies before the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -480,21 +476,21 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d (3 rows) CALL cagg_migrate('conditions_summary_daily', override => TRUE, drop_old => TRUE); -psql:include/cagg_migrate_common.sql:261: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -psql:include/cagg_migrate_common.sql:261: NOTICE: drop cascades to 6 other objects -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1002 not found, skipping -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1001 not found, skipping -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1000 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:254: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1002 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1001 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1000 not found, skipping -- cagg with the new format because it was overriden \d+ conditions_summary_daily - View "public.conditions_summary_daily" - Column | Type | Collation | Nullable | Default | Storage | Description ---------+--------------------------+-----------+----------+---------+---------+------------- - bucket | timestamp with time zone | | | | plain | - min | numeric | | | | main | - max | numeric | | | | main | - avg | numeric | | | | main | - sum | numeric | | | | main | + View "public.conditions_summary_daily" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+-----------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp without time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | View definition: SELECT _materialized_hypertable_10.bucket, _materialized_hypertable_10.min, @@ -502,7 +498,7 @@ View definition: _materialized_hypertable_10.avg, _materialized_hypertable_10.sum FROM _timescaledb_internal._materialized_hypertable_10 - WHERE _materialized_hypertable_10.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp with time zone) + WHERE _materialized_hypertable_10.bucket < COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, min(conditions.temperature) AS min, @@ -510,16 +506,16 @@ UNION ALL avg(conditions.temperature) AS avg, sum(conditions.temperature) AS sum FROM conditions - WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp with time zone) + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); \set ON_ERROR_STOP 0 -- should fail because the cagg was overriden SELECT * FROM conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:266: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +psql:include/cagg_migrate_common.sql:259: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 -- should fail because the old cagg was removed SELECT * FROM conditions_summary_daily_old; -psql:include/cagg_migrate_common.sql:268: ERROR: relation "conditions_summary_daily_old" does not exist at character 15 +psql:include/cagg_migrate_common.sql:261: ERROR: relation "conditions_summary_daily_old" does not exist at character 15 \set ON_ERROR_STOP 1 -- check policies after the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -544,9 +540,9 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d -- permissions test TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:278: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:271: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily; -psql:include/cagg_migrate_common.sql:279: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:272: NOTICE: drop cascades to 6 other objects GRANT ALL ON TABLE conditions TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE MATERIALIZED VIEW conditions_summary_daily @@ -565,11 +561,11 @@ FROM conditions GROUP BY bucket; -psql:include/cagg_migrate_common.sql:298: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +psql:include/cagg_migrate_common.sql:291: NOTICE: refreshing continuous aggregate "conditions_summary_daily" \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan' catalog table CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:302: ERROR: permission denied for table continuous_agg_migrate_plan +psql:include/cagg_migrate_common.sql:295: ERROR: permission denied for table continuous_agg_migrate_plan \set ON_ERROR_STOP 1 RESET ROLE; GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan TO :ROLE_DEFAULT_PERM_USER; @@ -577,7 +573,7 @@ SET ROLE :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step' catalog table CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:312: ERROR: permission denied for table continuous_agg_migrate_plan_step +psql:include/cagg_migrate_common.sql:305: ERROR: permission denied for table continuous_agg_migrate_plan_step \set ON_ERROR_STOP 1 RESET ROLE; GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step TO :ROLE_DEFAULT_PERM_USER; @@ -585,14 +581,14 @@ SET ROLE :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step_step_id_seq' catalog sequence CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:322: ERROR: permission denied for sequence continuous_agg_migrate_plan_step_step_id_seq +psql:include/cagg_migrate_common.sql:315: ERROR: permission denied for sequence continuous_agg_migrate_plan_step_step_id_seq \set ON_ERROR_STOP 1 RESET ROLE; GRANT USAGE ON SEQUENCE _timescaledb_catalog.continuous_agg_migrate_plan_step_step_id_seq TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; -- all necessary permissions granted CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:331: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:324: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -- check migrated data. should return 0 (zero) rows SELECT * FROM conditions_summary_daily EXCEPT @@ -602,18 +598,18 @@ SELECT * FROM conditions_summary_daily_new; (0 rows) SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; - mat_hypertable_id | step_id | status | type | config --------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 12 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} 12 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} 12 | 3 | FINISHED | DISABLE POLICIES | {"policies": null} - 12 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} - 12 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 12 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 12 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 12 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 12 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 12 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp without time zone"} + 12 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 00:00:00 2022", "start_ts": "Fri Dec 31 00:00:00 2021", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 12 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 00:00:00 2022", "start_ts": "Fri Mar 11 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 12 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 00:00:00 2022", "start_ts": "Fri May 20 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 12 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 00:00:00 2022", "start_ts": "Fri Jul 29 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 12 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 00:00:00 2022", "start_ts": "Fri Oct 07 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 12 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 00:00:00 2023", "start_ts": "Fri Dec 16 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} 12 | 11 | FINISHED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} 12 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} 12 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} @@ -628,14 +624,14 @@ RESET ROLE; -- execute transaction control statements. Transaction control statements are only -- allowed if CALL is executed in its own transaction.` TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:348: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:341: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:349: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:342: NOTICE: drop cascades to 6 other objects \set ON_ERROR_STOP 0 BEGIN; -- should fail with `invalid transaction termination` CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:354: ERROR: invalid transaction termination +psql:include/cagg_migrate_common.sql:347: ERROR: invalid transaction termination ROLLBACK; \set ON_ERROR_STOP 1 CREATE FUNCTION execute_migration() RETURNS void AS @@ -651,6 +647,6 @@ LANGUAGE plpgsql; BEGIN; -- should fail with `invalid transaction termination` SELECT execute_migration(); -psql:include/cagg_migrate_common.sql:371: ERROR: invalid transaction termination +psql:include/cagg_migrate_common.sql:364: ERROR: invalid transaction termination ROLLBACK; \set ON_ERROR_STOP 1 diff --git a/tsl/test/expected/cagg_migrate_timestamp_dist_ht.out b/tsl/test/expected/cagg_migrate_timestamp_dist_ht.out index f0f792610bc..8558ca16929 100644 --- a/tsl/test/expected/cagg_migrate_timestamp_dist_ht.out +++ b/tsl/test/expected/cagg_migrate_timestamp_dist_ht.out @@ -38,6 +38,7 @@ FROM ( GRANT USAGE ON FOREIGN SERVER :DATA_NODE_1, :DATA_NODE_2, :DATA_NODE_3 TO PUBLIC; \set IS_DISTRIBUTED TRUE \set IS_TIME_DIMENSION TRUE +\set TIME_DIMENSION_DATATYPE TIMESTAMP \ir include/cagg_migrate_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and @@ -49,18 +50,14 @@ Running distributed hypertable tests \else \echo 'Running local hypertable tests' \endif -\if :IS_TIME_DIMENSION - \set TIME_DATATYPE TIMESTAMPTZ -\else - \set TIME_DATATYPE INTEGER -\endif CREATE TABLE conditions ( - "time" :TIME_DATATYPE NOT NULL, + "time" :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC ); \if :IS_DISTRIBUTED \if :IS_TIME_DIMENSION SELECT table_name FROM create_distributed_hypertable('conditions', 'time', replication_factor => 2); +psql:include/cagg_migrate_common.sql:20: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices table_name ------------ conditions @@ -83,17 +80,16 @@ CREATE TABLE conditions ( 0.25; \else CREATE OR REPLACE FUNCTION integer_now() - RETURNS integer LANGUAGE SQL STABLE AS + RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$; \if :IS_DISTRIBUTED - CALL distributed_exec ( - $DIST$ - CREATE OR REPLACE FUNCTION integer_now() RETURNS integer LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$; - $DIST$ - ); + SELECT + 'CREATE OR REPLACE FUNCTION integer_now() RETURNS '||:'TIME_DIMENSION_DATATYPE'||' LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$;' AS "STMT" + \gset + CALL distributed_exec (:'STMT'); \endif SELECT set_integer_now_func('conditions', 'integer_now'); INSERT INTO conditions ("time", temperature) @@ -104,11 +100,11 @@ CREATE TABLE conditions ( \set ON_ERROR_STOP 0 -- should fail relation does not exist CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:70: ERROR: relation "conditions_summary_daily" does not exist at character 19 +psql:include/cagg_migrate_common.sql:63: ERROR: relation "conditions_summary_daily" does not exist at character 19 CREATE TABLE conditions_summary_daily(); -- should fail continuous agg does not exist CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:73: ERROR: continuous aggregate "public.conditions_summary_daily" does not exist +psql:include/cagg_migrate_common.sql:66: ERROR: continuous aggregate "public.conditions_summary_daily" does not exist \set ON_ERROR_STOP 1 DROP TABLE conditions_summary_daily; CREATE MATERIALIZED VIEW conditions_summary_daily_new @@ -131,7 +127,7 @@ WITH NO DATA; \set ON_ERROR_STOP 0 -- should fail because we don't need to migrate finalized caggs CALL cagg_migrate('conditions_summary_daily_new'); -psql:include/cagg_migrate_common.sql:98: ERROR: continuous aggregate "public.conditions_summary_daily_new" does not require any migration +psql:include/cagg_migrate_common.sql:91: ERROR: continuous aggregate "public.conditions_summary_daily_new" does not require any migration \set ON_ERROR_STOP 1 -- older continuous aggregate to be migrated CREATE MATERIALIZED VIEW conditions_summary_daily @@ -150,7 +146,7 @@ FROM conditions GROUP BY bucket; -psql:include/cagg_migrate_common.sql:117: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +psql:include/cagg_migrate_common.sql:110: NOTICE: refreshing continuous aggregate "conditions_summary_daily" SELECT ca.raw_hypertable_id AS "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", @@ -168,7 +164,7 @@ WHERE \set ON_ERROR_STOP 0 -- should fail because the new cagg with suffix '_new' already exists CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:136: ERROR: continuous aggregate "public.conditions_summary_daily_new" already exists +psql:include/cagg_migrate_common.sql:129: ERROR: continuous aggregate "public.conditions_summary_daily_new" already exists \set ON_ERROR_STOP 1 -- remove the new cagg to execute the migration DROP MATERIALIZED VIEW conditions_summary_daily_new; @@ -188,18 +184,18 @@ SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg_migrate_plan; (1 row) SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; - mat_hypertable_id | step_id | status | type | config --------------------+---------+-------------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + mat_hypertable_id | step_id | status | type | config +-------------------+---------+-------------+------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} 3 | 2 | NOT STARTED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} 3 | 3 | NOT STARTED | DISABLE POLICIES | {"policies": null} - 3 | 4 | NOT STARTED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} - 3 | 5 | NOT STARTED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 6 | NOT STARTED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 7 | NOT STARTED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 8 | NOT STARTED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 9 | NOT STARTED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 10 | NOT STARTED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 4 | NOT STARTED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp without time zone"} + 3 | 5 | NOT STARTED | COPY DATA | {"end_ts": "Fri Mar 11 00:00:00 2022", "start_ts": "Fri Dec 31 00:00:00 2021", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 6 | NOT STARTED | COPY DATA | {"end_ts": "Fri May 20 00:00:00 2022", "start_ts": "Fri Mar 11 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 7 | NOT STARTED | COPY DATA | {"end_ts": "Fri Jul 29 00:00:00 2022", "start_ts": "Fri May 20 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 8 | NOT STARTED | COPY DATA | {"end_ts": "Fri Oct 07 00:00:00 2022", "start_ts": "Fri Jul 29 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 9 | NOT STARTED | COPY DATA | {"end_ts": "Fri Dec 16 00:00:00 2022", "start_ts": "Fri Oct 07 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 10 | NOT STARTED | COPY DATA | {"end_ts": "Fri Feb 24 00:00:00 2023", "start_ts": "Fri Dec 16 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} 3 | 11 | NOT STARTED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} 3 | 12 | NOT STARTED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} 3 | 13 | NOT STARTED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} @@ -208,21 +204,21 @@ SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalo -- should resume the execution CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:156: WARNING: resuming the migration of the continuous aggregate "public.conditions_summary_daily" -psql:include/cagg_migrate_common.sql:156: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:149: WARNING: resuming the migration of the continuous aggregate "public.conditions_summary_daily" +psql:include/cagg_migrate_common.sql:149: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; - mat_hypertable_id | step_id | status | type | config --------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} 3 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} 3 | 3 | FINISHED | DISABLE POLICIES | {"policies": null} - 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} - 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp without time zone"} + 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 00:00:00 2022", "start_ts": "Fri Dec 31 00:00:00 2021", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 00:00:00 2022", "start_ts": "Fri Mar 11 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 00:00:00 2022", "start_ts": "Fri May 20 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 00:00:00 2022", "start_ts": "Fri Jul 29 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 00:00:00 2022", "start_ts": "Fri Oct 07 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 00:00:00 2023", "start_ts": "Fri Dec 16 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} 3 | 11 | FINISHED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} 3 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} 3 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} @@ -232,9 +228,9 @@ SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalo \set ON_ERROR_STOP 0 -- should error because plan already exists CALL _timescaledb_internal.cagg_migrate_create_plan(:'CAGG_DATA', 'conditions_summary_daily_new'); -psql:include/cagg_migrate_common.sql:161: ERROR: plan already exists for materialized hypertable 3 +psql:include/cagg_migrate_common.sql:154: ERROR: plan already exists for materialized hypertable 3 CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:162: ERROR: plan already exists for continuous aggregate public.conditions_summary_daily +psql:include/cagg_migrate_common.sql:155: ERROR: plan already exists for continuous aggregate public.conditions_summary_daily \set ON_ERROR_STOP 1 -- policies for test ALTER MATERIALIZED VIEW conditions_summary_daily SET (timescaledb.compress=true); @@ -276,11 +272,11 @@ AND job_id >= 1000; -- execute the migration DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:185: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:178: NOTICE: drop cascades to 6 other objects TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:186: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:179: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:187: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:180: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date SELECT ca.raw_hypertable_id AS "NEW_RAW_HYPERTABLE_ID", h.schema_name AS "NEW_MAT_SCHEMA_NAME", @@ -296,14 +292,14 @@ WHERE user_view_name = 'conditions_summary_daily_new' \gset \d+ conditions_summary_daily_new - View "public.conditions_summary_daily_new" - Column | Type | Collation | Nullable | Default | Storage | Description ---------+--------------------------+-----------+----------+---------+---------+------------- - bucket | timestamp with time zone | | | | plain | - min | numeric | | | | main | - max | numeric | | | | main | - avg | numeric | | | | main | - sum | numeric | | | | main | + View "public.conditions_summary_daily_new" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+-----------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp without time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | View definition: SELECT _materialized_hypertable_6.bucket, _materialized_hypertable_6.min, @@ -311,7 +307,7 @@ View definition: _materialized_hypertable_6.avg, _materialized_hypertable_6.sum FROM _timescaledb_internal._materialized_hypertable_6 - WHERE _materialized_hypertable_6.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp with time zone) + WHERE _materialized_hypertable_6.bucket < COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, min(conditions.temperature) AS min, @@ -319,7 +315,7 @@ UNION ALL avg(conditions.temperature) AS avg, sum(conditions.temperature) AS sum FROM conditions - WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp with time zone) + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); SELECT * @@ -335,18 +331,18 @@ AND job_id >= 1000; (3 rows) SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; - mat_hypertable_id | step_id | status | type | config --------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} 3 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} 3 | 3 | FINISHED | DISABLE POLICIES | {"policies": [1002, 1000]} - 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} - 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp without time zone"} + 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 00:00:00 2022", "start_ts": "Fri Dec 31 00:00:00 2021", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 00:00:00 2022", "start_ts": "Fri Mar 11 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 00:00:00 2022", "start_ts": "Fri May 20 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 00:00:00 2022", "start_ts": "Fri Jul 29 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 00:00:00 2022", "start_ts": "Fri Oct 07 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 00:00:00 2023", "start_ts": "Fri Dec 16 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} 3 | 11 | FINISHED | COPY POLICIES | {"policies": [1002, 1001, 1000], "cagg_name_new": "conditions_summary_daily_new"} 3 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} 3 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} @@ -399,9 +395,9 @@ JOIN _timescaledb_catalog.continuous_agg ON mat_hypertable_id = hypertable_id ORDER BY bgw_job.id; -- test migration overriding the new cagg and keeping the old DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:235: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:228: NOTICE: drop cascades to 6 other objects TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:236: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:229: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" -- check policies before the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone @@ -412,17 +408,17 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d (3 rows) CALL cagg_migrate('conditions_summary_daily', override => TRUE); -psql:include/cagg_migrate_common.sql:239: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:232: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -- cagg with the new format because it was overriden \d+ conditions_summary_daily - View "public.conditions_summary_daily" - Column | Type | Collation | Nullable | Default | Storage | Description ---------+--------------------------+-----------+----------+---------+---------+------------- - bucket | timestamp with time zone | | | | plain | - min | numeric | | | | main | - max | numeric | | | | main | - avg | numeric | | | | main | - sum | numeric | | | | main | + View "public.conditions_summary_daily" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+-----------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp without time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | View definition: SELECT _materialized_hypertable_8.bucket, _materialized_hypertable_8.min, @@ -430,7 +426,7 @@ View definition: _materialized_hypertable_8.avg, _materialized_hypertable_8.sum FROM _timescaledb_internal._materialized_hypertable_8 - WHERE _materialized_hypertable_8.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp with time zone) + WHERE _materialized_hypertable_8.bucket < COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, min(conditions.temperature) AS min, @@ -438,19 +434,19 @@ UNION ALL avg(conditions.temperature) AS avg, sum(conditions.temperature) AS sum FROM conditions - WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp with time zone) + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); -- cagg with the old format because it was overriden \d+ conditions_summary_daily_old - View "public.conditions_summary_daily_old" - Column | Type | Collation | Nullable | Default | Storage | Description ---------+--------------------------+-----------+----------+---------+---------+------------- - bucket | timestamp with time zone | | | | plain | - min | numeric | | | | main | - max | numeric | | | | main | - avg | numeric | | | | main | - sum | numeric | | | | main | + View "public.conditions_summary_daily_old" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+-----------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp without time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | View definition: SELECT _materialized_hypertable_3.bucket, _timescaledb_internal.finalize_agg('pg_catalog.min(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_2_2, NULL::numeric) AS min, @@ -458,7 +454,7 @@ View definition: _timescaledb_internal.finalize_agg('pg_catalog.avg(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_4_4, NULL::numeric) AS avg, _timescaledb_internal.finalize_agg('pg_catalog.sum(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_5_5, NULL::numeric) AS sum FROM _timescaledb_internal._materialized_hypertable_3 - WHERE _materialized_hypertable_3.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp with time zone) + WHERE _materialized_hypertable_3.bucket < COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp without time zone) GROUP BY _materialized_hypertable_3.bucket UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, @@ -467,13 +463,13 @@ UNION ALL avg(conditions.temperature) AS avg, sum(conditions.temperature) AS sum FROM conditions - WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp with time zone) + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); \set ON_ERROR_STOP 0 -- should fail because the cagg was overriden SELECT * FROM conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:246: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +psql:include/cagg_migrate_common.sql:239: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 \set ON_ERROR_STOP 1 -- check policies after the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -501,9 +497,9 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d -- test migration overriding the new cagg and removing the old TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:256: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:249: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily; -psql:include/cagg_migrate_common.sql:257: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:250: NOTICE: drop cascades to 6 other objects ALTER MATERIALIZED VIEW conditions_summary_daily_old RENAME TO conditions_summary_daily; -- check policies before the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -515,21 +511,21 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d (3 rows) CALL cagg_migrate('conditions_summary_daily', override => TRUE, drop_old => TRUE); -psql:include/cagg_migrate_common.sql:261: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -psql:include/cagg_migrate_common.sql:261: NOTICE: drop cascades to 6 other objects -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1002 not found, skipping -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1001 not found, skipping -psql:include/cagg_migrate_common.sql:261: NOTICE: job 1000 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:254: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1002 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1001 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1000 not found, skipping -- cagg with the new format because it was overriden \d+ conditions_summary_daily - View "public.conditions_summary_daily" - Column | Type | Collation | Nullable | Default | Storage | Description ---------+--------------------------+-----------+----------+---------+---------+------------- - bucket | timestamp with time zone | | | | plain | - min | numeric | | | | main | - max | numeric | | | | main | - avg | numeric | | | | main | - sum | numeric | | | | main | + View "public.conditions_summary_daily" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+-----------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp without time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | View definition: SELECT _materialized_hypertable_10.bucket, _materialized_hypertable_10.min, @@ -537,7 +533,7 @@ View definition: _materialized_hypertable_10.avg, _materialized_hypertable_10.sum FROM _timescaledb_internal._materialized_hypertable_10 - WHERE _materialized_hypertable_10.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp with time zone) + WHERE _materialized_hypertable_10.bucket < COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, min(conditions.temperature) AS min, @@ -545,16 +541,16 @@ UNION ALL avg(conditions.temperature) AS avg, sum(conditions.temperature) AS sum FROM conditions - WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp with time zone) + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp_without_timezone(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); \set ON_ERROR_STOP 0 -- should fail because the cagg was overriden SELECT * FROM conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:266: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +psql:include/cagg_migrate_common.sql:259: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 -- should fail because the old cagg was removed SELECT * FROM conditions_summary_daily_old; -psql:include/cagg_migrate_common.sql:268: ERROR: relation "conditions_summary_daily_old" does not exist at character 15 +psql:include/cagg_migrate_common.sql:261: ERROR: relation "conditions_summary_daily_old" does not exist at character 15 \set ON_ERROR_STOP 1 -- check policies after the migration SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; @@ -579,9 +575,9 @@ SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_d -- permissions test TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:278: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:271: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily; -psql:include/cagg_migrate_common.sql:279: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:272: NOTICE: drop cascades to 6 other objects GRANT ALL ON TABLE conditions TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE MATERIALIZED VIEW conditions_summary_daily @@ -600,11 +596,11 @@ FROM conditions GROUP BY bucket; -psql:include/cagg_migrate_common.sql:298: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +psql:include/cagg_migrate_common.sql:291: NOTICE: refreshing continuous aggregate "conditions_summary_daily" \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan' catalog table CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:302: ERROR: permission denied for table continuous_agg_migrate_plan +psql:include/cagg_migrate_common.sql:295: ERROR: permission denied for table continuous_agg_migrate_plan \set ON_ERROR_STOP 1 RESET ROLE; GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan TO :ROLE_DEFAULT_PERM_USER; @@ -612,7 +608,7 @@ SET ROLE :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step' catalog table CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:312: ERROR: permission denied for table continuous_agg_migrate_plan_step +psql:include/cagg_migrate_common.sql:305: ERROR: permission denied for table continuous_agg_migrate_plan_step \set ON_ERROR_STOP 1 RESET ROLE; GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step TO :ROLE_DEFAULT_PERM_USER; @@ -620,14 +616,14 @@ SET ROLE :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step_step_id_seq' catalog sequence CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:322: ERROR: permission denied for sequence continuous_agg_migrate_plan_step_step_id_seq +psql:include/cagg_migrate_common.sql:315: ERROR: permission denied for sequence continuous_agg_migrate_plan_step_step_id_seq \set ON_ERROR_STOP 1 RESET ROLE; GRANT USAGE ON SEQUENCE _timescaledb_catalog.continuous_agg_migrate_plan_step_step_id_seq TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; -- all necessary permissions granted CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:331: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:324: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date -- check migrated data. should return 0 (zero) rows SELECT * FROM conditions_summary_daily EXCEPT @@ -637,18 +633,18 @@ SELECT * FROM conditions_summary_daily_new; (0 rows) SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; - mat_hypertable_id | step_id | status | type | config --------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 12 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} 12 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} 12 | 3 | FINISHED | DISABLE POLICIES | {"policies": null} - 12 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} - 12 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 12 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 12 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 12 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 12 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} - 12 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp without time zone"} + 12 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 00:00:00 2022", "start_ts": "Fri Dec 31 00:00:00 2021", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 12 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 00:00:00 2022", "start_ts": "Fri Mar 11 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 12 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 00:00:00 2022", "start_ts": "Fri May 20 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 12 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 00:00:00 2022", "start_ts": "Fri Jul 29 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 12 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 00:00:00 2022", "start_ts": "Fri Oct 07 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} + 12 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 00:00:00 2023", "start_ts": "Fri Dec 16 00:00:00 2022", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp without time zone"} 12 | 11 | FINISHED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} 12 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} 12 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} @@ -663,14 +659,14 @@ RESET ROLE; -- execute transaction control statements. Transaction control statements are only -- allowed if CALL is executed in its own transaction.` TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; -psql:include/cagg_migrate_common.sql:348: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +psql:include/cagg_migrate_common.sql:341: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" DROP MATERIALIZED VIEW conditions_summary_daily_new; -psql:include/cagg_migrate_common.sql:349: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:342: NOTICE: drop cascades to 6 other objects \set ON_ERROR_STOP 0 BEGIN; -- should fail with `invalid transaction termination` CALL cagg_migrate('conditions_summary_daily'); -psql:include/cagg_migrate_common.sql:354: ERROR: invalid transaction termination +psql:include/cagg_migrate_common.sql:347: ERROR: invalid transaction termination ROLLBACK; \set ON_ERROR_STOP 1 CREATE FUNCTION execute_migration() RETURNS void AS @@ -686,7 +682,7 @@ LANGUAGE plpgsql; BEGIN; -- should fail with `invalid transaction termination` SELECT execute_migration(); -psql:include/cagg_migrate_common.sql:371: ERROR: invalid transaction termination +psql:include/cagg_migrate_common.sql:364: ERROR: invalid transaction termination ROLLBACK; \set ON_ERROR_STOP 1 -- cleanup diff --git a/tsl/test/expected/cagg_migrate_timestamptz.out b/tsl/test/expected/cagg_migrate_timestamptz.out new file mode 100644 index 00000000000..c237d2e52c8 --- /dev/null +++ b/tsl/test/expected/cagg_migrate_timestamptz.out @@ -0,0 +1,651 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +\set IS_DISTRIBUTED FALSE +\set IS_TIME_DIMENSION TRUE +\set TIME_DIMENSION_DATATYPE TIMESTAMPTZ +\ir include/cagg_migrate_common.sql +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER +\if :IS_DISTRIBUTED +\echo 'Running distributed hypertable tests' +\else +\echo 'Running local hypertable tests' +Running local hypertable tests +\endif +CREATE TABLE conditions ( + "time" :TIME_DIMENSION_DATATYPE NOT NULL, + temperature NUMERIC +); +\if :IS_DISTRIBUTED + \if :IS_TIME_DIMENSION + SELECT table_name FROM create_distributed_hypertable('conditions', 'time', replication_factor => 2); + \else + SELECT table_name FROM create_distributed_hypertable('conditions', 'time', chunk_time_interval => 10, replication_factor => 2); + \endif +\else + \if :IS_TIME_DIMENSION + SELECT table_name FROM create_hypertable('conditions', 'time'); + table_name +------------ + conditions +(1 row) + + \else + SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); + \endif +\endif +\if :IS_TIME_DIMENSION + INSERT INTO conditions ("time", temperature) + SELECT + generate_series('2022-01-01 00:00:00-00'::timestamptz, '2022-12-31 23:59:59-00'::timestamptz, '1 hour'), + 0.25; +\else + CREATE OR REPLACE FUNCTION integer_now() + RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS + $$ + SELECT coalesce(max(time), 0) + FROM public.conditions + $$; + \if :IS_DISTRIBUTED + SELECT + 'CREATE OR REPLACE FUNCTION integer_now() RETURNS '||:'TIME_DIMENSION_DATATYPE'||' LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$;' AS "STMT" + \gset + CALL distributed_exec (:'STMT'); + \endif + SELECT set_integer_now_func('conditions', 'integer_now'); + INSERT INTO conditions ("time", temperature) + SELECT + generate_series(1, 1000, 1), + 0.25; +\endif +\set ON_ERROR_STOP 0 +-- should fail relation does not exist +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:63: ERROR: relation "conditions_summary_daily" does not exist at character 19 +CREATE TABLE conditions_summary_daily(); +-- should fail continuous agg does not exist +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:66: ERROR: continuous aggregate "public.conditions_summary_daily" does not exist +\set ON_ERROR_STOP 1 +DROP TABLE conditions_summary_daily; +CREATE MATERIALIZED VIEW conditions_summary_daily_new +WITH (timescaledb.continuous) AS +SELECT +\if :IS_TIME_DIMENSION + time_bucket(INTERVAL '1 day', "time") AS bucket, +\else + time_bucket(INTEGER '24', "time") AS bucket, +\endif + MIN(temperature), + MAX(temperature), + AVG(temperature), + SUM(temperature) +FROM + conditions +GROUP BY + bucket +WITH NO DATA; +\set ON_ERROR_STOP 0 +-- should fail because we don't need to migrate finalized caggs +CALL cagg_migrate('conditions_summary_daily_new'); +psql:include/cagg_migrate_common.sql:91: ERROR: continuous aggregate "public.conditions_summary_daily_new" does not require any migration +\set ON_ERROR_STOP 1 +-- older continuous aggregate to be migrated +CREATE MATERIALIZED VIEW conditions_summary_daily +WITH (timescaledb.continuous, timescaledb.finalized=false) AS +SELECT +\if :IS_TIME_DIMENSION + time_bucket(INTERVAL '1 day', "time") AS bucket, +\else + time_bucket(INTEGER '24', "time") AS bucket, +\endif + MIN(temperature), + MAX(temperature), + AVG(temperature), + SUM(temperature) +FROM + conditions +GROUP BY + bucket; +psql:include/cagg_migrate_common.sql:110: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +SELECT + ca.raw_hypertable_id AS "RAW_HYPERTABLE_ID", + h.schema_name AS "MAT_SCHEMA_NAME", + h.table_name AS "MAT_TABLE_NAME", + partial_view_name AS "PART_VIEW_NAME", + partial_view_schema AS "PART_VIEW_SCHEMA", + direct_view_name AS "DIR_VIEW_NAME", + direct_view_schema AS "DIR_VIEW_SCHEMA" +FROM + _timescaledb_catalog.continuous_agg ca + JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) +WHERE + user_view_name = 'conditions_summary_daily' +\gset +\set ON_ERROR_STOP 0 +-- should fail because the new cagg with suffix '_new' already exists +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:129: ERROR: continuous aggregate "public.conditions_summary_daily_new" already exists +\set ON_ERROR_STOP 1 +-- remove the new cagg to execute the migration +DROP MATERIALIZED VIEW conditions_summary_daily_new; +-- get and set all the cagg data +SELECT + _timescaledb_internal.cagg_migrate_pre_validation( + 'public', + 'conditions_summary_daily', + 'conditions_summary_daily_new' + ) AS "CAGG_DATA" +\gset +CALL _timescaledb_internal.cagg_migrate_create_plan(:'CAGG_DATA', 'conditions_summary_daily_new'); +SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg_migrate_plan; + mat_hypertable_id +------------------- + 3 +(1 row) + +SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; + mat_hypertable_id | step_id | status | type | config +-------------------+---------+-------------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} + 3 | 2 | NOT STARTED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} + 3 | 3 | NOT STARTED | DISABLE POLICIES | {"policies": null} + 3 | 4 | NOT STARTED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} + 3 | 5 | NOT STARTED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 6 | NOT STARTED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 7 | NOT STARTED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 8 | NOT STARTED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 9 | NOT STARTED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 10 | NOT STARTED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 11 | NOT STARTED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 12 | NOT STARTED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 13 | NOT STARTED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 14 | NOT STARTED | ENABLE POLICIES | +(14 rows) + +-- should resume the execution +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:149: WARNING: resuming the migration of the continuous aggregate "public.conditions_summary_daily" +psql:include/cagg_migrate_common.sql:149: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} + 3 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} + 3 | 3 | FINISHED | DISABLE POLICIES | {"policies": null} + 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} + 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 11 | FINISHED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 14 | FINISHED | ENABLE POLICIES | +(14 rows) + +\set ON_ERROR_STOP 0 +-- should error because plan already exists +CALL _timescaledb_internal.cagg_migrate_create_plan(:'CAGG_DATA', 'conditions_summary_daily_new'); +psql:include/cagg_migrate_common.sql:154: ERROR: plan already exists for materialized hypertable 3 +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:155: ERROR: plan already exists for continuous aggregate public.conditions_summary_daily +\set ON_ERROR_STOP 1 +-- policies for test +ALTER MATERIALIZED VIEW conditions_summary_daily SET (timescaledb.compress=true); +\if :IS_TIME_DIMENSION +SELECT add_retention_policy('conditions_summary_daily', '30 days'::interval); + add_retention_policy +---------------------- + 1000 +(1 row) + +SELECT add_continuous_aggregate_policy('conditions_summary_daily', '30 days'::interval, '1 day'::interval, '1 hour'::interval); + add_continuous_aggregate_policy +--------------------------------- + 1001 +(1 row) + +SELECT add_compression_policy('conditions_summary_daily', '45 days'::interval); + add_compression_policy +------------------------ + 1002 +(1 row) + +\else +SELECT add_retention_policy('conditions_summary_daily', '400'::integer); +SELECT add_continuous_aggregate_policy('conditions_summary_daily', '50'::integer, '1'::integer, '1 hour'::interval); +SELECT add_compression_policy('conditions_summary_daily', '100'::integer); +\endif +SELECT * +FROM timescaledb_information.jobs +WHERE hypertable_schema = :'MAT_SCHEMA_NAME' +AND hypertable_name = :'MAT_TABLE_NAME' +AND job_id >= 1000; + job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name +--------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+--------------------------------------------------------------------------------+------------+---------------+-----------------------+----------------------------+-----------------------+------------------------------------------- + 1002 | Compression Policy [1002] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | {"hypertable_id": 3, "compress_after": "@ 45 days"} | | | _timescaledb_internal | _materialized_hypertable_3 | _timescaledb_internal | policy_compression_check + 1001 | Refresh Continuous Aggregate Policy [1001] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 3} | | | _timescaledb_internal | _materialized_hypertable_3 | _timescaledb_internal | policy_refresh_continuous_aggregate_check + 1000 | Retention Policy [1000] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | {"drop_after": "@ 30 days", "hypertable_id": 3} | | | _timescaledb_internal | _materialized_hypertable_3 | _timescaledb_internal | policy_retention_check +(3 rows) + +-- execute the migration +DROP MATERIALIZED VIEW conditions_summary_daily_new; +psql:include/cagg_migrate_common.sql:178: NOTICE: drop cascades to 6 other objects +TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; +psql:include/cagg_migrate_common.sql:179: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:180: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +SELECT + ca.raw_hypertable_id AS "NEW_RAW_HYPERTABLE_ID", + h.schema_name AS "NEW_MAT_SCHEMA_NAME", + h.table_name AS "NEW_MAT_TABLE_NAME", + partial_view_name AS "NEW_PART_VIEW_NAME", + partial_view_schema AS "NEW_PART_VIEW_SCHEMA", + direct_view_name AS "NEW_DIR_VIEW_NAME", + direct_view_schema AS "NEW_DIR_VIEW_SCHEMA" +FROM + _timescaledb_catalog.continuous_agg ca + JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) +WHERE + user_view_name = 'conditions_summary_daily_new' +\gset +\d+ conditions_summary_daily_new + View "public.conditions_summary_daily_new" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+--------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp with time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | +View definition: + SELECT _materialized_hypertable_6.bucket, + _materialized_hypertable_6.min, + _materialized_hypertable_6.max, + _materialized_hypertable_6.avg, + _materialized_hypertable_6.sum + FROM _timescaledb_internal._materialized_hypertable_6 + WHERE _materialized_hypertable_6.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp with time zone) +UNION ALL + SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, + min(conditions.temperature) AS min, + max(conditions.temperature) AS max, + avg(conditions.temperature) AS avg, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); + +SELECT * +FROM timescaledb_information.jobs +WHERE hypertable_schema = :'NEW_MAT_SCHEMA_NAME' +AND hypertable_name = :'NEW_MAT_TABLE_NAME' +AND job_id >= 1000; + job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name +--------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+--------------------------------------------------------------------------------+------------+---------------+-----------------------+----------------------------+-----------------------+------------------------------------------- + 1005 | Compression Policy [1005] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | {"hypertable_id": 6, "compress_after": "@ 45 days"} | | | _timescaledb_internal | _materialized_hypertable_6 | _timescaledb_internal | policy_compression_check + 1004 | Refresh Continuous Aggregate Policy [1004] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 6} | | | _timescaledb_internal | _materialized_hypertable_6 | _timescaledb_internal | policy_refresh_continuous_aggregate_check + 1003 | Retention Policy [1003] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | {"drop_after": "@ 30 days", "hypertable_id": 6} | | | _timescaledb_internal | _materialized_hypertable_6 | _timescaledb_internal | policy_retention_check +(3 rows) + +SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} + 3 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} + 3 | 3 | FINISHED | DISABLE POLICIES | {"policies": [1002, 1000]} + 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} + 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 11 | FINISHED | COPY POLICIES | {"policies": [1002, 1001, 1000], "cagg_name_new": "conditions_summary_daily_new"} + 3 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 14 | FINISHED | ENABLE POLICIES | {"policies": [1003, 1004, 1005, 1002, 1001, 1000]} +(14 rows) + +-- check migrated data. should return 0 (zero) rows +SELECT * FROM conditions_summary_daily +EXCEPT +SELECT * FROM conditions_summary_daily_new; + bucket | min | max | avg | sum +--------+-----+-----+-----+----- +(0 rows) + +-- compress both caggs +SELECT compress_chunk(c) FROM show_chunks('conditions_summary_daily') c ORDER BY c::regclass::text; + compress_chunk +----------------------------------------- + _timescaledb_internal._hyper_3_54_chunk + _timescaledb_internal._hyper_3_55_chunk + _timescaledb_internal._hyper_3_56_chunk + _timescaledb_internal._hyper_3_57_chunk + _timescaledb_internal._hyper_3_58_chunk + _timescaledb_internal._hyper_3_59_chunk +(6 rows) + +SELECT compress_chunk(c) FROM show_chunks('conditions_summary_daily_new') c ORDER BY c::regclass::text; + compress_chunk +----------------------------------------- + _timescaledb_internal._hyper_6_66_chunk + _timescaledb_internal._hyper_6_67_chunk + _timescaledb_internal._hyper_6_68_chunk + _timescaledb_internal._hyper_6_69_chunk + _timescaledb_internal._hyper_6_70_chunk + _timescaledb_internal._hyper_6_71_chunk +(6 rows) + +-- check migrated data after compression. should return 0 (zero) rows +SELECT * FROM conditions_summary_daily +EXCEPT +SELECT * FROM conditions_summary_daily_new; + bucket | min | max | avg | sum +--------+-----+-----+-----+----- +(0 rows) + +CREATE VIEW cagg_jobs AS +SELECT user_view_schema AS schema, user_view_name AS name, bgw_job.* +FROM _timescaledb_config.bgw_job +JOIN _timescaledb_catalog.continuous_agg ON mat_hypertable_id = hypertable_id +ORDER BY bgw_job.id; +-- test migration overriding the new cagg and keeping the old +DROP MATERIALIZED VIEW conditions_summary_daily_new; +psql:include/cagg_migrate_common.sql:228: NOTICE: drop cascades to 6 other objects +TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; +psql:include/cagg_migrate_common.sql:229: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +-- check policies before the migration +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+--------------------------+------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+---------------+---------------+--------------------------------------------------------------------------------+-----------------------+-------------------------------------------+---------- + public | conditions_summary_daily | 1000 | Retention Policy [1000] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | | 3 | {"drop_after": "@ 30 days", "hypertable_id": 3} | _timescaledb_internal | policy_retention_check | + public | conditions_summary_daily | 1001 | Refresh Continuous Aggregate Policy [1001] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | | 3 | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 3} | _timescaledb_internal | policy_refresh_continuous_aggregate_check | + public | conditions_summary_daily | 1002 | Compression Policy [1002] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | | 3 | {"hypertable_id": 3, "compress_after": "@ 45 days"} | _timescaledb_internal | policy_compression_check | +(3 rows) + +CALL cagg_migrate('conditions_summary_daily', override => TRUE); +psql:include/cagg_migrate_common.sql:232: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +-- cagg with the new format because it was overriden +\d+ conditions_summary_daily + View "public.conditions_summary_daily" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+--------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp with time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | +View definition: + SELECT _materialized_hypertable_8.bucket, + _materialized_hypertable_8.min, + _materialized_hypertable_8.max, + _materialized_hypertable_8.avg, + _materialized_hypertable_8.sum + FROM _timescaledb_internal._materialized_hypertable_8 + WHERE _materialized_hypertable_8.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp with time zone) +UNION ALL + SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, + min(conditions.temperature) AS min, + max(conditions.temperature) AS max, + avg(conditions.temperature) AS avg, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); + +-- cagg with the old format because it was overriden +\d+ conditions_summary_daily_old + View "public.conditions_summary_daily_old" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+--------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp with time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | +View definition: + SELECT _materialized_hypertable_3.bucket, + _timescaledb_internal.finalize_agg('pg_catalog.min(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_2_2, NULL::numeric) AS min, + _timescaledb_internal.finalize_agg('pg_catalog.max(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_3_3, NULL::numeric) AS max, + _timescaledb_internal.finalize_agg('pg_catalog.avg(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_4_4, NULL::numeric) AS avg, + _timescaledb_internal.finalize_agg('pg_catalog.sum(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_5_5, NULL::numeric) AS sum + FROM _timescaledb_internal._materialized_hypertable_3 + WHERE _materialized_hypertable_3.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp with time zone) + GROUP BY _materialized_hypertable_3.bucket +UNION ALL + SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, + min(conditions.temperature) AS min, + max(conditions.temperature) AS max, + avg(conditions.temperature) AS avg, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); + +\set ON_ERROR_STOP 0 +-- should fail because the cagg was overriden +SELECT * FROM conditions_summary_daily_new; +psql:include/cagg_migrate_common.sql:239: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +\set ON_ERROR_STOP 1 +-- check policies after the migration +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+--------------------------+------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+---------------+---------------+--------------------------------------------------------------------------------+-----------------------+-------------------------------------------+---------- + public | conditions_summary_daily | 1006 | Retention Policy [1006] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | | 8 | {"drop_after": "@ 30 days", "hypertable_id": 8} | _timescaledb_internal | policy_retention_check | + public | conditions_summary_daily | 1007 | Refresh Continuous Aggregate Policy [1007] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | | 8 | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 8} | _timescaledb_internal | policy_refresh_continuous_aggregate_check | + public | conditions_summary_daily | 1008 | Compression Policy [1008] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | | 8 | {"hypertable_id": 8, "compress_after": "@ 45 days"} | _timescaledb_internal | policy_compression_check | +(3 rows) + +-- should return the old cagg jobs +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily_old'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+------------------------------+------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+---------------+---------------+--------------------------------------------------------------------------------+-----------------------+-------------------------------------------+---------- + public | conditions_summary_daily_old | 1000 | Retention Policy [1000] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | | 3 | {"drop_after": "@ 30 days", "hypertable_id": 3} | _timescaledb_internal | policy_retention_check | + public | conditions_summary_daily_old | 1001 | Refresh Continuous Aggregate Policy [1001] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | | 3 | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 3} | _timescaledb_internal | policy_refresh_continuous_aggregate_check | + public | conditions_summary_daily_old | 1002 | Compression Policy [1002] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | | 3 | {"hypertable_id": 3, "compress_after": "@ 45 days"} | _timescaledb_internal | policy_compression_check | +(3 rows) + +-- should return no rows because the cagg was overwritten +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily_new'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+------+----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- +(0 rows) + +-- test migration overriding the new cagg and removing the old +TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; +psql:include/cagg_migrate_common.sql:249: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +DROP MATERIALIZED VIEW conditions_summary_daily; +psql:include/cagg_migrate_common.sql:250: NOTICE: drop cascades to 6 other objects +ALTER MATERIALIZED VIEW conditions_summary_daily_old RENAME TO conditions_summary_daily; +-- check policies before the migration +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+--------------------------+------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+---------------+---------------+--------------------------------------------------------------------------------+-----------------------+-------------------------------------------+---------- + public | conditions_summary_daily | 1000 | Retention Policy [1000] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | | 3 | {"drop_after": "@ 30 days", "hypertable_id": 3} | _timescaledb_internal | policy_retention_check | + public | conditions_summary_daily | 1001 | Refresh Continuous Aggregate Policy [1001] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | | 3 | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 3} | _timescaledb_internal | policy_refresh_continuous_aggregate_check | + public | conditions_summary_daily | 1002 | Compression Policy [1002] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | | 3 | {"hypertable_id": 3, "compress_after": "@ 45 days"} | _timescaledb_internal | policy_compression_check | +(3 rows) + +CALL cagg_migrate('conditions_summary_daily', override => TRUE, drop_old => TRUE); +psql:include/cagg_migrate_common.sql:254: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:254: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1002 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1001 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1000 not found, skipping +-- cagg with the new format because it was overriden +\d+ conditions_summary_daily + View "public.conditions_summary_daily" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+--------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp with time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | +View definition: + SELECT _materialized_hypertable_10.bucket, + _materialized_hypertable_10.min, + _materialized_hypertable_10.max, + _materialized_hypertable_10.avg, + _materialized_hypertable_10.sum + FROM _timescaledb_internal._materialized_hypertable_10 + WHERE _materialized_hypertable_10.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp with time zone) +UNION ALL + SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, + min(conditions.temperature) AS min, + max(conditions.temperature) AS max, + avg(conditions.temperature) AS avg, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); + +\set ON_ERROR_STOP 0 +-- should fail because the cagg was overriden +SELECT * FROM conditions_summary_daily_new; +psql:include/cagg_migrate_common.sql:259: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +-- should fail because the old cagg was removed +SELECT * FROM conditions_summary_daily_old; +psql:include/cagg_migrate_common.sql:261: ERROR: relation "conditions_summary_daily_old" does not exist at character 15 +\set ON_ERROR_STOP 1 +-- check policies after the migration +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+--------------------------+------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+---------------+---------------+---------------------------------------------------------------------------------+-----------------------+-------------------------------------------+---------- + public | conditions_summary_daily | 1009 | Retention Policy [1009] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | | 10 | {"drop_after": "@ 30 days", "hypertable_id": 10} | _timescaledb_internal | policy_retention_check | + public | conditions_summary_daily | 1010 | Refresh Continuous Aggregate Policy [1010] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | | 10 | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 10} | _timescaledb_internal | policy_refresh_continuous_aggregate_check | + public | conditions_summary_daily | 1011 | Compression Policy [1011] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | | 10 | {"hypertable_id": 10, "compress_after": "@ 45 days"} | _timescaledb_internal | policy_compression_check | +(3 rows) + +-- should return no rows because the old cagg was removed +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily_old'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+------+----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- +(0 rows) + +-- should return no rows because the cagg was overwritten +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily_new'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+------+----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- +(0 rows) + +-- permissions test +TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; +psql:include/cagg_migrate_common.sql:271: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +DROP MATERIALIZED VIEW conditions_summary_daily; +psql:include/cagg_migrate_common.sql:272: NOTICE: drop cascades to 6 other objects +GRANT ALL ON TABLE conditions TO :ROLE_DEFAULT_PERM_USER; +SET ROLE :ROLE_DEFAULT_PERM_USER; +CREATE MATERIALIZED VIEW conditions_summary_daily +WITH (timescaledb.continuous, timescaledb.finalized=false) AS +SELECT +\if :IS_TIME_DIMENSION + time_bucket(INTERVAL '1 day', "time") AS bucket, +\else + time_bucket(INTEGER '24', "time") AS bucket, +\endif + MIN(temperature), + MAX(temperature), + AVG(temperature), + SUM(temperature) +FROM + conditions +GROUP BY + bucket; +psql:include/cagg_migrate_common.sql:291: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +\set ON_ERROR_STOP 0 +-- should fail because the lack of permissions on 'continuous_agg_migrate_plan' catalog table +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:295: ERROR: permission denied for table continuous_agg_migrate_plan +\set ON_ERROR_STOP 1 +RESET ROLE; +GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan TO :ROLE_DEFAULT_PERM_USER; +SET ROLE :ROLE_DEFAULT_PERM_USER; +\set ON_ERROR_STOP 0 +-- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step' catalog table +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:305: ERROR: permission denied for table continuous_agg_migrate_plan_step +\set ON_ERROR_STOP 1 +RESET ROLE; +GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step TO :ROLE_DEFAULT_PERM_USER; +SET ROLE :ROLE_DEFAULT_PERM_USER; +\set ON_ERROR_STOP 0 +-- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step_step_id_seq' catalog sequence +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:315: ERROR: permission denied for sequence continuous_agg_migrate_plan_step_step_id_seq +\set ON_ERROR_STOP 1 +RESET ROLE; +GRANT USAGE ON SEQUENCE _timescaledb_catalog.continuous_agg_migrate_plan_step_step_id_seq TO :ROLE_DEFAULT_PERM_USER; +SET ROLE :ROLE_DEFAULT_PERM_USER; +-- all necessary permissions granted +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:324: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +-- check migrated data. should return 0 (zero) rows +SELECT * FROM conditions_summary_daily +EXCEPT +SELECT * FROM conditions_summary_daily_new; + bucket | min | max | avg | sum +--------+-----+-----+-----+----- +(0 rows) + +SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 12 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} + 12 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} + 12 | 3 | FINISHED | DISABLE POLICIES | {"policies": null} + 12 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} + 12 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 11 | FINISHED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} + 12 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 12 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 12 | 14 | FINISHED | ENABLE POLICIES | +(14 rows) + +RESET ROLE; +-- according to the official documentation trying to execute a procedure with +-- transaction control statements inside an explicit transaction should fail: +-- https://www.postgresql.org/docs/current/sql-call.html +-- `If CALL is executed in a transaction block, then the called procedure cannot +-- execute transaction control statements. Transaction control statements are only +-- allowed if CALL is executed in its own transaction.` +TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; +psql:include/cagg_migrate_common.sql:341: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +DROP MATERIALIZED VIEW conditions_summary_daily_new; +psql:include/cagg_migrate_common.sql:342: NOTICE: drop cascades to 6 other objects +\set ON_ERROR_STOP 0 +BEGIN; +-- should fail with `invalid transaction termination` +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:347: ERROR: invalid transaction termination +ROLLBACK; +\set ON_ERROR_STOP 1 +CREATE FUNCTION execute_migration() RETURNS void AS +$$ +BEGIN + CALL cagg_migrate('conditions_summary_daily'); + RETURN; +END; +$$ +LANGUAGE plpgsql; +\set ON_ERROR_STOP 0 +-- execute migration inside a plpgsql function +BEGIN; +-- should fail with `invalid transaction termination` +SELECT execute_migration(); +psql:include/cagg_migrate_common.sql:364: ERROR: invalid transaction termination +ROLLBACK; +\set ON_ERROR_STOP 1 diff --git a/tsl/test/expected/cagg_migrate_timestamptz_dist_ht.out b/tsl/test/expected/cagg_migrate_timestamptz_dist_ht.out new file mode 100644 index 00000000000..1ece09c5567 --- /dev/null +++ b/tsl/test/expected/cagg_migrate_timestamptz_dist_ht.out @@ -0,0 +1,691 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +------------------------------------ +-- Set up a distributed environment +------------------------------------ +\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER +\set DATA_NODE_1 :TEST_DBNAME _1 +\set DATA_NODE_2 :TEST_DBNAME _2 +\set DATA_NODE_3 :TEST_DBNAME _3 +\ir include/remote_exec.sql +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +CREATE SCHEMA IF NOT EXISTS test; +psql:include/remote_exec.sql:5: NOTICE: schema "test" already exists, skipping +GRANT USAGE ON SCHEMA test TO PUBLIC; +CREATE OR REPLACE FUNCTION test.remote_exec(srv_name name[], command text) +RETURNS VOID +AS :TSL_MODULE_PATHNAME, 'ts_remote_exec' +LANGUAGE C; +CREATE OR REPLACE FUNCTION test.remote_exec_get_result_strings(srv_name name[], command text) +RETURNS TABLE("table_record" CSTRING[]) +AS :TSL_MODULE_PATHNAME, 'ts_remote_exec_get_result_strings' +LANGUAGE C; +SELECT node_name, database, node_created, database_created, extension_created +FROM ( + SELECT (add_data_node(name, host => 'localhost', DATABASE => name)).* + FROM (VALUES (:'DATA_NODE_1'), (:'DATA_NODE_2'), (:'DATA_NODE_3')) v(name) +) a; + node_name | database | node_created | database_created | extension_created +---------------------------------------+---------------------------------------+--------------+------------------+------------------- + db_cagg_migrate_timestamptz_dist_ht_1 | db_cagg_migrate_timestamptz_dist_ht_1 | t | t | t + db_cagg_migrate_timestamptz_dist_ht_2 | db_cagg_migrate_timestamptz_dist_ht_2 | t | t | t + db_cagg_migrate_timestamptz_dist_ht_3 | db_cagg_migrate_timestamptz_dist_ht_3 | t | t | t +(3 rows) + +GRANT USAGE ON FOREIGN SERVER :DATA_NODE_1, :DATA_NODE_2, :DATA_NODE_3 TO PUBLIC; +\set IS_DISTRIBUTED TRUE +\set IS_TIME_DIMENSION TRUE +\set TIME_DIMENSION_DATATYPE TIMESTAMPTZ +\ir include/cagg_migrate_common.sql +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER +\if :IS_DISTRIBUTED +\echo 'Running distributed hypertable tests' +Running distributed hypertable tests +\else +\echo 'Running local hypertable tests' +\endif +CREATE TABLE conditions ( + "time" :TIME_DIMENSION_DATATYPE NOT NULL, + temperature NUMERIC +); +\if :IS_DISTRIBUTED + \if :IS_TIME_DIMENSION + SELECT table_name FROM create_distributed_hypertable('conditions', 'time', replication_factor => 2); + table_name +------------ + conditions +(1 row) + + \else + SELECT table_name FROM create_distributed_hypertable('conditions', 'time', chunk_time_interval => 10, replication_factor => 2); + \endif +\else + \if :IS_TIME_DIMENSION + SELECT table_name FROM create_hypertable('conditions', 'time'); + \else + SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); + \endif +\endif +\if :IS_TIME_DIMENSION + INSERT INTO conditions ("time", temperature) + SELECT + generate_series('2022-01-01 00:00:00-00'::timestamptz, '2022-12-31 23:59:59-00'::timestamptz, '1 hour'), + 0.25; +\else + CREATE OR REPLACE FUNCTION integer_now() + RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS + $$ + SELECT coalesce(max(time), 0) + FROM public.conditions + $$; + \if :IS_DISTRIBUTED + SELECT + 'CREATE OR REPLACE FUNCTION integer_now() RETURNS '||:'TIME_DIMENSION_DATATYPE'||' LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$;' AS "STMT" + \gset + CALL distributed_exec (:'STMT'); + \endif + SELECT set_integer_now_func('conditions', 'integer_now'); + INSERT INTO conditions ("time", temperature) + SELECT + generate_series(1, 1000, 1), + 0.25; +\endif +\set ON_ERROR_STOP 0 +-- should fail relation does not exist +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:63: ERROR: relation "conditions_summary_daily" does not exist at character 19 +CREATE TABLE conditions_summary_daily(); +-- should fail continuous agg does not exist +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:66: ERROR: continuous aggregate "public.conditions_summary_daily" does not exist +\set ON_ERROR_STOP 1 +DROP TABLE conditions_summary_daily; +CREATE MATERIALIZED VIEW conditions_summary_daily_new +WITH (timescaledb.continuous) AS +SELECT +\if :IS_TIME_DIMENSION + time_bucket(INTERVAL '1 day', "time") AS bucket, +\else + time_bucket(INTEGER '24', "time") AS bucket, +\endif + MIN(temperature), + MAX(temperature), + AVG(temperature), + SUM(temperature) +FROM + conditions +GROUP BY + bucket +WITH NO DATA; +\set ON_ERROR_STOP 0 +-- should fail because we don't need to migrate finalized caggs +CALL cagg_migrate('conditions_summary_daily_new'); +psql:include/cagg_migrate_common.sql:91: ERROR: continuous aggregate "public.conditions_summary_daily_new" does not require any migration +\set ON_ERROR_STOP 1 +-- older continuous aggregate to be migrated +CREATE MATERIALIZED VIEW conditions_summary_daily +WITH (timescaledb.continuous, timescaledb.finalized=false) AS +SELECT +\if :IS_TIME_DIMENSION + time_bucket(INTERVAL '1 day', "time") AS bucket, +\else + time_bucket(INTEGER '24', "time") AS bucket, +\endif + MIN(temperature), + MAX(temperature), + AVG(temperature), + SUM(temperature) +FROM + conditions +GROUP BY + bucket; +psql:include/cagg_migrate_common.sql:110: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +SELECT + ca.raw_hypertable_id AS "RAW_HYPERTABLE_ID", + h.schema_name AS "MAT_SCHEMA_NAME", + h.table_name AS "MAT_TABLE_NAME", + partial_view_name AS "PART_VIEW_NAME", + partial_view_schema AS "PART_VIEW_SCHEMA", + direct_view_name AS "DIR_VIEW_NAME", + direct_view_schema AS "DIR_VIEW_SCHEMA" +FROM + _timescaledb_catalog.continuous_agg ca + JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) +WHERE + user_view_name = 'conditions_summary_daily' +\gset +\set ON_ERROR_STOP 0 +-- should fail because the new cagg with suffix '_new' already exists +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:129: ERROR: continuous aggregate "public.conditions_summary_daily_new" already exists +\set ON_ERROR_STOP 1 +-- remove the new cagg to execute the migration +DROP MATERIALIZED VIEW conditions_summary_daily_new; +-- get and set all the cagg data +SELECT + _timescaledb_internal.cagg_migrate_pre_validation( + 'public', + 'conditions_summary_daily', + 'conditions_summary_daily_new' + ) AS "CAGG_DATA" +\gset +CALL _timescaledb_internal.cagg_migrate_create_plan(:'CAGG_DATA', 'conditions_summary_daily_new'); +SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg_migrate_plan; + mat_hypertable_id +------------------- + 3 +(1 row) + +SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; + mat_hypertable_id | step_id | status | type | config +-------------------+---------+-------------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} + 3 | 2 | NOT STARTED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} + 3 | 3 | NOT STARTED | DISABLE POLICIES | {"policies": null} + 3 | 4 | NOT STARTED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} + 3 | 5 | NOT STARTED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 6 | NOT STARTED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 7 | NOT STARTED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 8 | NOT STARTED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 9 | NOT STARTED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 10 | NOT STARTED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 11 | NOT STARTED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 12 | NOT STARTED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 13 | NOT STARTED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 14 | NOT STARTED | ENABLE POLICIES | +(14 rows) + +-- should resume the execution +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:149: WARNING: resuming the migration of the continuous aggregate "public.conditions_summary_daily" +psql:include/cagg_migrate_common.sql:149: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} + 3 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} + 3 | 3 | FINISHED | DISABLE POLICIES | {"policies": null} + 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} + 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 11 | FINISHED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 14 | FINISHED | ENABLE POLICIES | +(14 rows) + +\set ON_ERROR_STOP 0 +-- should error because plan already exists +CALL _timescaledb_internal.cagg_migrate_create_plan(:'CAGG_DATA', 'conditions_summary_daily_new'); +psql:include/cagg_migrate_common.sql:154: ERROR: plan already exists for materialized hypertable 3 +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:155: ERROR: plan already exists for continuous aggregate public.conditions_summary_daily +\set ON_ERROR_STOP 1 +-- policies for test +ALTER MATERIALIZED VIEW conditions_summary_daily SET (timescaledb.compress=true); +\if :IS_TIME_DIMENSION +SELECT add_retention_policy('conditions_summary_daily', '30 days'::interval); + add_retention_policy +---------------------- + 1000 +(1 row) + +SELECT add_continuous_aggregate_policy('conditions_summary_daily', '30 days'::interval, '1 day'::interval, '1 hour'::interval); + add_continuous_aggregate_policy +--------------------------------- + 1001 +(1 row) + +SELECT add_compression_policy('conditions_summary_daily', '45 days'::interval); + add_compression_policy +------------------------ + 1002 +(1 row) + +\else +SELECT add_retention_policy('conditions_summary_daily', '400'::integer); +SELECT add_continuous_aggregate_policy('conditions_summary_daily', '50'::integer, '1'::integer, '1 hour'::interval); +SELECT add_compression_policy('conditions_summary_daily', '100'::integer); +\endif +SELECT * +FROM timescaledb_information.jobs +WHERE hypertable_schema = :'MAT_SCHEMA_NAME' +AND hypertable_name = :'MAT_TABLE_NAME' +AND job_id >= 1000; + job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name +--------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+--------------------------------------------------------------------------------+------------+---------------+-----------------------+----------------------------+-----------------------+------------------------------------------- + 1002 | Compression Policy [1002] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | {"hypertable_id": 3, "compress_after": "@ 45 days"} | | | _timescaledb_internal | _materialized_hypertable_3 | _timescaledb_internal | policy_compression_check + 1001 | Refresh Continuous Aggregate Policy [1001] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 3} | | | _timescaledb_internal | _materialized_hypertable_3 | _timescaledb_internal | policy_refresh_continuous_aggregate_check + 1000 | Retention Policy [1000] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | {"drop_after": "@ 30 days", "hypertable_id": 3} | | | _timescaledb_internal | _materialized_hypertable_3 | _timescaledb_internal | policy_retention_check +(3 rows) + +-- execute the migration +DROP MATERIALIZED VIEW conditions_summary_daily_new; +psql:include/cagg_migrate_common.sql:178: NOTICE: drop cascades to 6 other objects +TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; +psql:include/cagg_migrate_common.sql:179: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:180: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +SELECT + ca.raw_hypertable_id AS "NEW_RAW_HYPERTABLE_ID", + h.schema_name AS "NEW_MAT_SCHEMA_NAME", + h.table_name AS "NEW_MAT_TABLE_NAME", + partial_view_name AS "NEW_PART_VIEW_NAME", + partial_view_schema AS "NEW_PART_VIEW_SCHEMA", + direct_view_name AS "NEW_DIR_VIEW_NAME", + direct_view_schema AS "NEW_DIR_VIEW_SCHEMA" +FROM + _timescaledb_catalog.continuous_agg ca + JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) +WHERE + user_view_name = 'conditions_summary_daily_new' +\gset +\d+ conditions_summary_daily_new + View "public.conditions_summary_daily_new" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+--------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp with time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | +View definition: + SELECT _materialized_hypertable_6.bucket, + _materialized_hypertable_6.min, + _materialized_hypertable_6.max, + _materialized_hypertable_6.avg, + _materialized_hypertable_6.sum + FROM _timescaledb_internal._materialized_hypertable_6 + WHERE _materialized_hypertable_6.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp with time zone) +UNION ALL + SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, + min(conditions.temperature) AS min, + max(conditions.temperature) AS max, + avg(conditions.temperature) AS avg, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(6)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); + +SELECT * +FROM timescaledb_information.jobs +WHERE hypertable_schema = :'NEW_MAT_SCHEMA_NAME' +AND hypertable_name = :'NEW_MAT_TABLE_NAME' +AND job_id >= 1000; + job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name +--------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+--------------------------------------------------------------------------------+------------+---------------+-----------------------+----------------------------+-----------------------+------------------------------------------- + 1005 | Compression Policy [1005] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | {"hypertable_id": 6, "compress_after": "@ 45 days"} | | | _timescaledb_internal | _materialized_hypertable_6 | _timescaledb_internal | policy_compression_check + 1004 | Refresh Continuous Aggregate Policy [1004] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 6} | | | _timescaledb_internal | _materialized_hypertable_6 | _timescaledb_internal | policy_refresh_continuous_aggregate_check + 1003 | Retention Policy [1003] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | {"drop_after": "@ 30 days", "hypertable_id": 6} | | | _timescaledb_internal | _materialized_hypertable_6 | _timescaledb_internal | policy_retention_check +(3 rows) + +SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 3 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} + 3 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} + 3 | 3 | FINISHED | DISABLE POLICIES | {"policies": [1002, 1000]} + 3 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} + 3 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 3 | 11 | FINISHED | COPY POLICIES | {"policies": [1002, 1001, 1000], "cagg_name_new": "conditions_summary_daily_new"} + 3 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 3 | 14 | FINISHED | ENABLE POLICIES | {"policies": [1003, 1004, 1005, 1002, 1001, 1000]} +(14 rows) + +-- check migrated data. should return 0 (zero) rows +SELECT * FROM conditions_summary_daily +EXCEPT +SELECT * FROM conditions_summary_daily_new; + bucket | min | max | avg | sum +--------+-----+-----+-----+----- +(0 rows) + +-- compress both caggs +SELECT compress_chunk(c) FROM show_chunks('conditions_summary_daily') c ORDER BY c::regclass::text; + compress_chunk +----------------------------------------- + _timescaledb_internal._hyper_3_54_chunk + _timescaledb_internal._hyper_3_55_chunk + _timescaledb_internal._hyper_3_56_chunk + _timescaledb_internal._hyper_3_57_chunk + _timescaledb_internal._hyper_3_58_chunk + _timescaledb_internal._hyper_3_59_chunk +(6 rows) + +SELECT compress_chunk(c) FROM show_chunks('conditions_summary_daily_new') c ORDER BY c::regclass::text; + compress_chunk +----------------------------------------- + _timescaledb_internal._hyper_6_66_chunk + _timescaledb_internal._hyper_6_67_chunk + _timescaledb_internal._hyper_6_68_chunk + _timescaledb_internal._hyper_6_69_chunk + _timescaledb_internal._hyper_6_70_chunk + _timescaledb_internal._hyper_6_71_chunk +(6 rows) + +-- check migrated data after compression. should return 0 (zero) rows +SELECT * FROM conditions_summary_daily +EXCEPT +SELECT * FROM conditions_summary_daily_new; + bucket | min | max | avg | sum +--------+-----+-----+-----+----- +(0 rows) + +CREATE VIEW cagg_jobs AS +SELECT user_view_schema AS schema, user_view_name AS name, bgw_job.* +FROM _timescaledb_config.bgw_job +JOIN _timescaledb_catalog.continuous_agg ON mat_hypertable_id = hypertable_id +ORDER BY bgw_job.id; +-- test migration overriding the new cagg and keeping the old +DROP MATERIALIZED VIEW conditions_summary_daily_new; +psql:include/cagg_migrate_common.sql:228: NOTICE: drop cascades to 6 other objects +TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; +psql:include/cagg_migrate_common.sql:229: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +-- check policies before the migration +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+--------------------------+------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+---------------+---------------+--------------------------------------------------------------------------------+-----------------------+-------------------------------------------+---------- + public | conditions_summary_daily | 1000 | Retention Policy [1000] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | | 3 | {"drop_after": "@ 30 days", "hypertable_id": 3} | _timescaledb_internal | policy_retention_check | + public | conditions_summary_daily | 1001 | Refresh Continuous Aggregate Policy [1001] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | | 3 | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 3} | _timescaledb_internal | policy_refresh_continuous_aggregate_check | + public | conditions_summary_daily | 1002 | Compression Policy [1002] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | | 3 | {"hypertable_id": 3, "compress_after": "@ 45 days"} | _timescaledb_internal | policy_compression_check | +(3 rows) + +CALL cagg_migrate('conditions_summary_daily', override => TRUE); +psql:include/cagg_migrate_common.sql:232: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +-- cagg with the new format because it was overriden +\d+ conditions_summary_daily + View "public.conditions_summary_daily" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+--------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp with time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | +View definition: + SELECT _materialized_hypertable_8.bucket, + _materialized_hypertable_8.min, + _materialized_hypertable_8.max, + _materialized_hypertable_8.avg, + _materialized_hypertable_8.sum + FROM _timescaledb_internal._materialized_hypertable_8 + WHERE _materialized_hypertable_8.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp with time zone) +UNION ALL + SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, + min(conditions.temperature) AS min, + max(conditions.temperature) AS max, + avg(conditions.temperature) AS avg, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(8)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); + +-- cagg with the old format because it was overriden +\d+ conditions_summary_daily_old + View "public.conditions_summary_daily_old" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+--------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp with time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | +View definition: + SELECT _materialized_hypertable_3.bucket, + _timescaledb_internal.finalize_agg('pg_catalog.min(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_2_2, NULL::numeric) AS min, + _timescaledb_internal.finalize_agg('pg_catalog.max(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_3_3, NULL::numeric) AS max, + _timescaledb_internal.finalize_agg('pg_catalog.avg(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_4_4, NULL::numeric) AS avg, + _timescaledb_internal.finalize_agg('pg_catalog.sum(numeric)'::text, NULL::name, NULL::name, '{{pg_catalog,numeric}}'::name[], _materialized_hypertable_3.agg_5_5, NULL::numeric) AS sum + FROM _timescaledb_internal._materialized_hypertable_3 + WHERE _materialized_hypertable_3.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp with time zone) + GROUP BY _materialized_hypertable_3.bucket +UNION ALL + SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, + min(conditions.temperature) AS min, + max(conditions.temperature) AS max, + avg(conditions.temperature) AS avg, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(3)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); + +\set ON_ERROR_STOP 0 +-- should fail because the cagg was overriden +SELECT * FROM conditions_summary_daily_new; +psql:include/cagg_migrate_common.sql:239: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +\set ON_ERROR_STOP 1 +-- check policies after the migration +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+--------------------------+------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+---------------+---------------+--------------------------------------------------------------------------------+-----------------------+-------------------------------------------+---------- + public | conditions_summary_daily | 1006 | Retention Policy [1006] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | | 8 | {"drop_after": "@ 30 days", "hypertable_id": 8} | _timescaledb_internal | policy_retention_check | + public | conditions_summary_daily | 1007 | Refresh Continuous Aggregate Policy [1007] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | | 8 | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 8} | _timescaledb_internal | policy_refresh_continuous_aggregate_check | + public | conditions_summary_daily | 1008 | Compression Policy [1008] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | | 8 | {"hypertable_id": 8, "compress_after": "@ 45 days"} | _timescaledb_internal | policy_compression_check | +(3 rows) + +-- should return the old cagg jobs +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily_old'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+------------------------------+------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+---------------+---------------+--------------------------------------------------------------------------------+-----------------------+-------------------------------------------+---------- + public | conditions_summary_daily_old | 1000 | Retention Policy [1000] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | | 3 | {"drop_after": "@ 30 days", "hypertable_id": 3} | _timescaledb_internal | policy_retention_check | + public | conditions_summary_daily_old | 1001 | Refresh Continuous Aggregate Policy [1001] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | | 3 | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 3} | _timescaledb_internal | policy_refresh_continuous_aggregate_check | + public | conditions_summary_daily_old | 1002 | Compression Policy [1002] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | | 3 | {"hypertable_id": 3, "compress_after": "@ 45 days"} | _timescaledb_internal | policy_compression_check | +(3 rows) + +-- should return no rows because the cagg was overwritten +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily_new'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+------+----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- +(0 rows) + +-- test migration overriding the new cagg and removing the old +TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; +psql:include/cagg_migrate_common.sql:249: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +DROP MATERIALIZED VIEW conditions_summary_daily; +psql:include/cagg_migrate_common.sql:250: NOTICE: drop cascades to 6 other objects +ALTER MATERIALIZED VIEW conditions_summary_daily_old RENAME TO conditions_summary_daily; +-- check policies before the migration +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+--------------------------+------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+---------------+---------------+--------------------------------------------------------------------------------+-----------------------+-------------------------------------------+---------- + public | conditions_summary_daily | 1000 | Retention Policy [1000] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | | 3 | {"drop_after": "@ 30 days", "hypertable_id": 3} | _timescaledb_internal | policy_retention_check | + public | conditions_summary_daily | 1001 | Refresh Continuous Aggregate Policy [1001] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | | 3 | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 3} | _timescaledb_internal | policy_refresh_continuous_aggregate_check | + public | conditions_summary_daily | 1002 | Compression Policy [1002] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | | 3 | {"hypertable_id": 3, "compress_after": "@ 45 days"} | _timescaledb_internal | policy_compression_check | +(3 rows) + +CALL cagg_migrate('conditions_summary_daily', override => TRUE, drop_old => TRUE); +psql:include/cagg_migrate_common.sql:254: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +psql:include/cagg_migrate_common.sql:254: NOTICE: drop cascades to 6 other objects +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1002 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1001 not found, skipping +psql:include/cagg_migrate_common.sql:254: NOTICE: job 1000 not found, skipping +-- cagg with the new format because it was overriden +\d+ conditions_summary_daily + View "public.conditions_summary_daily" + Column | Type | Collation | Nullable | Default | Storage | Description +--------+--------------------------+-----------+----------+---------+---------+------------- + bucket | timestamp with time zone | | | | plain | + min | numeric | | | | main | + max | numeric | | | | main | + avg | numeric | | | | main | + sum | numeric | | | | main | +View definition: + SELECT _materialized_hypertable_10.bucket, + _materialized_hypertable_10.min, + _materialized_hypertable_10.max, + _materialized_hypertable_10.avg, + _materialized_hypertable_10.sum + FROM _timescaledb_internal._materialized_hypertable_10 + WHERE _materialized_hypertable_10.bucket < COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp with time zone) +UNION ALL + SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, + min(conditions.temperature) AS min, + max(conditions.temperature) AS max, + avg(conditions.temperature) AS avg, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions."time" >= COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(10)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); + +\set ON_ERROR_STOP 0 +-- should fail because the cagg was overriden +SELECT * FROM conditions_summary_daily_new; +psql:include/cagg_migrate_common.sql:259: ERROR: relation "conditions_summary_daily_new" does not exist at character 15 +-- should fail because the old cagg was removed +SELECT * FROM conditions_summary_daily_old; +psql:include/cagg_migrate_common.sql:261: ERROR: relation "conditions_summary_daily_old" does not exist at character 15 +\set ON_ERROR_STOP 1 +-- check policies after the migration +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+--------------------------+------+--------------------------------------------+-------------------+-------------+-------------+--------------+-----------------------+-------------------------------------+--------------------+-----------+----------------+---------------+---------------+---------------------------------------------------------------------------------+-----------------------+-------------------------------------------+---------- + public | conditions_summary_daily | 1009 | Retention Policy [1009] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_internal | policy_retention | cluster_super_user | t | f | | 10 | {"drop_after": "@ 30 days", "hypertable_id": 10} | _timescaledb_internal | policy_retention_check | + public | conditions_summary_daily | 1010 | Refresh Continuous Aggregate Policy [1010] | @ 1 hour | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_refresh_continuous_aggregate | cluster_super_user | t | f | | 10 | {"end_offset": "@ 1 day", "start_offset": "@ 30 days", "mat_hypertable_id": 10} | _timescaledb_internal | policy_refresh_continuous_aggregate_check | + public | conditions_summary_daily | 1011 | Compression Policy [1011] | @ 35 days | @ 0 | -1 | @ 1 hour | _timescaledb_internal | policy_compression | cluster_super_user | t | f | | 10 | {"hypertable_id": 10, "compress_after": "@ 45 days"} | _timescaledb_internal | policy_compression_check | +(3 rows) + +-- should return no rows because the old cagg was removed +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily_old'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+------+----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- +(0 rows) + +-- should return no rows because the cagg was overwritten +SELECT * FROM cagg_jobs WHERE schema = 'public' AND name = 'conditions_summary_daily_new'; + schema | name | id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone +--------+------+----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- +(0 rows) + +-- permissions test +TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; +psql:include/cagg_migrate_common.sql:271: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +DROP MATERIALIZED VIEW conditions_summary_daily; +psql:include/cagg_migrate_common.sql:272: NOTICE: drop cascades to 6 other objects +GRANT ALL ON TABLE conditions TO :ROLE_DEFAULT_PERM_USER; +SET ROLE :ROLE_DEFAULT_PERM_USER; +CREATE MATERIALIZED VIEW conditions_summary_daily +WITH (timescaledb.continuous, timescaledb.finalized=false) AS +SELECT +\if :IS_TIME_DIMENSION + time_bucket(INTERVAL '1 day', "time") AS bucket, +\else + time_bucket(INTEGER '24', "time") AS bucket, +\endif + MIN(temperature), + MAX(temperature), + AVG(temperature), + SUM(temperature) +FROM + conditions +GROUP BY + bucket; +psql:include/cagg_migrate_common.sql:291: NOTICE: refreshing continuous aggregate "conditions_summary_daily" +\set ON_ERROR_STOP 0 +-- should fail because the lack of permissions on 'continuous_agg_migrate_plan' catalog table +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:295: ERROR: permission denied for table continuous_agg_migrate_plan +\set ON_ERROR_STOP 1 +RESET ROLE; +GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan TO :ROLE_DEFAULT_PERM_USER; +SET ROLE :ROLE_DEFAULT_PERM_USER; +\set ON_ERROR_STOP 0 +-- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step' catalog table +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:305: ERROR: permission denied for table continuous_agg_migrate_plan_step +\set ON_ERROR_STOP 1 +RESET ROLE; +GRANT SELECT, INSERT, UPDATE ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step TO :ROLE_DEFAULT_PERM_USER; +SET ROLE :ROLE_DEFAULT_PERM_USER; +\set ON_ERROR_STOP 0 +-- should fail because the lack of permissions on 'continuous_agg_migrate_plan_step_step_id_seq' catalog sequence +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:315: ERROR: permission denied for sequence continuous_agg_migrate_plan_step_step_id_seq +\set ON_ERROR_STOP 1 +RESET ROLE; +GRANT USAGE ON SEQUENCE _timescaledb_catalog.continuous_agg_migrate_plan_step_step_id_seq TO :ROLE_DEFAULT_PERM_USER; +SET ROLE :ROLE_DEFAULT_PERM_USER; +-- all necessary permissions granted +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:324: NOTICE: continuous aggregate "conditions_summary_daily_new" is already up-to-date +-- check migrated data. should return 0 (zero) rows +SELECT * FROM conditions_summary_daily +EXCEPT +SELECT * FROM conditions_summary_daily_new; + bucket | min | max | avg | sum +--------+-----+-----+-----+----- +(0 rows) + +SELECT mat_hypertable_id, step_id, status, type, config FROM _timescaledb_catalog.continuous_agg_migrate_plan_step ORDER BY step_id; + mat_hypertable_id | step_id | status | type | config +-------------------+---------+----------+------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + 12 | 1 | FINISHED | SAVE WATERMARK | {"watermark": "Sun Jan 01 00:00:00 2023"} + 12 | 2 | FINISHED | CREATE NEW CAGG | {"cagg_name_new": "conditions_summary_daily_new"} + 12 | 3 | FINISHED | DISABLE POLICIES | {"policies": null} + 12 | 4 | FINISHED | REFRESH NEW CAGG | {"window_start": "Sun Jan 01 00:00:00 2023", "cagg_name_new": "conditions_summary_daily_new", "window_start_type": "timestamp with time zone"} + 12 | 5 | FINISHED | COPY DATA | {"end_ts": "Fri Mar 11 16:00:00 2022 PST", "start_ts": "Fri Dec 31 16:00:00 2021 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 6 | FINISHED | COPY DATA | {"end_ts": "Fri May 20 16:00:00 2022 PDT", "start_ts": "Fri Mar 11 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 7 | FINISHED | COPY DATA | {"end_ts": "Fri Jul 29 16:00:00 2022 PDT", "start_ts": "Fri May 20 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 8 | FINISHED | COPY DATA | {"end_ts": "Fri Oct 07 16:00:00 2022 PDT", "start_ts": "Fri Jul 29 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 9 | FINISHED | COPY DATA | {"end_ts": "Fri Dec 16 16:00:00 2022 PST", "start_ts": "Fri Oct 07 16:00:00 2022 PDT", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 10 | FINISHED | COPY DATA | {"end_ts": "Fri Feb 24 16:00:00 2023 PST", "start_ts": "Fri Dec 16 16:00:00 2022 PST", "cagg_name_new": "conditions_summary_daily_new", "bucket_column_name": "bucket", "bucket_column_type": "timestamp with time zone"} + 12 | 11 | FINISHED | COPY POLICIES | {"policies": null, "cagg_name_new": "conditions_summary_daily_new"} + 12 | 12 | FINISHED | OVERRIDE CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 12 | 13 | FINISHED | DROP OLD CAGG | {"drop_old": false, "override": false, "cagg_name_new": "conditions_summary_daily_new"} + 12 | 14 | FINISHED | ENABLE POLICIES | +(14 rows) + +RESET ROLE; +-- according to the official documentation trying to execute a procedure with +-- transaction control statements inside an explicit transaction should fail: +-- https://www.postgresql.org/docs/current/sql-call.html +-- `If CALL is executed in a transaction block, then the called procedure cannot +-- execute transaction control statements. Transaction control statements are only +-- allowed if CALL is executed in its own transaction.` +TRUNCATE _timescaledb_catalog.continuous_agg_migrate_plan RESTART IDENTITY CASCADE; +psql:include/cagg_migrate_common.sql:341: NOTICE: truncate cascades to table "continuous_agg_migrate_plan_step" +DROP MATERIALIZED VIEW conditions_summary_daily_new; +psql:include/cagg_migrate_common.sql:342: NOTICE: drop cascades to 6 other objects +\set ON_ERROR_STOP 0 +BEGIN; +-- should fail with `invalid transaction termination` +CALL cagg_migrate('conditions_summary_daily'); +psql:include/cagg_migrate_common.sql:347: ERROR: invalid transaction termination +ROLLBACK; +\set ON_ERROR_STOP 1 +CREATE FUNCTION execute_migration() RETURNS void AS +$$ +BEGIN + CALL cagg_migrate('conditions_summary_daily'); + RETURN; +END; +$$ +LANGUAGE plpgsql; +\set ON_ERROR_STOP 0 +-- execute migration inside a plpgsql function +BEGIN; +-- should fail with `invalid transaction termination` +SELECT execute_migration(); +psql:include/cagg_migrate_common.sql:364: ERROR: invalid transaction termination +ROLLBACK; +\set ON_ERROR_STOP 1 +-- cleanup +\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER; +DROP DATABASE :DATA_NODE_1; +DROP DATABASE :DATA_NODE_2; +DROP DATABASE :DATA_NODE_3; diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index a3675fbe85d..f63639d974e 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -58,6 +58,8 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) cagg_migrate_integer_dist_ht.sql cagg_migrate_timestamp.sql cagg_migrate_timestamp_dist_ht.sql + cagg_migrate_timestamptz.sql + cagg_migrate_timestamptz_dist_ht.sql cagg_multi.sql continuous_aggs_deprecated.sql cagg_tableam.sql diff --git a/tsl/test/sql/cagg_migrate_integer.sql b/tsl/test/sql/cagg_migrate_integer.sql index bf8dda79f62..c6b7f4f2883 100644 --- a/tsl/test/sql/cagg_migrate_integer.sql +++ b/tsl/test/sql/cagg_migrate_integer.sql @@ -4,5 +4,6 @@ \set IS_DISTRIBUTED FALSE \set IS_TIME_DIMENSION FALSE +\set TIME_DIMENSION_DATATYPE INTEGER \ir include/cagg_migrate_common.sql diff --git a/tsl/test/sql/cagg_migrate_integer_dist_ht.sql b/tsl/test/sql/cagg_migrate_integer_dist_ht.sql index 91aeb532728..bc8751c8853 100644 --- a/tsl/test/sql/cagg_migrate_integer_dist_ht.sql +++ b/tsl/test/sql/cagg_migrate_integer_dist_ht.sql @@ -23,6 +23,7 @@ GRANT USAGE ON FOREIGN SERVER :DATA_NODE_1, :DATA_NODE_2, :DATA_NODE_3 TO PUBLIC \set IS_DISTRIBUTED TRUE \set IS_TIME_DIMENSION FALSE +\set TIME_DIMENSION_DATATYPE INTEGER \ir include/cagg_migrate_common.sql diff --git a/tsl/test/sql/cagg_migrate_timestamp.sql b/tsl/test/sql/cagg_migrate_timestamp.sql index 34f3810f952..98bc69e95d1 100644 --- a/tsl/test/sql/cagg_migrate_timestamp.sql +++ b/tsl/test/sql/cagg_migrate_timestamp.sql @@ -4,5 +4,6 @@ \set IS_DISTRIBUTED FALSE \set IS_TIME_DIMENSION TRUE +\set TIME_DIMENSION_DATATYPE TIMESTAMP \ir include/cagg_migrate_common.sql diff --git a/tsl/test/sql/cagg_migrate_timestamp_dist_ht.sql b/tsl/test/sql/cagg_migrate_timestamp_dist_ht.sql index 435534b52ab..302a31853ae 100644 --- a/tsl/test/sql/cagg_migrate_timestamp_dist_ht.sql +++ b/tsl/test/sql/cagg_migrate_timestamp_dist_ht.sql @@ -23,6 +23,7 @@ GRANT USAGE ON FOREIGN SERVER :DATA_NODE_1, :DATA_NODE_2, :DATA_NODE_3 TO PUBLIC \set IS_DISTRIBUTED TRUE \set IS_TIME_DIMENSION TRUE +\set TIME_DIMENSION_DATATYPE TIMESTAMP \ir include/cagg_migrate_common.sql diff --git a/tsl/test/sql/cagg_migrate_timestamptz.sql b/tsl/test/sql/cagg_migrate_timestamptz.sql new file mode 100644 index 00000000000..6c057a10aef --- /dev/null +++ b/tsl/test/sql/cagg_migrate_timestamptz.sql @@ -0,0 +1,9 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +\set IS_DISTRIBUTED FALSE +\set IS_TIME_DIMENSION TRUE +\set TIME_DIMENSION_DATATYPE TIMESTAMPTZ + +\ir include/cagg_migrate_common.sql diff --git a/tsl/test/sql/cagg_migrate_timestamptz_dist_ht.sql b/tsl/test/sql/cagg_migrate_timestamptz_dist_ht.sql new file mode 100644 index 00000000000..3a3ffa41358 --- /dev/null +++ b/tsl/test/sql/cagg_migrate_timestamptz_dist_ht.sql @@ -0,0 +1,34 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +------------------------------------ +-- Set up a distributed environment +------------------------------------ +\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER + +\set DATA_NODE_1 :TEST_DBNAME _1 +\set DATA_NODE_2 :TEST_DBNAME _2 +\set DATA_NODE_3 :TEST_DBNAME _3 + +\ir include/remote_exec.sql + +SELECT node_name, database, node_created, database_created, extension_created +FROM ( + SELECT (add_data_node(name, host => 'localhost', DATABASE => name)).* + FROM (VALUES (:'DATA_NODE_1'), (:'DATA_NODE_2'), (:'DATA_NODE_3')) v(name) +) a; + +GRANT USAGE ON FOREIGN SERVER :DATA_NODE_1, :DATA_NODE_2, :DATA_NODE_3 TO PUBLIC; + +\set IS_DISTRIBUTED TRUE +\set IS_TIME_DIMENSION TRUE +\set TIME_DIMENSION_DATATYPE TIMESTAMPTZ + +\ir include/cagg_migrate_common.sql + +-- cleanup +\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER; +DROP DATABASE :DATA_NODE_1; +DROP DATABASE :DATA_NODE_2; +DROP DATABASE :DATA_NODE_3; diff --git a/tsl/test/sql/include/cagg_migrate_common.sql b/tsl/test/sql/include/cagg_migrate_common.sql index 2eaa86f9647..54b927d6486 100644 --- a/tsl/test/sql/include/cagg_migrate_common.sql +++ b/tsl/test/sql/include/cagg_migrate_common.sql @@ -10,14 +10,8 @@ \echo 'Running local hypertable tests' \endif -\if :IS_TIME_DIMENSION - \set TIME_DATATYPE TIMESTAMPTZ -\else - \set TIME_DATATYPE INTEGER -\endif - CREATE TABLE conditions ( - "time" :TIME_DATATYPE NOT NULL, + "time" :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC ); @@ -42,18 +36,17 @@ CREATE TABLE conditions ( 0.25; \else CREATE OR REPLACE FUNCTION integer_now() - RETURNS integer LANGUAGE SQL STABLE AS + RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$; \if :IS_DISTRIBUTED - CALL distributed_exec ( - $DIST$ - CREATE OR REPLACE FUNCTION integer_now() RETURNS integer LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$; - $DIST$ - ); + SELECT + 'CREATE OR REPLACE FUNCTION integer_now() RETURNS '||:'TIME_DIMENSION_DATATYPE'||' LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM public.conditions $$;' AS "STMT" + \gset + CALL distributed_exec (:'STMT'); \endif SELECT set_integer_now_func('conditions', 'integer_now');