From ef80c3d63672e41d8c0218982dcaedae57409cbc Mon Sep 17 00:00:00 2001 From: Mats Kindahl Date: Tue, 7 Jul 2020 10:00:30 +0200 Subject: [PATCH] Add update test for repair table script This commit creates an update repair test that breaks a few tables for pre-2.0 versions to ensure that the repair script actually fixes them. The integrity check for the update tests already contain a check that dimension slices are valid, so there is no need to add a test for that. Fixes #2824 --- scripts/test_update_from_tag.sh | 10 ++ sql/updates/2.0.0-rc1--2.0.0-rc2.sql | 16 +-- test/sql/updates/setup.repair.sql | 163 +++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 8 deletions(-) create mode 100644 test/sql/updates/setup.repair.sql diff --git a/scripts/test_update_from_tag.sh b/scripts/test_update_from_tag.sh index 30aa3c4a190..34bfaf5992a 100755 --- a/scripts/test_update_from_tag.sh +++ b/scripts/test_update_from_tag.sh @@ -179,6 +179,16 @@ echo "Executing setup script on container running ${UPDATE_FROM_IMAGE}:${UPDATE_ docker_pgscript ${CONTAINER_ORIG} /src/test/sql/updates/setup.${TEST_VERSION}.sql docker_pgcmd ${CONTAINER_ORIG} "CHECKPOINT;" +# We only break constraints on the version that we will updated and +# not on the clean rerun below since it is not possible to restore +# from a dump when constraints are broken. +if [[ "${TEST_VERSION}" > "v6" ]] || [[ "${TEST_VERSION}" = "v6" ]]; then + echo "Executing repair setup script" + docker_pgscript ${CONTAINER_ORIG} /src/test/sql/updates/setup.repair.sql "single" +fi + +docker_pgcmd ${CONTAINER_ORIG} "CHECKPOINT;" + # Remove container but keep volume docker rm -f ${CONTAINER_ORIG} diff --git a/sql/updates/2.0.0-rc1--2.0.0-rc2.sql b/sql/updates/2.0.0-rc1--2.0.0-rc2.sql index 83d4a1ba245..9f5ee45f30b 100644 --- a/sql/updates/2.0.0-rc1--2.0.0-rc2.sql +++ b/sql/updates/2.0.0-rc1--2.0.0-rc2.sql @@ -137,28 +137,28 @@ SELECT DISTINCT dimension_id, CASE WHEN column_type = 'timestamptz'::regtype THEN - EXTRACT(EPOCH FROM range_start::timestamptz) * 1000000 + EXTRACT(EPOCH FROM range_start::timestamptz)::bigint * 1000000 WHEN column_type = 'timestamp'::regtype THEN - EXTRACT(EPOCH FROM range_start::timestamp) * 1000000 + EXTRACT(EPOCH FROM range_start::timestamp)::bigint * 1000000 WHEN column_type = 'date'::regtype THEN - EXTRACT(EPOCH FROM range_start::date) * 1000000 + EXTRACT(EPOCH FROM range_start::date)::bigint * 1000000 ELSE CASE WHEN range_start IS NULL - THEN -9223372036854775808 + THEN (-9223372036854775808)::bigint ELSE range_start::bigint END END AS range_start, CASE WHEN column_type = 'timestamptz'::regtype THEN - EXTRACT(EPOCH FROM range_end::timestamptz) * 1000000 + EXTRACT(EPOCH FROM range_end::timestamptz)::bigint * 1000000 WHEN column_type = 'timestamp'::regtype THEN - EXTRACT(EPOCH FROM range_end::timestamp) * 1000000 + EXTRACT(EPOCH FROM range_end::timestamp)::bigint * 1000000 WHEN column_type = 'date'::regtype THEN - EXTRACT(EPOCH FROM range_end::date) * 1000000 + EXTRACT(EPOCH FROM range_end::date)::bigint * 1000000 ELSE CASE WHEN range_end IS NULL - THEN 9223372036854775807 + THEN 9223372036854775807::bigint ELSE range_end::bigint END END AS range_end diff --git a/test/sql/updates/setup.repair.sql b/test/sql/updates/setup.repair.sql new file mode 100644 index 00000000000..b17a8172c77 --- /dev/null +++ b/test/sql/updates/setup.repair.sql @@ -0,0 +1,163 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +-- Test file to check that the repair script works. It will create a +-- bunch of tables and "break" them by removing dimension slices from +-- the dimension slice table. The repair script should then repair all +-- of them and there should be no dimension slices missing. + +SELECT (extversion BETWEEN '1.7.2' AND '2.0.0') AS runs_repair_script + FROM pg_extension + WHERE extname = 'timescaledb' \gset + +\if :runs_repair_script +CREATE TABLE repair_test_int(time integer not null, temp float8, tag integer, color integer); +CREATE TABLE repair_test_timestamptz(time timestamptz not null, temp float8, tag integer, color integer); +CREATE TABLE repair_test_timestamp(time timestamp not null, temp float8, tag integer, color integer); +CREATE TABLE repair_test_date(time date not null, temp float8, tag integer, color integer); + +SELECT create_hypertable('repair_test_int', 'time', 'tag', 2, chunk_time_interval => '3'::bigint); +SELECT create_hypertable('repair_test_timestamptz', 'time', 'tag', 2, chunk_time_interval => '1 day'::interval); +SELECT create_hypertable('repair_test_timestamp', 'time', 'tag', 2, chunk_time_interval => '1 day'::interval); +SELECT create_hypertable('repair_test_date', 'time', 'tag', 2, chunk_time_interval => '1 day'::interval); + +-- These rows will create four constraints for each table. +INSERT INTO repair_test_int VALUES + (4, 24.3, 1, 1), + (4, 24.3, 2, 1), + (10, 24.3, 2, 1); + +INSERT INTO repair_test_timestamptz VALUES + ('2020-01-01 10:11:12', 24.3, 1, 1), + ('2020-01-01 10:11:13', 24.3, 2, 1), + ('2020-01-02 10:11:14', 24.3, 2, 1); + +INSERT INTO repair_test_timestamp VALUES + ('2020-01-01 10:11:12', 24.3, 1, 1), + ('2020-01-01 10:11:13', 24.3, 2, 1), + ('2020-01-02 10:11:14', 24.3, 2, 1); + +INSERT INTO repair_test_date VALUES + ('2020-01-01 10:11:12', 24.3, 1, 1), + ('2020-01-01 10:11:13', 24.3, 2, 1), + ('2020-01-02 10:11:14', 24.3, 2, 1); + +ALTER TABLE _timescaledb_catalog.chunk_constraint + DROP CONSTRAINT chunk_constraint_dimension_slice_id_fkey; + +CREATE TABLE dimension_slice_copy (LIKE _timescaledb_catalog.dimension_slice); + +CREATE VIEW slices AS ( + SELECT hypertable_id, + ( + SELECT format('%I.%I', schema_name, table_name)::regclass + FROM _timescaledb_catalog.hypertable ht + WHERE ht.id = ch.hypertable_id + ) AS hypertable, + chunk_id, + dimension_slice_id, + constraint_name, + attname AS column_name, + pg_get_expr(conbin, conrelid) AS constraint_expr + FROM _timescaledb_catalog.chunk_constraint cc + JOIN _timescaledb_catalog.chunk ch ON cc.chunk_id = ch.id + JOIN pg_constraint ON conname = constraint_name + JOIN pg_namespace ns ON connamespace = ns.oid AND ns.nspname = ch.schema_name + JOIN pg_attribute ON attnum = conkey[1] AND attrelid = conrelid + ); + +-- Break the first dimension on each table, which is one of the time +-- constraints for the table. These are different depending on the +-- time type for the table. +DELETE FROM _timescaledb_catalog.dimension_slice WHERE id IN ( + SELECT dimension_slice_id FROM slices + WHERE hypertable = 'repair_test_int'::regclass AND column_name = 'time' + ORDER BY dimension_slice_id LIMIT 1 +); + +DELETE FROM _timescaledb_catalog.dimension_slice WHERE id IN ( + SELECT dimension_slice_id FROM slices + WHERE hypertable = 'repair_test_timestamp'::regclass AND column_name = 'time' + ORDER BY dimension_slice_id LIMIT 1 +); + +DELETE FROM _timescaledb_catalog.dimension_slice WHERE id IN ( + SELECT dimension_slice_id FROM slices + WHERE hypertable = 'repair_test_timestamptz'::regclass AND column_name = 'time' + ORDER BY dimension_slice_id LIMIT 1 +); + +DELETE FROM _timescaledb_catalog.dimension_slice WHERE id IN ( + SELECT dimension_slice_id FROM slices + WHERE hypertable = 'repair_test_date'::regclass AND column_name = 'time' + ORDER BY dimension_slice_id LIMIT 1 +); + +-- Break the partition constraints on some of the tables. The +-- partition constraints look the same in all tables so we create a +-- mix of tables with just one missing dimension slice and several +-- missing dimension slices. +DELETE FROM _timescaledb_catalog.dimension_slice WHERE id IN ( + SELECT dimension_slice_id FROM slices + WHERE hypertable = 'repair_test_timestamp'::regclass AND column_name = 'tag' + ORDER BY dimension_slice_id LIMIT 1 +); + +DELETE FROM _timescaledb_catalog.dimension_slice WHERE id IN ( + SELECT dimension_slice_id FROM slices + WHERE hypertable = 'repair_test_date'::regclass AND column_name = 'tag' + ORDER BY dimension_slice_id +); + +CREATE VIEW unparsed_slices AS +SELECT di.id AS dimension_id, + dimension_slice_id, + constraint_name, + column_type, + column_name, + (SELECT SUBSTRING(constraint_expr, $$>=\s*'?([\w\d\s:+-]+)'?$$)) AS range_start, + (SELECT SUBSTRING(constraint_expr, $$<\s*'?([\w\d\s:+-]+)'?$$)) AS range_end + FROM slices JOIN _timescaledb_catalog.dimension di USING (hypertable_id, column_name); + +INSERT INTO dimension_slice_copy +SELECT DISTINCT + dimension_slice_id, + dimension_id, + CASE + WHEN column_type = 'timestamptz'::regtype THEN + EXTRACT(EPOCH FROM range_start::timestamptz)::bigint * 1000000 + WHEN column_type = 'timestamp'::regtype THEN + EXTRACT(EPOCH FROM range_start::timestamp)::bigint * 1000000 + WHEN column_type = 'date'::regtype THEN + EXTRACT(EPOCH FROM range_start::date)::bigint * 1000000 + ELSE + CASE + WHEN range_start IS NULL + THEN (-9223372036854775808)::bigint + ELSE range_start::bigint + END + END AS range_start, + CASE + WHEN column_type = 'timestamptz'::regtype THEN + EXTRACT(EPOCH FROM range_end::timestamptz)::bigint * 1000000 + WHEN column_type = 'timestamp'::regtype THEN + EXTRACT(EPOCH FROM range_end::timestamp)::bigint * 1000000 + WHEN column_type = 'date'::regtype THEN + EXTRACT(EPOCH FROM range_end::date)::bigint * 1000000 + ELSE + CASE WHEN range_end IS NULL + THEN 9223372036854775807::bigint + ELSE range_end::bigint + END + END AS range_end + FROM unparsed_slices + WHERE dimension_slice_id NOT IN (SELECT id FROM _timescaledb_catalog.dimension_slice); + +\echo **** Expected repairs **** +SELECT * FROM dimension_slice_copy; +\endif + +DROP VIEW unparsed_slices; +DROP VIEW slices; +DROP TABLE dimension_slice_copy;