Pass join related structs to the cagg rte

In case of joins in the continuous aggregates, pass the required
structs to the new rte created. These values are required by the
planner to finally query the materialized view.

Fixes #5433

CHANGELOG.md

@@ -33,12 +33,14 @@ accidentally triggering the load of a previous DB version.**
 * #5497 Allow named time_bucket arguments in Cagg definition
 * #5499 Do not segfault on large histogram() parameters
 * #5500 Fix when no FROM clause in continuous aggregate definition
+* #5433 Fix join rte in CAggs with joins
 
 **Thanks**
 * @nikolaps for reporting an issue with the COPY fetcher
 * @S-imo-n for reporting the issue on Background Worker Scheduler crash
 * @kovetskiy and @DZDomi for reporting peformance regression in Realtime Continuous Aggregates
 * @geezhu for reporting issue on segfault in historgram()
+* @mwahlthuetter for reporting the issue with joins in CAggs
 
 ## 2.10.1 (2023-03-07)
 

scripts/test_repair_from_tag.sh

@@ -15,7 +15,7 @@ UPDATE_TO_TAG=${UPDATE_TO_TAG:-${GIT_ID}}
 
 # Extra options to pass to psql
 PGOPTS="-v TEST_VERSION=${TEST_VERSION} -v TEST_REPAIR=true -v WITH_SUPERUSER=${WITH_SUPERUSER} -v WITH_ROLES=false"
-PSQL="psql -qX $PGOPTS"
+PSQL="psql -X $PGOPTS"
 
 DOCKEROPTS="--env TIMESCALEDB_TELEMETRY=off --env POSTGRES_HOST_AUTH_METHOD=trust"
 

scripts/test_updates_pg13.sh

@@ -12,5 +12,8 @@ run_tests "$@" -v7 \
           2.4.0-pg13 2.4.1-pg13 2.4.2-pg13
 run_tests "$@" -v8 \
           2.5.0-pg13 2.5.1-pg13 2.5.2-pg13 2.6.0-pg13 2.6.1-pg13 2.7.0-pg13 2.7.1-pg13 2.7.2-pg13 \
-          2.8.0-pg13 2.8.1-pg13 2.9.0-pg13 2.9.1-pg13 2.9.2-pg13 2.9.3-pg13 2.10.0-pg13 2.10.1-pg13
+          2.8.0-pg13 2.8.1-pg13 2.9.0-pg13 2.9.1-pg13 2.9.2-pg13 2.9.3-pg13
 
+# Also run repair tests for >=2.10.x versions due to PR #5441
+run_tests "$@" -r -v8 \
+          2.10.0-pg13 2.10.1-pg13

scripts/test_updates_pg14.sh

@@ -11,5 +11,9 @@ run_tests "$@" -v7 \
           2.5.0-pg14 2.5.1-pg14
 run_tests "$@" -v8 \
           2.5.0-pg14 2.5.1-pg14 2.5.2-pg14 2.6.0-pg14 2.6.1-pg14 2.7.0-pg14 2.7.1-pg14 2.7.2-pg14 \
-          2.8.0-pg14 2.8.1-pg14 2.9.0-pg14 2.9.1-pg14 2.9.2-pg14 2.9.3-pg14 2.10.0-pg14 2.10.1-pg14
+          2.8.0-pg14 2.8.1-pg14 2.9.0-pg14 2.9.1-pg14 2.9.2-pg14 2.9.3-pg14
+
+# Also run repair tests for >=2.10.x versions due to PR #5441
+run_tests "$@" -r -v8 \
+          2.10.0-pg14 2.10.1-pg14
 

scripts/test_updates_pg15.sh

@@ -8,4 +8,8 @@ SCRIPT_DIR=$(dirname $0)
 source ${SCRIPT_DIR}/test_functions.inc
 
 run_tests "$@" -v8 \
-          2.9.0-pg15 2.9.1-pg15 2.9.2-pg15 2.9.3-pg15 2.10.0-pg15 2.10.1-pg15
+          2.9.0-pg15 2.9.1-pg15 2.9.2-pg15 2.9.3-pg15
+
+# Also run repair tests for >=2.10.x versions due to PR #5441
+run_tests "$@" -r -v8 \
+          2.10.0-pg15 2.10.1-pg15

sql/updates/post-update.sql

@@ -8,7 +8,7 @@ BEGIN
     SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
     IF ts_version >= '2.7.0' THEN
             CREATE PROCEDURE _timescaledb_internal.post_update_cagg_try_repair(
-                cagg_view REGCLASS
+                cagg_view REGCLASS, force_rebuild boolean
             ) AS '@MODULE_PATHNAME@', 'ts_cagg_try_repair' LANGUAGE C;
     END IF;
     FOR vname, materialized_only IN select format('%I.%I', cagg.user_view_schema, cagg.user_view_name)::regclass, cagg.materialized_only from _timescaledb_catalog.continuous_agg cagg
@@ -26,7 +26,7 @@ BEGIN
             EXECUTE format('ALTER MATERIALIZED VIEW %s SET (timescaledb.materialized_only=%L) ', vname::text, materialized_only);
         ELSE
             SET log_error_verbosity TO VERBOSE;
-            CALL _timescaledb_internal.post_update_cagg_try_repair(vname);
+            CALL _timescaledb_internal.post_update_cagg_try_repair(vname, false);
         END IF;
     END LOOP;
     IF ts_version >= '2.7.0' THEN
@@ -36,6 +36,33 @@ BEGIN
 END
 $$;
 
+-- For tsdb >= v2.10.0 apply the cagg repair when necessary
+DO $$
+DECLARE
+ vname regclass;
+ materialized_only bool;
+ ts_version TEXT;
+BEGIN
+    SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
+     IF ts_version >= '2.10.0' THEN
+	        CREATE PROCEDURE _timescaledb_internal.post_update_cagg_try_repair(
+	            cagg_view REGCLASS, force_rebuild BOOLEAN
+	        ) AS '@MODULE_PATHNAME@', 'ts_cagg_try_repair' LANGUAGE C;
+
+	        FOR vname, materialized_only IN select format('%I.%I', cagg.user_view_schema, cagg.user_view_name)::regclass, cagg.materialized_only from _timescaledb_catalog.continuous_agg cagg
+	        LOOP
+	            IF ts_version >= '2.10.0' THEN
+	                SET log_error_verbosity TO VERBOSE;
+	                CALL _timescaledb_internal.post_update_cagg_try_repair(vname, true);
+	            END IF;
+	        END LOOP;
+
+	        DROP PROCEDURE IF EXISTS _timescaledb_internal.post_update_cagg_try_repair(REGCLASS, BOOLEAN);
+	    END IF;
+    EXCEPTION WHEN OTHERS THEN RAISE;
+END
+$$;
+
 -- can only be dropped after views have been rebuilt
 DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_watermark(oid);
 

test/sql/updates/post.repair.cagg_joins.sql

@@ -0,0 +1,24 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+SELECT current_setting('server_version_num')::int >=  130000 AS has_cagg_join_using \gset
+
+\d+ cagg_joins_upgrade_test_with_realtime
+SELECT * FROM cagg_joins_upgrade_test_with_realtime ORDER BY bucket;
+
+\d+ cagg_joins_upgrade_test
+SELECT * FROM cagg_joins_upgrade_test ORDER BY bucket;
+
+\d+ cagg_joins_where
+SELECT * FROM cagg_joins_where ORDER BY bucket;
+
+\if :has_cagg_join_using
+    \d+ cagg_joins_upgrade_test_with_realtime_using
+    SELECT * FROM cagg_joins_upgrade_test_with_realtime_using ORDER BY bucket;
+
+    \d+ cagg_joins_upgrade_test_using
+    SELECT * FROM cagg_joins_upgrade_test_using ORDER BY bucket;
+
+\endif
+

test/sql/updates/post.repair.sql

@@ -2,9 +2,26 @@
 -- Please see the included NOTICE for copyright information and
 -- LICENSE-APACHE for a copy of the license.
 
--- Re-add the dropped foreign key constraint that was dropped for
--- repair testing.
-ALTER TABLE _timescaledb_catalog.chunk_constraint
+SELECT extversion < '2.10.0' AS test_repair_dimension
+FROM pg_extension
+WHERE extname = 'timescaledb' \gset
+
+SELECT extversion >= '2.10.0' AS has_cagg_joins
+FROM pg_extension
+WHERE extname = 'timescaledb' \gset
+
+SELECT :'TEST_VERSION' >= 'v8' AS is_GTE_v8 \gset
+
+\if :test_repair_dimension
+    -- Re-add the dropped foreign key constraint that was dropped for
+    -- repair testing.
+    ALTER TABLE _timescaledb_catalog.chunk_constraint
     ADD CONSTRAINT chunk_constraint_dimension_slice_id_fkey
     FOREIGN KEY (dimension_slice_id) REFERENCES _timescaledb_catalog.dimension_slice (id);
+\endif
+
+\if :has_cagg_joins AND :is_GTE_v8
+--Check if the repaired cagg with joins work alright now
+\ir post.repair.cagg_joins.sql
+\endif
 

test/sql/updates/setup.repair.cagg_joins.sql

@@ -0,0 +1,90 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+SELECT current_setting('server_version_num')::int >=  130000 AS has_cagg_join_using \gset
+
+CREATE TABLE ht_cagg_joins(
+  day TIMESTAMPTZ NOT NULL,
+  city text NOT NULL,
+  temperature INT NOT NULL,
+device_id int NOT NULL);
+SELECT create_hypertable(
+  'ht_cagg_joins', 'day',
+  chunk_time_interval => INTERVAL '1 day'
+);
+INSERT INTO ht_cagg_joins (day, city, temperature, device_id) VALUES
+  ('2021-06-14 00:00:00-00'::timestamptz, 'Moscow', 26,1),
+  ('2021-06-15 00:00:00-00'::timestamptz, 'Moscow', 22,2),
+  ('2021-06-16 00:00:00-00'::timestamptz, 'Berlin', 24,3),
+  ('2021-06-17 00:00:00-00'::timestamptz, 'London', 24,4),
+  ('2021-06-18 00:00:00-00'::timestamptz, 'Stockholm', 27,4),
+  ('2021-06-19 00:00:00-00'::timestamptz, 'Moscow', 28,4),
+  ('2021-06-20 00:00:00-00'::timestamptz, 'Stockholm', 30,1),
+  ('2021-06-21 00:00:00-00'::timestamptz, 'London', 31,1),
+  ('2021-06-22 00:00:00-00'::timestamptz, 'Stockholm', 34,1),
+  ('2021-06-23 00:00:00-00'::timestamptz, 'Moscow', 34,2),
+  ('2021-06-24 00:00:00-00'::timestamptz, 'London', 34,2),
+  ('2021-06-25 00:00:00-00'::timestamptz, 'Stockholm', 32,3),
+  ('2021-06-26 00:00:00-00'::timestamptz, 'Berlin', 32,3),
+  ('2021-06-27 00:00:00-00'::timestamptz, 'Stockholm', 31,3);
+
+CREATE TABLE nt_cagg_joins ( device_id int not null, name text, location text);
+INSERT INTO nt_cagg_joins values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm');
+
+--Create a cagg with join between a hypertable and a normal table
+-- with equality condition on inner join type and realtime aggregation enabled
+CREATE MATERIALIZED VIEW cagg_joins_upgrade_test_with_realtime
+WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS
+SELECT time_bucket(INTERVAL '1 day', day) AS bucket,
+  AVG(temperature),
+  name
+FROM ht_cagg_joins JOIN nt_cagg_joins
+ON ht_cagg_joins.device_id = nt_cagg_joins.device_id
+GROUP BY 1,3;
+
+--Create a cagg with join between a hypertable and a normal table
+-- with equality condition on inner join type and realtime aggregation disabled
+CREATE MATERIALIZED VIEW cagg_joins_upgrade_test
+WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS
+SELECT time_bucket(INTERVAL '1 day', day) AS bucket,
+  AVG(temperature),
+  name
+FROM ht_cagg_joins JOIN nt_cagg_joins
+ON ht_cagg_joins.device_id = nt_cagg_joins.device_id
+GROUP BY 1,3;
+
+--Create a Cagg with JOIN and additional WHERE conditions
+CREATE MATERIALIZED VIEW cagg_joins_where
+WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS
+SELECT time_bucket(INTERVAL '1 day', day) AS bucket,
+  AVG(temperature),
+  name
+FROM ht_cagg_joins JOIN nt_cagg_joins
+ON ht_cagg_joins.device_id = nt_cagg_joins.device_id
+WHERE ht_cagg_joins.city = nt_cagg_joins.location
+GROUP BY 1,3;
+
+-- Only test joins with using clause for postgresql versions above 12
+\if :has_cagg_join_using
+    --Create a cagg with join with using clause and realtime aggregation enabled
+    CREATE MATERIALIZED VIEW cagg_joins_upgrade_test_with_realtime_using
+    WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS
+    SELECT time_bucket(INTERVAL '1 day', day) AS bucket,
+    AVG(temperature),
+    name
+    FROM ht_cagg_joins JOIN nt_cagg_joins
+    USING (device_id)
+    GROUP BY 1,3;
+
+    --Create a cagg with join with using and realtime aggregation disabled
+    CREATE MATERIALIZED VIEW cagg_joins_upgrade_test_using
+    WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS
+    SELECT time_bucket(INTERVAL '1 day', day) AS bucket,
+    AVG(temperature),
+    name
+    FROM ht_cagg_joins JOIN nt_cagg_joins
+    USING (device_id)
+    GROUP BY 1,3;
+
+\endif

test/sql/updates/setup.repair.sql

@@ -7,6 +7,14 @@
 -- the dimension slice table. The repair script should then repair all
 -- of them and there should be no dimension slices missing.
 
+SELECT extversion < '2.10.0' AS test_repair_dimension
+FROM pg_extension
+WHERE extname = 'timescaledb' \gset
+
+SELECT extversion >= '2.10.0' AS has_cagg_joins
+FROM pg_extension
+WHERE extname = 'timescaledb' \gset
+
 CREATE TABLE repair_test_int(time integer not null, temp float8, tag integer, color integer);
 CREATE TABLE repair_test_timestamptz(time timestamptz not null, temp float8, tag integer, color integer);
 CREATE TABLE repair_test_extra(time timestamptz not null, temp float8, tag integer, color integer);
@@ -55,10 +63,10 @@ INSERT INTO repair_test_date VALUES
 -- repaired properly, we will notice them when restoring the
 -- constraint.
 ALTER TABLE _timescaledb_catalog.chunk_constraint
-      DROP CONSTRAINT chunk_constraint_dimension_slice_id_fkey;
+    DROP CONSTRAINT chunk_constraint_dimension_slice_id_fkey;
 
 CREATE VIEW slices AS (
-      SELECT ch.hypertable_id,
+       SELECT ch.hypertable_id,
              (
                  SELECT format('%I.%I', schema_name, table_name)::regclass
                  FROM _timescaledb_catalog.hypertable ht
@@ -80,6 +88,7 @@ CREATE VIEW slices AS (
            ON di.hypertable_id = ch.hypertable_id AND attname = di.column_name
    );
 
+\if :test_repair_dimension
 -- Break the first time dimension on each table. These are different
 -- depending on the time type for the table and we need to check all
 -- versions.
@@ -143,6 +152,7 @@ WITH unparsed_slices AS (
 	   (SELECT SUBSTRING(constraint_expr, $$<\s*'?([\w\d\s:+-]+)'?$$)) AS range_end
       FROM slices
 )
+
 SELECT DISTINCT
        dimension_slice_id,
        dimension_id,
@@ -177,5 +187,10 @@ SELECT DISTINCT
   WHERE dimension_slice_id NOT IN (SELECT id FROM _timescaledb_catalog.dimension_slice);
 
 DROP VIEW slices;
+\endif
 
 \ir setup.repair.cagg.sql
+
+\if :has_cagg_joins
+     \ir setup.repair.cagg_joins.sql
+\endif