Release 2.11.0

This release contains new features and bug fixes since the 2.10.3 release. We deem it moderate priority for upgrading. This release includes these noteworthy features: * Support for DML operations on compressed chunks: * UPDATE/DELETE support * Support for unique constraints on compressed chunks * Support for `ON CONFLICT DO UPDATE` * Support for `ON CONFLICT DO NOTHING` * Join support for hierarchical Continuous Aggregates **Features** * #5212 Allow pushdown of reference table joins * #5221 Improve Realtime Continuous Aggregate performance * #5252 Improve unique constraint support on compressed hypertables * #5339 Support UPDATE/DELETE on compressed hypertables * #5344 Enable JOINS for Hierarchical Continuous Aggregates * #5361 Add parallel support for partialize_agg() * #5417 Refactor and optimize distributed COPY * #5454 Add support for ON CONFLICT DO UPDATE for compressed hypertables * #5547 Skip Ordered Append when only 1 child node is present * #5510 Propagate vacuum/analyze to compressed chunks * #5584 Reduce decompression during constraint checking * #5530 Optimize compressed chunk resorting * #5639 Support sending telemetry event reports **Bugfixes** * #5396 Fix SEGMENTBY columns predicates to be pushed down * #5427 Handle user-defined FDW options properly * #5442 Decompression may have lost DEFAULT values * #5459 Fix issue creating dimensional constraints * #5570 Improve interpolate error message on datatype mismatch * #5573 Fix unique constraint on compressed tables * #5615 Add permission checks to run_job() * #5614 Enable run_job() for telemetry job * #5578 Fix on-insert decompression after schema changes * #5613 Quote username identifier appropriately * #5525 Fix tablespace for compressed hypertable and corresponding toast * #5642 Fix ALTER TABLE SET with normal tables * #5666 Reduce memory usage for distributed analyze * #5668 Fix subtransaction resource owner **Thanks** * @kovetskiy and @DZDomi for reporting peformance regression in Realtime Continuous Aggregates * @ollz272 for reporting an issue with interpolate error messages
timescale · May 19, 2023 · bf3a8d7 · bf3a8d7
1 parent 3947c01
commit bf3a8d7
Show file tree

Hide file tree

Showing 7 changed files with 323 additions and 309 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,7 +4,19 @@
 `psql` with the `-X` flag to prevent any `.psqlrc` commands from
 accidentally triggering the load of a previous DB version.**
 
-## Unreleased
+## 2.11.0 (2023-05-19)
+
+This release contains new features and bug fixes since the 2.10.3 release.
+We deem it moderate priority for upgrading.
+
+This release includes these noteworthy features:
+* Support for DML operations on compressed chunks:
+  * UPDATE/DELETE support
+  * Support for unique constraints on compressed chunks
+  * Support for `ON CONFLICT DO UPDATE`
+  * Support for `ON CONFLICT DO NOTHING`
+* Join support for Hierarchical Continuous Aggregates
+* Performance improvements for real-time Hierarchical Continuous Aggregates
 
 **Features**
 * #5212 Allow pushdown of reference table joins

diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
@@ -47,11 +47,12 @@ set(MOD_FILES
     updates/2.9.3--2.10.0.sql
     updates/2.10.0--2.10.1.sql
     updates/2.10.1--2.10.2.sql
-    updates/2.10.2--2.10.3.sql)
+    updates/2.10.2--2.10.3.sql
+    updates/2.10.3--2.11.0.sql)
 
 # The downgrade file to generate a downgrade script for the current version, as
 # specified in version.config
-set(CURRENT_REV_FILE reverse-dev.sql)
+set(CURRENT_REV_FILE 2.11.0--2.10.3.sql)
 # Files for generating old downgrade scripts. This should only include files for
 # downgrade from one version to its previous version since we do not support
 # skipping versions when downgrading.
@@ -77,7 +78,8 @@ set(OLD_REV_FILES
     2.10.0--2.9.3.sql
     2.10.1--2.10.0.sql
     2.10.2--2.10.1.sql
-    2.10.3--2.10.2.sql)
+    2.10.3--2.10.2.sql
+    2.11.0--2.10.3.sql)
 
 set(MODULE_PATHNAME "$libdir/timescaledb-${PROJECT_VERSION_MOD}")
 set(LOADER_PATHNAME "$libdir/timescaledb")

diff --git a/sql/updates/2.10.3--2.11.0.sql b/sql/updates/2.10.3--2.11.0.sql
@@ -0,0 +1,72 @@
+CREATE TABLE _timescaledb_catalog.continuous_aggs_watermark (
+  mat_hypertable_id integer NOT NULL,
+  watermark bigint NOT NULL,
+  -- table constraints
+  CONSTRAINT continuous_aggs_watermark_pkey PRIMARY KEY (mat_hypertable_id),
+  CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE
+);
+
+GRANT SELECT ON _timescaledb_catalog.continuous_aggs_watermark TO PUBLIC;
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_watermark', '');
+
+CREATE FUNCTION _timescaledb_internal.cagg_watermark_materialized(hypertable_id INTEGER)
+RETURNS INT8 AS '@MODULE_PATHNAME@', 'ts_continuous_agg_watermark_materialized' LANGUAGE C STABLE STRICT PARALLEL SAFE;
+CREATE FUNCTION _timescaledb_internal.recompress_chunk_segmentwise(REGCLASS, BOOLEAN) RETURNS REGCLASS
+AS '@MODULE_PATHNAME@', 'ts_recompress_chunk_segmentwise' LANGUAGE C STRICT VOLATILE;
+CREATE FUNCTION _timescaledb_internal.get_compressed_chunk_index_for_recompression(REGCLASS) RETURNS REGCLASS
+AS '@MODULE_PATHNAME@', 'ts_get_compressed_chunk_index_for_recompression' LANGUAGE C STRICT VOLATILE;
+
+DROP FUNCTION _timescaledb_internal.dimension_is_finite;
+DROP FUNCTION _timescaledb_internal.dimension_slice_get_constraint_sql;
+
+CREATE SCHEMA _timescaledb_functions;
+GRANT USAGE ON SCHEMA _timescaledb_functions TO PUBLIC;
+
+-- migrate histogram support functions into _timescaledb_functions schema
+ALTER FUNCTION _timescaledb_internal.hist_sfunc (state INTERNAL, val DOUBLE PRECISION, MIN DOUBLE PRECISION, MAX DOUBLE PRECISION, nbuckets INTEGER) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.hist_combinefunc(state1 INTERNAL, state2 INTERNAL) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.hist_serializefunc(INTERNAL) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.hist_deserializefunc(bytea, INTERNAL) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.hist_finalfunc(state INTERNAL, val DOUBLE PRECISION, MIN DOUBLE PRECISION, MAX DOUBLE PRECISION, nbuckets INTEGER) SET SCHEMA _timescaledb_functions;
+
+-- migrate first/last support functions into _timescaledb_functions schema
+ALTER FUNCTION _timescaledb_internal.first_sfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.first_combinefunc(internal, internal) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.last_sfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.last_combinefunc(internal, internal) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.bookend_finalfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.bookend_serializefunc(internal) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.bookend_deserializefunc(bytea, internal) SET SCHEMA _timescaledb_functions;
+
+DROP FUNCTION IF EXISTS _timescaledb_internal.is_main_table(regclass);
+DROP FUNCTION IF EXISTS _timescaledb_internal.is_main_table(name, name);
+DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_from_main_table(regclass);
+DROP FUNCTION IF EXISTS _timescaledb_internal.main_table_from_hypertable(integer);
+DROP FUNCTION IF EXISTS _timescaledb_internal.time_literal_sql(bigint, regtype);
+
+ALTER FUNCTION _timescaledb_internal.compressed_data_in(CSTRING) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.compressed_data_out(_timescaledb_internal.compressed_data) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.compressed_data_send(_timescaledb_internal.compressed_data) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.compressed_data_recv(internal) SET SCHEMA _timescaledb_functions;
+
+ALTER FUNCTION _timescaledb_internal.rxid_in(cstring) SET SCHEMA _timescaledb_functions;
+ALTER FUNCTION _timescaledb_internal.rxid_out(@extschema@.rxid) SET SCHEMA _timescaledb_functions;
+
+ALTER TABLE _timescaledb_config.bgw_job
+    ALTER COLUMN owner SET DEFAULT pg_catalog.quote_ident(current_role)::regrole;
+
+ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan
+  ADD COLUMN user_view_definition TEXT,
+  DROP CONSTRAINT continuous_agg_migrate_plan_mat_hypertable_id_fkey;
+
+-- Log with events that will be sent out with the telemetry. The log
+-- will be flushed after it has been sent out. We do not save it to
+-- backups since it should not contain important data.
+CREATE TABLE _timescaledb_catalog.telemetry_event (
+       created timestamptz NOT NULL DEFAULT current_timestamp,
+       tag name NOT NULL,
+       body jsonb NOT NULL
+);
+
+GRANT SELECT ON _timescaledb_catalog.telemetry_event TO PUBLIC;
diff --git a/sql/updates/2.11.0--2.10.3.sql b/sql/updates/2.11.0--2.10.3.sql
@@ -0,0 +1,232 @@
+DROP FUNCTION IF EXISTS _timescaledb_internal.get_approx_row_count(REGCLASS);
+
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_watermark;
+
+DROP TABLE IF EXISTS _timescaledb_catalog.continuous_aggs_watermark;
+
+DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_watermark_materialized(hypertable_id INTEGER);
+DROP FUNCTION _timescaledb_internal.recompress_chunk_segmentwise(REGCLASS, BOOLEAN);
+DROP FUNCTION _timescaledb_internal.get_compressed_chunk_index_for_recompression(REGCLASS);
+
+CREATE OR REPLACE FUNCTION _timescaledb_internal.dimension_is_finite(
+    val      BIGINT
+)
+    RETURNS BOOLEAN LANGUAGE SQL IMMUTABLE PARALLEL SAFE AS
+$BODY$
+    --end values of bigint reserved for infinite
+    SELECT val > (-9223372036854775808)::bigint AND val < 9223372036854775807::bigint
+$BODY$ SET search_path TO pg_catalog, pg_temp;
+
+CREATE OR REPLACE FUNCTION _timescaledb_internal.dimension_slice_get_constraint_sql(
+    dimension_slice_id  INTEGER
+)
+    RETURNS TEXT LANGUAGE PLPGSQL VOLATILE AS
+$BODY$
+DECLARE
+    dimension_slice_row _timescaledb_catalog.dimension_slice;
+    dimension_row _timescaledb_catalog.dimension;
+    dimension_def TEXT;
+    dimtype REGTYPE;
+    parts TEXT[];
+BEGIN
+    SELECT * INTO STRICT dimension_slice_row
+    FROM _timescaledb_catalog.dimension_slice
+    WHERE id = dimension_slice_id;
+
+    SELECT * INTO STRICT dimension_row
+    FROM _timescaledb_catalog.dimension
+    WHERE id = dimension_slice_row.dimension_id;
+
+    IF dimension_row.partitioning_func_schema IS NOT NULL AND
+       dimension_row.partitioning_func IS NOT NULL THEN
+        SELECT prorettype INTO STRICT dimtype
+        FROM pg_catalog.pg_proc pro
+        WHERE pro.oid = format('%I.%I', dimension_row.partitioning_func_schema, dimension_row.partitioning_func)::regproc::oid;
+
+        dimension_def := format('%1$I.%2$I(%3$I)',
+             dimension_row.partitioning_func_schema,
+             dimension_row.partitioning_func,
+             dimension_row.column_name);
+    ELSE
+        dimension_def := format('%1$I', dimension_row.column_name);
+        dimtype := dimension_row.column_type;
+    END IF;
+
+    IF dimension_row.num_slices IS NOT NULL THEN
+
+        IF  _timescaledb_internal.dimension_is_finite(dimension_slice_row.range_start) THEN
+            parts = parts || format(' %1$s >= %2$L ', dimension_def, dimension_slice_row.range_start);
+        END IF;
+
+        IF _timescaledb_internal.dimension_is_finite(dimension_slice_row.range_end) THEN
+            parts = parts || format(' %1$s < %2$L ', dimension_def, dimension_slice_row.range_end);
+        END IF;
+
+        IF array_length(parts, 1) = 0 THEN
+            RETURN NULL;
+        END IF;
+        return array_to_string(parts, 'AND');
+    ELSE
+        -- only works with time for now
+        IF _timescaledb_internal.time_literal_sql(dimension_slice_row.range_start, dimtype) =
+           _timescaledb_internal.time_literal_sql(dimension_slice_row.range_end, dimtype) THEN
+            RAISE 'time-based constraints have the same start and end values for column "%": %',
+                    dimension_row.column_name,
+                    _timescaledb_internal.time_literal_sql(dimension_slice_row.range_end, dimtype);
+        END IF;
+
+        parts = ARRAY[]::text[];
+
+        IF _timescaledb_internal.dimension_is_finite(dimension_slice_row.range_start) THEN
+            parts = parts || format(' %1$s >= %2$s ',
+            dimension_def,
+            _timescaledb_internal.time_literal_sql(dimension_slice_row.range_start, dimtype));
+        END IF;
+
+        IF _timescaledb_internal.dimension_is_finite(dimension_slice_row.range_end) THEN
+            parts = parts || format(' %1$s < %2$s ',
+            dimension_def,
+            _timescaledb_internal.time_literal_sql(dimension_slice_row.range_end, dimtype));
+        END IF;
+
+        return array_to_string(parts, 'AND');
+    END IF;
+END
+$BODY$ SET search_path TO pg_catalog, pg_temp;
+
+ALTER FUNCTION _timescaledb_functions.hist_sfunc (state INTERNAL, val DOUBLE PRECISION, MIN DOUBLE PRECISION, MAX DOUBLE PRECISION, nbuckets INTEGER) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.hist_combinefunc(state1 INTERNAL, state2 INTERNAL) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.hist_serializefunc(INTERNAL) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.hist_deserializefunc(bytea, INTERNAL) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.hist_finalfunc(state INTERNAL, val DOUBLE PRECISION, MIN DOUBLE PRECISION, MAX DOUBLE PRECISION, nbuckets INTEGER) SET SCHEMA _timescaledb_internal;
+
+ALTER FUNCTION _timescaledb_functions.first_sfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.first_combinefunc(internal, internal) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.last_sfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.last_combinefunc(internal, internal) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.bookend_finalfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.bookend_serializefunc(internal) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.bookend_deserializefunc(bytea, internal) SET SCHEMA _timescaledb_internal;
+
+ALTER FUNCTION _timescaledb_functions.compressed_data_in(CSTRING) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.compressed_data_out(_timescaledb_internal.compressed_data) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.compressed_data_send(_timescaledb_internal.compressed_data) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.compressed_data_recv(internal) SET SCHEMA _timescaledb_internal;
+
+ALTER FUNCTION _timescaledb_functions.rxid_in(cstring) SET SCHEMA _timescaledb_internal;
+ALTER FUNCTION _timescaledb_functions.rxid_out(@extschema@.rxid) SET SCHEMA _timescaledb_internal;
+
+DROP SCHEMA _timescaledb_functions;
+
+CREATE FUNCTION _timescaledb_internal.is_main_table(
+    table_oid regclass
+)
+    RETURNS bool LANGUAGE SQL STABLE AS
+$BODY$
+    SELECT EXISTS(SELECT 1 FROM _timescaledb_catalog.hypertable WHERE table_name = relname AND schema_name = nspname)
+    FROM pg_class c
+    INNER JOIN pg_namespace n ON (n.OID = c.relnamespace)
+    WHERE c.OID = table_oid;
+$BODY$ SET search_path TO pg_catalog, pg_temp;
+
+-- Check if given table is a hypertable's main table
+CREATE FUNCTION _timescaledb_internal.is_main_table(
+    schema_name NAME,
+    table_name  NAME
+)
+    RETURNS BOOLEAN LANGUAGE SQL STABLE AS
+$BODY$
+     SELECT EXISTS(
+         SELECT 1 FROM _timescaledb_catalog.hypertable h
+         WHERE h.schema_name = is_main_table.schema_name AND
+               h.table_name = is_main_table.table_name
+     );
+$BODY$ SET search_path TO pg_catalog, pg_temp;
+
+-- Get a hypertable given its main table OID
+CREATE FUNCTION _timescaledb_internal.hypertable_from_main_table(
+    table_oid regclass
+)
+    RETURNS _timescaledb_catalog.hypertable LANGUAGE SQL STABLE AS
+$BODY$
+    SELECT h.*
+    FROM pg_class c
+    INNER JOIN pg_namespace n ON (n.OID = c.relnamespace)
+    INNER JOIN _timescaledb_catalog.hypertable h ON (h.table_name = c.relname AND h.schema_name = n.nspname)
+    WHERE c.OID = table_oid;
+$BODY$ SET search_path TO pg_catalog, pg_temp;
+
+CREATE FUNCTION _timescaledb_internal.main_table_from_hypertable(
+    hypertable_id int
+)
+    RETURNS regclass LANGUAGE SQL STABLE AS
+$BODY$
+    SELECT format('%I.%I',h.schema_name, h.table_name)::regclass
+    FROM _timescaledb_catalog.hypertable h
+    WHERE id = hypertable_id;
+$BODY$ SET search_path TO pg_catalog, pg_temp;
+
+-- Gets the sql code for representing the literal for the given time value (in the internal representation) as the column_type.
+CREATE FUNCTION _timescaledb_internal.time_literal_sql(
+    time_value      BIGINT,
+    column_type     REGTYPE
+)
+    RETURNS text LANGUAGE PLPGSQL STABLE AS
+$BODY$
+DECLARE
+    ret text;
+BEGIN
+    IF time_value IS NULL THEN
+        RETURN format('%L', NULL);
+    END IF;
+    CASE column_type
+      WHEN 'BIGINT'::regtype, 'INTEGER'::regtype, 'SMALLINT'::regtype THEN
+        RETURN format('%L', time_value); -- scale determined by user.
+      WHEN 'TIMESTAMP'::regtype THEN
+        --the time_value for timestamps w/o tz does not depend on local timezones. So perform at UTC.
+        RETURN format('TIMESTAMP %1$L', timezone('UTC',_timescaledb_internal.to_timestamp(time_value))); -- microseconds
+      WHEN 'TIMESTAMPTZ'::regtype THEN
+        -- assume time_value is in microsec
+        RETURN format('TIMESTAMPTZ %1$L', _timescaledb_internal.to_timestamp(time_value)); -- microseconds
+      WHEN 'DATE'::regtype THEN
+        RETURN format('%L', timezone('UTC',_timescaledb_internal.to_timestamp(time_value))::date);
+      ELSE
+         EXECUTE 'SELECT format(''%L'', $1::' || column_type::text || ')' into ret using time_value;
+         RETURN ret;
+    END CASE;
+END
+$BODY$ SET search_path TO pg_catalog, pg_temp;
+
+ALTER TABLE _timescaledb_config.bgw_job
+    ALTER COLUMN owner SET DEFAULT current_role::regrole;
+
+-- Rebuild the _timescaledb_catalog.continuous_agg_migrate_plan_step definition
+ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step
+    DROP CONSTRAINT continuous_agg_migrate_plan_step_mat_hypertable_id_fkey;
+
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg_migrate_plan;
+CREATE TABLE _timescaledb_catalog._tmp_continuous_agg_migrate_plan AS SELECT mat_hypertable_id, start_ts, end_ts FROM _timescaledb_catalog.continuous_agg_migrate_plan;
+DROP TABLE _timescaledb_catalog.continuous_agg_migrate_plan;
+
+CREATE TABLE _timescaledb_catalog.continuous_agg_migrate_plan (
+  mat_hypertable_id integer NOT NULL,
+  start_ts TIMESTAMPTZ NOT NULL DEFAULT pg_catalog.now(),
+  end_ts TIMESTAMPTZ,
+  -- table constraints
+  CONSTRAINT continuous_agg_migrate_plan_pkey PRIMARY KEY (mat_hypertable_id),
+  CONSTRAINT continuous_agg_migrate_plan_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id)
+);
+
+INSERT INTO _timescaledb_catalog.continuous_agg_migrate_plan SELECT * FROM _timescaledb_catalog._tmp_continuous_agg_migrate_plan;
+DROP TABLE _timescaledb_catalog._tmp_continuous_agg_migrate_plan;
+
+ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step
+    ADD CONSTRAINT continuous_agg_migrate_plan_step_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg_migrate_plan (mat_hypertable_id) ON DELETE CASCADE;
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg_migrate_plan', '');
+
+GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan TO PUBLIC;
+
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.telemetry_event;
+
+DROP TABLE IF EXISTS _timescaledb_catalog.telemetry_event;