timescale · svenklemm · Oct 31, 2019 · Oct 31, 2019
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,16 +4,63 @@
 `psql` with the `-X` flag to prevent any `.psqlrc` commands from
 accidentally triggering the load of a previous DB version.**
 
-## 1.5.0 (unreleased)
+## 1.5.0 (2019-10-31)
+
+This release adds major new features and bugfixes since the 1.4.2 release.
+We deem it moderate priority for upgrading.
+
+This release adds compression as a major new feature.
+Multiple type-specific compression options are available in this release
+(including DeltaDelta with run-length-encoding for integers and
+timestamps; Gorilla compression for floats; dictionary-based compression
+for any data type, but specifically for low-cardinality datasets;
+and other LZ-based techniques). Individual columns can be compressed with
+type-specific compression algorithms as Postgres' native row-based format
+are rolled up into columnar-like arrays on a per chunk basis.
+The query planner then handles transparent decompression for compressed
+chunks at execution time.
+
+This release also adds support for basic data tiering by supporting
+the migration of chunks between tablespaces, as well as support for
+parallel query coordination to the ChunkAppend node.
+Previously ChunkAppend would rely on parallel coordination in the
+underlying scans for parallel plans.
+
+More information can be found on [our blog](https://blog.timescale.com/blog/building-columnar-compression-in-a-row-oriented-database)
+or in this [tutorial](https://docs.timescale.com/latest/tutorials/compression-tutorial)
 
 **For this release only**, you will need to restart the database before running
 `ALTER EXTENSION`
 
-**Features**
+**Major Features**
+* #1393 Moving chunks between different tablespaces
 * #1433 Make ChunkAppend parallel aware
+* #1434 Introducing native compression, multiple compression algorithms, and hybrid row/columnar projections
+
+**Minor Features**
+* #1471 Allow setting reloptions on chunks
+* #1479 Add next_start option to alter_job_schedule
+* #1481 Add last_successful_finish to bgw_job_stats
 
 **Bugfixes**
 * #1444 Prevent LIMIT pushdown in JOINs
+* #1447 Fix runtime exclusion memory leak
+* #1464 Fix ordered append with expressions in ORDER BY clause with space partitioning
+* #1476 Fix logic for BGW rescheduling
+* #1477 Fix gapfill treat_null_as_missing
+* #1493 Prevent recursion in invalidation processing
+* #1498 Fix overflow in gapfill's interpolate
+* #1499 Fix error for exported_uuid in pg_restore
+* #1503 Fix bug with histogram function in parallel
+
+**Thanks**
+* @dhyun-obsec for reporting an issue with pg_restore
+* @rhaymo for reporting an issue with interpolate
+* @optijon for reporting an issue with locf treat_null_as_missing
+* @favnee for reporting an issue with runtime exclusion
+* @Lectem for reporting an issue with histograms
+* @rogerdwan for reporting an issue with BGW rescheduling
+* @od0 for reporting an issue with alter_job_schedule
 
 ## 1.4.2 (2019-09-11)
 

diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
@@ -95,6 +95,7 @@ set(MOD_FILES
   updates/1.3.2--1.4.0.sql
   updates/1.4.0--1.4.1.sql
   updates/1.4.1--1.4.2.sql
+  updates/1.4.2--1.5.0.sql
 )
 
 set(MODULE_PATHNAME "$libdir/timescaledb-${PROJECT_VERSION_MOD}")

diff --git a/sql/updates/1.4.2--1.5.0.sql b/sql/updates/1.4.2--1.5.0.sql
@@ -0,0 +1,216 @@
+DROP FUNCTION IF EXISTS drop_chunks("any",name,name,boolean,"any",boolean,boolean);
+DROP FUNCTION IF EXISTS add_drop_chunks_policy(REGCLASS,INTERVAL,BOOL,BOOL,BOOL);
+
+ALTER TABLE _timescaledb_catalog.dimension
+ADD COLUMN integer_now_func_schema     NAME     NULL;
+
+ALTER TABLE _timescaledb_catalog.dimension
+ADD COLUMN integer_now_func            NAME     NULL;
+
+ALTER TABLE _timescaledb_catalog.dimension
+ADD CONSTRAINT dimension_check2
+CHECK (
+        (integer_now_func_schema IS NULL AND integer_now_func IS NULL) OR
+        (integer_now_func_schema IS NOT NULL AND integer_now_func IS NOT NULL)
+);
+-- ----------------------
+CREATE TYPE _timescaledb_catalog.ts_interval AS (
+    is_time_interval        BOOLEAN,
+    time_interval		    INTERVAL,
+    integer_interval        BIGINT
+    );
+
+-- q -- todo:: this is probably necessary if we keep the validation constraint in the table definition.
+CREATE OR REPLACE FUNCTION _timescaledb_internal.valid_ts_interval(invl _timescaledb_catalog.ts_interval)
+RETURNS BOOLEAN AS '@MODULE_PATHNAME@', 'ts_valid_ts_interval' LANGUAGE C VOLATILE STRICT;
+
+DROP VIEW IF EXISTS timescaledb_information.drop_chunks_policies;
+DROP VIEW IF EXISTS timescaledb_information.policy_stats;
+
+CREATE TABLE IF NOT EXISTS _timescaledb_config.bgw_policy_drop_chunks_tmp (
+    job_id          		    INTEGER                 PRIMARY KEY REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE,
+    hypertable_id   		    INTEGER     UNIQUE      NOT NULL REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE,
+    older_than	    _timescaledb_catalog.ts_interval    NOT NULL,
+	cascade					    BOOLEAN                 NOT NULL,
+    cascade_to_materializations BOOLEAN                 NOT NULL,
+    CONSTRAINT valid_older_than CHECK(_timescaledb_internal.valid_ts_interval(older_than))
+);
+
+INSERT INTO _timescaledb_config.bgw_policy_drop_chunks_tmp
+(SELECT job_id, hypertable_id, ROW('t',older_than,NULL)::_timescaledb_catalog.ts_interval  as older_than, cascade, cascade_to_materializations
+FROM _timescaledb_config.bgw_policy_drop_chunks);
+
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_config.bgw_policy_drop_chunks;
+DROP TABLE _timescaledb_config.bgw_policy_drop_chunks;
+
+CREATE TABLE IF NOT EXISTS _timescaledb_config.bgw_policy_drop_chunks (
+    job_id          		    INTEGER                 PRIMARY KEY REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE,
+    hypertable_id   		    INTEGER     UNIQUE      NOT NULL REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE,
+    older_than	    _timescaledb_catalog.ts_interval    NOT NULL,
+	cascade					    BOOLEAN                 NOT NULL,
+    cascade_to_materializations BOOLEAN                 NOT NULL,
+    CONSTRAINT valid_older_than CHECK(_timescaledb_internal.valid_ts_interval(older_than))
+);
+
+INSERT INTO _timescaledb_config.bgw_policy_drop_chunks
+(SELECT * FROM _timescaledb_config.bgw_policy_drop_chunks_tmp);
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_policy_drop_chunks', '');
+DROP TABLE _timescaledb_config.bgw_policy_drop_chunks_tmp;
+GRANT SELECT ON _timescaledb_config.bgw_policy_drop_chunks TO PUBLIC;
+
+DROP FUNCTION IF EXISTS alter_job_schedule(INTEGER, INTERVAL, INTERVAL, INTEGER, INTERVAL, BOOL);
+
+
+--ADDS last_successful_finish column
+--Must remove from extension first
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_internal.bgw_job_stat;
+DROP VIEW IF EXISTS timescaledb_information.policy_stats;
+DROP VIEW IF EXISTS timescaledb_information.continuous_aggregate_stats;
+
+--create table and drop instead of rename so that all indexes dropped as well
+CREATE TABLE _timescaledb_internal.bgw_job_stat_tmp AS SELECT * FROM _timescaledb_internal.bgw_job_stat;
+DROP TABLE _timescaledb_internal.bgw_job_stat;
+
+CREATE TABLE IF NOT EXISTS _timescaledb_internal.bgw_job_stat (
+    job_id                  INTEGER         PRIMARY KEY REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE,
+    last_start              TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    last_finish             TIMESTAMPTZ NOT NULL,
+    next_start              TIMESTAMPTZ NOT NULL,
+    last_successful_finish  TIMESTAMPTZ NOT NULL,
+    last_run_success        BOOL        NOT NULL,
+    total_runs              BIGINT      NOT NULL,
+    total_duration          INTERVAL    NOT NULL,
+    total_successes         BIGINT      NOT NULL,
+    total_failures          BIGINT      NOT NULL,
+    total_crashes           BIGINT      NOT NULL,
+    consecutive_failures    INT         NOT NULL,
+    consecutive_crashes     INT         NOT NULL
+);
+--The job_stat table is not dumped by pg_dump on purpose because (see tables.sql for details)
+
+INSERT INTO _timescaledb_internal.bgw_job_stat
+    SELECT job_id,
+        last_start,
+        last_finish,
+        next_start,
+        CASE WHEN last_run_success THEN last_finish ELSE '-infinity'::timestamptz END as last_successful_finish,
+        last_run_success,
+        total_runs,
+        total_duration,
+        total_successes,
+        total_failures,
+        total_crashes,
+        consecutive_failures,
+        consecutive_crashes
+    FROM _timescaledb_internal.bgw_job_stat_tmp;
+
+DROP TABLE _timescaledb_internal.bgw_job_stat_tmp;
+GRANT SELECT ON _timescaledb_internal.bgw_job_stat TO PUBLIC;
+
+
+
+
+
+ALTER TABLE _timescaledb_catalog.hypertable add column compressed boolean NOT NULL default false;
+ALTER TABLE _timescaledb_catalog.hypertable add column compressed_hypertable_id          INTEGER   REFERENCES _timescaledb_catalog.hypertable(id);
+ALTER TABLE _timescaledb_catalog.hypertable drop constraint hypertable_num_dimensions_check;
+ALTER TABLE _timescaledb_catalog.hypertable add constraint hypertable_dim_compress_check check ( num_dimensions > 0  or compressed = true );
+alter table _timescaledb_catalog.hypertable add constraint hypertable_compress_check check ( compressed = false or (compressed = true and compressed_hypertable_id is null ));
+
+ALTER TABLE _timescaledb_catalog.chunk add column compressed_chunk_id integer references _timescaledb_catalog.chunk(id);
+CREATE INDEX IF NOT EXISTS chunk_compressed_chunk_id_idx
+ON _timescaledb_catalog.chunk(compressed_chunk_id);
+
+CREATE TABLE _timescaledb_catalog.compression_algorithm(
+	id SMALLINT PRIMARY KEY,
+	version SMALLINT NOT NULL,
+	name NAME NOT NULL,
+	description TEXT
+);
+
+CREATE TABLE IF NOT EXISTS _timescaledb_catalog.hypertable_compression (
+	hypertable_id INTEGER REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE,
+	attname NAME NOT NULL,
+	compression_algorithm_id SMALLINT REFERENCES _timescaledb_catalog.compression_algorithm(id),
+    segmentby_column_index SMALLINT ,
+    orderby_column_index SMALLINT,
+    orderby_asc BOOLEAN,
+    orderby_nullsfirst BOOLEAN,
+	PRIMARY KEY (hypertable_id, attname),
+    UNIQUE (hypertable_id, segmentby_column_index),
+    UNIQUE (hypertable_id, orderby_column_index)
+);
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_compression', '');
+
+CREATE TABLE IF NOT EXISTS _timescaledb_catalog.compression_chunk_size (
+
+    chunk_id            INTEGER REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE,
+    compressed_chunk_id   INTEGER REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE,
+    uncompressed_heap_size BIGINT NOT NULL,
+    uncompressed_toast_size BIGINT NOT NULL,
+    uncompressed_index_size BIGINT NOT NULL,
+    compressed_heap_size BIGINT NOT NULL,
+    compressed_toast_size BIGINT NOT NULL,
+    compressed_index_size BIGINT NOT NULL,
+    PRIMARY KEY( chunk_id, compressed_chunk_id)
+);
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_chunk_size', '');
+
+CREATE TABLE IF NOT EXISTS _timescaledb_config.bgw_policy_compress_chunks(
+    job_id                      INTEGER                 PRIMARY KEY REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE,
+    hypertable_id               INTEGER     UNIQUE      NOT NULL REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE,
+    older_than      _timescaledb_catalog.ts_interval    NOT NULL,
+    CONSTRAINT valid_older_than CHECK(_timescaledb_internal.valid_ts_interval(older_than))
+);
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_policy_compress_chunks', '');
+
+GRANT SELECT ON _timescaledb_catalog.compression_algorithm TO PUBLIC;
+GRANT SELECT ON _timescaledb_catalog.hypertable_compression TO PUBLIC;
+GRANT SELECT ON _timescaledb_catalog.compression_chunk_size TO PUBLIC;
+GRANT SELECT ON _timescaledb_config.bgw_policy_compress_chunks TO PUBLIC;
+
+CREATE TYPE _timescaledb_internal.compressed_data;
+
+--the textual input/output is simply base64 encoding of the binary representation
+CREATE FUNCTION _timescaledb_internal.compressed_data_in(CSTRING)
+   RETURNS _timescaledb_internal.compressed_data
+   AS '@MODULE_PATHNAME@', 'ts_compressed_data_in'
+   LANGUAGE C IMMUTABLE STRICT;
+
+CREATE FUNCTION _timescaledb_internal.compressed_data_out(_timescaledb_internal.compressed_data)
+   RETURNS CSTRING
+   AS '@MODULE_PATHNAME@', 'ts_compressed_data_out'
+   LANGUAGE C IMMUTABLE STRICT;
+
+CREATE FUNCTION _timescaledb_internal.compressed_data_send(_timescaledb_internal.compressed_data)
+   RETURNS BYTEA
+   AS '@MODULE_PATHNAME@', 'ts_compressed_data_send'
+   LANGUAGE C IMMUTABLE STRICT;
+
+CREATE FUNCTION _timescaledb_internal.compressed_data_recv(internal)
+   RETURNS _timescaledb_internal.compressed_data
+   AS '@MODULE_PATHNAME@', 'ts_compressed_data_recv'
+   LANGUAGE C IMMUTABLE STRICT;
+
+CREATE TYPE _timescaledb_internal.compressed_data (
+    INTERNALLENGTH = VARIABLE,
+    STORAGE = EXTERNAL,
+    ALIGNMENT = DOUBLE, --needed for alignment in ARRAY type compression
+    INPUT = _timescaledb_internal.compressed_data_in,
+    OUTPUT = _timescaledb_internal.compressed_data_out,
+    RECEIVE = _timescaledb_internal.compressed_data_recv,
+    SEND = _timescaledb_internal.compressed_data_send
+);
+
+--insert data for compression_algorithm --
+insert into _timescaledb_catalog.compression_algorithm values
+( 0, 1, 'COMPRESSION_ALGORITHM_NONE', 'no compression'),
+( 1, 1, 'COMPRESSION_ALGORITHM_ARRAY', 'array'),
+( 2, 1, 'COMPRESSION_ALGORITHM_DICTIONARY', 'dictionary'),
+( 3, 1, 'COMPRESSION_ALGORITHM_GORILLA', 'gorilla'),
+( 4, 1, 'COMPRESSION_ALGORITHM_DELTADELTA', 'deltadelta')
+on conflict(id) do update set (version, name, description)
+= (excluded.version, excluded.name, excluded.description);