From e45a5572a90f07aa178ab867dba22673e3b612ae Mon Sep 17 00:00:00 2001 From: shhnwz Date: Thu, 13 Oct 2022 20:38:05 +0530 Subject: [PATCH] Index support for compress chunk It allows to override tuplesort with indexscan if compression setting keys matches with Index keys. Moreover this feature has Enable/Disable Toggle. To Disable from the client use the following command, SET timescaledb.enable_compression_indexscan = 'OFF' --- src/guc.c | 12 + src/guc.h | 1 + tsl/src/compression/compression.c | 238 ++++- tsl/test/expected/compression_indexscan.out | 919 ++++++++++++++++++++ tsl/test/sql/CMakeLists.txt | 1 + tsl/test/sql/compression_indexscan.sql | 258 ++++++ 6 files changed, 1405 insertions(+), 24 deletions(-) create mode 100644 tsl/test/expected/compression_indexscan.out create mode 100644 tsl/test/sql/compression_indexscan.sql diff --git a/src/guc.c b/src/guc.c index acbb380ceb1..c17901f6e45 100644 --- a/src/guc.c +++ b/src/guc.c @@ -77,6 +77,7 @@ bool ts_guc_enable_osm_reads = true; TSDLLEXPORT bool ts_guc_enable_transparent_decompression = true; bool ts_guc_enable_per_data_node_queries = true; bool ts_guc_enable_async_append = true; +TSDLLEXPORT bool ts_guc_enable_compression_indexscan = true; TSDLLEXPORT bool ts_guc_enable_skip_scan = true; int ts_guc_max_open_chunks_per_insert = 10; int ts_guc_max_cached_chunks_per_hypertable = 10; @@ -386,6 +387,17 @@ _guc_init(void) NULL, NULL); + DefineCustomBoolVariable("timescaledb.enable_compression_indexscan", + "Enable compression to take indexscan path", + "Enable indexscan during compression, if matching index is found", + &ts_guc_enable_compression_indexscan, + true, + PGC_USERSET, + 0, + NULL, + NULL, + NULL); + DefineCustomEnumVariable("timescaledb.remote_data_fetcher", "Set remote data fetcher type", "Pick data fetcher type based on type of queries you plan to run " diff --git a/src/guc.h b/src/guc.h index 405b705abd3..a1dd0d530e9 100644 --- a/src/guc.h +++ b/src/guc.h @@ -56,6 +56,7 @@ extern TSDLLEXPORT bool ts_guc_enable_client_ddl_on_data_nodes; extern TSDLLEXPORT char *ts_guc_ssl_dir; extern TSDLLEXPORT char *ts_guc_passfile; extern TSDLLEXPORT bool ts_guc_enable_remote_explain; +extern TSDLLEXPORT bool ts_guc_enable_compression_indexscan; typedef enum DataFetcherType { diff --git a/tsl/src/compression/compression.c b/tsl/src/compression/compression.c index 5e674e2b549..07a3ad8a02d 100644 --- a/tsl/src/compression/compression.c +++ b/tsl/src/compression/compression.c @@ -32,7 +32,7 @@ #include #include #include - +#include #include #include "compat/compat.h" @@ -49,7 +49,7 @@ #include "segment_meta.h" #include "ts_catalog/hypertable_compression.h" #include "ts_catalog/catalog.h" - +#include "guc.h" #include #define MAX_ROWS_PER_COMPRESSION 1000 @@ -165,6 +165,17 @@ static void row_compressor_init(RowCompressor *row_compressor, TupleDesc uncompr static void row_compressor_append_sorted_rows(RowCompressor *row_compressor, Tuplesortstate *sorted_rel, TupleDesc sorted_desc); static void row_compressor_finish(RowCompressor *row_compressor); +static void row_compressor_update_group(RowCompressor *row_compressor, TupleTableSlot *row); +static bool row_compressor_new_row_is_in_new_group(RowCompressor *row_compressor, + TupleTableSlot *row); +static void row_compressor_append_row(RowCompressor *row_compressor, TupleTableSlot *row); +static void row_compressor_flush(RowCompressor *row_compressor, CommandId mycid, + bool changed_groups); + +static SegmentInfo *segment_info_new(Form_pg_attribute column_attr); +static void segment_info_update(SegmentInfo *segment_info, Datum val, bool is_null); +static bool segment_info_datum_is_in_group(SegmentInfo *segment_info, Datum datum, bool is_null); +static void run_analyze_on_chunk(Oid chunk_relid); /******************** ** compress_chunk ** @@ -300,6 +311,18 @@ compress_chunk(Oid in_table, Oid out_table, const ColumnCompressionInfo **column int num_compression_infos) { int n_keys; + ListCell *lc; + int indexscan_direction = NoMovementScanDirection; + List *in_rel_index_oids; + Relation matched_index_rel = NULL; + TupleTableSlot *slot; + IndexScanDesc index_scan; + bool first_iteration = true; + bool changed_groups, compressed_row_is_full; + MemoryContext old_ctx; + CommandId mycid = GetCurrentCommandId(true); + HeapTuple in_table_tp = NULL, index_tp = NULL; + Form_pg_attribute in_table_attr_tp, index_attr_tp; const ColumnCompressionInfo **keys; CompressionStats cstat; @@ -323,14 +346,139 @@ compress_chunk(Oid in_table, Oid out_table, const ColumnCompressionInfo **column TupleDesc in_desc = RelationGetDescr(in_rel); TupleDesc out_desc = RelationGetDescr(out_rel); + in_rel_index_oids = RelationGetIndexList(in_rel); + int i = 0; + /* Before calling row compressor relation should be segmented and sorted as per + * compress_segmentby and compress_orderby column/s configured in ColumnCompressionInfo. + * Cost of sorting can be mitigated if we find an existing BTREE index defined for + * uncompressed chunk otherwise expensive tuplesort will come into play. + * + * The following code is trying to find an existing index that + * matches the ColumnCompressionInfo so that we can skip sequential scan and + * tuplesort. + * + * Matching Criteria for Each IndexAtt[i] and ColumnCompressionInfo Keys[i] + * ======================================================================== + * a) Index attnum must match with ColumnCompressionInfo Key {keys[i]}. + * b) Index attOption(ASC/DESC and NULL_FIRST) can be mapped with ColumnCompressionInfo + * orderby_asc and null_first. + * + * BTREE Indexes Ordering + * ===================== + * a) ASC[Null_Last] ==> [1]->[2]->NULL + * b) [Null_First]ASC ==> NULL->[1]->[2] + * c) DSC[Null_Last] ==> [2]->[1]->NULL + * d) [Null_First]DSC ==> NULL->[2]->[1] + */ + if (ts_guc_enable_compression_indexscan) + { + foreach (lc, in_rel_index_oids) + { + Oid index_oid = lfirst_oid(lc); + Relation index_rel = index_open(index_oid, AccessShareLock); + IndexInfo *index_info = BuildIndexInfo(index_rel); + int previous_direction = NoMovementScanDirection; + int current_direction = NoMovementScanDirection; - Tuplesortstate *sorted_rel = compress_chunk_sort_relation(in_rel, n_keys, keys); + if (n_keys <= index_info->ii_NumIndexKeyAttrs && index_info->ii_Am == BTREE_AM_OID) + { + for (i = 0; i < n_keys; i++) + { + int16 att_num = get_attnum(in_table, NameStr(keys[i]->attname)); + + int16 option = index_rel->rd_indoption[i]; + bool index_orderby_asc = ((option & INDOPTION_DESC) == 0); + bool index_null_first = ((option & INDOPTION_NULLS_FIRST) != 0); + bool is_orderby_asc = + COMPRESSIONCOL_IS_SEGMENT_BY(keys[i]) ? true : keys[i]->orderby_asc; + bool is_null_first = + COMPRESSIONCOL_IS_SEGMENT_BY(keys[i]) ? false : keys[i]->orderby_nullsfirst; + + if (att_num == 0 || index_info->ii_IndexAttrNumbers[i] != att_num) + { + break; + } + + in_table_tp = SearchSysCacheAttNum(in_table, att_num); + if (!HeapTupleIsValid(in_table_tp)) + elog(ERROR, + "table \"%s\" does not have column \"%s\"", + get_rel_name(in_table), + NameStr(keys[i]->attname)); + + index_tp = SearchSysCacheAttNum(index_oid, i + 1); + if (!HeapTupleIsValid(index_tp)) + elog(ERROR, + "index \"%s\" does not have column \"%s\"", + get_rel_name(index_oid), + NameStr(keys[i]->attname)); + + in_table_attr_tp = (Form_pg_attribute) GETSTRUCT(in_table_tp); + index_attr_tp = (Form_pg_attribute) GETSTRUCT(index_tp); + + if (index_orderby_asc == is_orderby_asc && index_null_first == is_null_first && + in_table_attr_tp->attcollation == index_attr_tp->attcollation) + { + current_direction = ForwardScanDirection; + } + else if (index_orderby_asc != is_orderby_asc && + index_null_first != is_null_first && + in_table_attr_tp->attcollation == index_attr_tp->attcollation) + { + current_direction = BackwardScanDirection; + } + else + { + current_direction = NoMovementScanDirection; + break; + } + + ReleaseSysCache(in_table_tp); + in_table_tp = NULL; + ReleaseSysCache(index_tp); + index_tp = NULL; + if (previous_direction == NoMovementScanDirection) + { + previous_direction = current_direction; + } + else if (previous_direction != current_direction) + { + break; + } + } - RowCompressor row_compressor; + if (n_keys == i && (previous_direction == current_direction && + current_direction != NoMovementScanDirection)) + { + matched_index_rel = index_rel; + indexscan_direction = current_direction; + break; + } + else + { + if (HeapTupleIsValid(in_table_tp)) + { + ReleaseSysCache(in_table_tp); + in_table_tp = NULL; + } + if (HeapTupleIsValid(index_tp)) + { + ReleaseSysCache(index_tp); + index_tp = NULL; + } + index_close(index_rel, AccessShareLock); + } + } + else + { + index_close(index_rel, AccessShareLock); + } + } + } Assert(num_compression_infos <= in_desc->natts); Assert(num_compression_infos <= out_desc->natts); - + RowCompressor row_compressor; row_compressor_init(&row_compressor, in_desc, out_rel, @@ -340,12 +488,67 @@ compress_chunk(Oid in_table, Oid out_table, const ColumnCompressionInfo **column out_desc->natts, true /*need_bistate*/); - row_compressor_append_sorted_rows(&row_compressor, sorted_rel, in_desc); + if (matched_index_rel != NULL) + { +#ifdef TS_DEBUG + const char *compression_path = + GetConfigOption("timescaledb.show_compression_path_info", true, false); + if (compression_path != NULL && strcmp(compression_path, "on") == 0) + elog(INFO, + "compress_chunk_indexscan_start matched index \"%s\"", + get_rel_name(matched_index_rel->rd_id)); +#endif + index_scan = index_beginscan(in_rel, matched_index_rel, GetTransactionSnapshot(), 0, 0); + slot = table_slot_create(in_rel, NULL); + index_rescan(index_scan, NULL, 0, NULL, 0); + while (index_getnext_slot(index_scan, indexscan_direction, slot)) + { + slot_getallattrs(slot); + old_ctx = MemoryContextSwitchTo(row_compressor.per_row_ctx); + /* first time through */ + if (first_iteration) + { + row_compressor_update_group(&row_compressor, slot); + first_iteration = false; + } + changed_groups = row_compressor_new_row_is_in_new_group(&row_compressor, slot); + compressed_row_is_full = + row_compressor.rows_compressed_into_current_value >= MAX_ROWS_PER_COMPRESSION; + if (compressed_row_is_full || changed_groups) + { + if (row_compressor.rows_compressed_into_current_value > 0) + row_compressor_flush(&row_compressor, mycid, changed_groups); + if (changed_groups) + row_compressor_update_group(&row_compressor, slot); + } - row_compressor_finish(&row_compressor); + row_compressor_append_row(&row_compressor, slot); + MemoryContextSwitchTo(old_ctx); + ExecClearTuple(slot); + } - tuplesort_end(sorted_rel); + run_analyze_on_chunk(in_rel->rd_id); + if (row_compressor.rows_compressed_into_current_value > 0) + row_compressor_flush(&row_compressor, mycid, true); + ExecDropSingleTupleTableSlot(slot); + index_endscan(index_scan); + index_close(matched_index_rel, AccessShareLock); + } + else + { +#ifdef TS_DEBUG + const char *compression_path = + GetConfigOption("timescaledb.show_compression_path_info", true, false); + if (compression_path != NULL && strcmp(compression_path, "on") == 0) + elog(INFO, "compress_chunk_tuplesort_start"); +#endif + Tuplesortstate *sorted_rel = compress_chunk_sort_relation(in_rel, n_keys, keys); + row_compressor_append_sorted_rows(&row_compressor, sorted_rel, in_desc); + tuplesort_end(sorted_rel); + } + + row_compressor_finish(&row_compressor); truncate_relation(in_table); /* Recreate all indexes on out rel, we already have an exclusive lock on it, @@ -416,7 +619,6 @@ static void compress_chunk_populate_sort_info_for_column(Oid table, const ColumnCompressionInfo *column, AttrNumber *att_nums, Oid *sort_operator, Oid *collation, bool *nulls_first); -static void run_analyze_on_chunk(Oid chunk_relid); static Tuplesortstate * compress_chunk_sort_relation(Relation in_rel, int n_keys, const ColumnCompressionInfo **keys) @@ -540,21 +742,6 @@ run_analyze_on_chunk(Oid chunk_relid) ExecVacuum(NULL, &vs, true); } -/******************** - ** row_compressor ** - ********************/ - -static void row_compressor_update_group(RowCompressor *row_compressor, TupleTableSlot *row); -static bool row_compressor_new_row_is_in_new_group(RowCompressor *row_compressor, - TupleTableSlot *row); -static void row_compressor_append_row(RowCompressor *row_compressor, TupleTableSlot *row); -static void row_compressor_flush(RowCompressor *row_compressor, CommandId mycid, - bool changed_groups); - -static SegmentInfo *segment_info_new(Form_pg_attribute column_attr); -static void segment_info_update(SegmentInfo *segment_info, Datum val, bool is_null); -static bool segment_info_datum_is_in_group(SegmentInfo *segment_info, Datum datum, bool is_null); - /* Find segment by index for setting the correct sequence number if * we are trying to roll up chunks while compressing */ @@ -775,6 +962,9 @@ get_sequence_number_for_current_group(Relation table_rel, Oid index_oid, return result + SEQUENCE_NUM_GAP; } +/******************** + ** row_compressor ** + ********************/ /* num_compression_infos is the number of columns we will write to in the compressed table */ static void row_compressor_init(RowCompressor *row_compressor, TupleDesc uncompressed_tuple_desc, diff --git a/tsl/test/expected/compression_indexscan.out b/tsl/test/expected/compression_indexscan.out new file mode 100644 index 00000000000..6afc209e239 --- /dev/null +++ b/tsl/test/expected/compression_indexscan.out @@ -0,0 +1,919 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +--Enable compression path info +SET timescaledb.show_compression_path_info= 'on'; +--Table creation +CREATE TABLE tab1 ( + time timestamptz not null, + id integer not null, + c1 double precision null, + c2 double precision null +); +CREATE TABLE tab2 ( + time timestamptz not null, + id integer not null, + c1 double precision null, + c2 double precision null +); +--Hypertable creation +SELECT FROM create_hypertable('tab1', 'time'); +-- +(1 row) + +SELECT FROM create_hypertable('tab2', 'time'); +-- +(1 row) + +--Data generation +INSERT INTO tab1 +SELECT +time + (INTERVAL '1 minute' * random()) AS time, +id, +random() AS c1, +random()* 100 AS c2 +FROM +generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-28 1:00', '1 hour') AS g1(time), +generate_series(1, 100, 1 ) AS g2(id) +ORDER BY +time; +--Test Set 1.1 [ Index(ASC, Null_First), Compression(ASC, Null_First) ] +CREATE INDEX idx_asc_null_first ON tab1(id, time ASC NULLS FIRST); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | t | t +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_indexscan_start matched index "_hyper_1_1_chunk_idx_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_2_chunk_idx_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_3_chunk_idx_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_4_chunk_idx_asc_null_first" + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 1.1 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Test Set 1.2 [Index(ASC, Null_First), Compression(ASC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | t | f +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 1.2 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Test Set 1.3 [Index(ASC, Null_First), Compression(DESC,Null_First)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | f | t +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 1.3 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--DROP INDEX idx_asc_null_last +--Test Set 1.4 [Index(ASC, Null_First), Compression(DESC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC NULLS LAST'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | f | f +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 1.4 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +DROP INDEX idx_asc_null_first; +--Test Set 2.1 [Index(ASC, Null_Last), Compression(ASC,Null_First)] +CREATE INDEX idx_asc_null_last ON tab1(id, time); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | t | t +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 2.1 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Test Set 2.2 [Index(ASC, Null_Last), Compression(ASC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | t | f +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_indexscan_start matched index "_hyper_1_1_chunk_idx_asc_null_last" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_2_chunk_idx_asc_null_last" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_3_chunk_idx_asc_null_last" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_4_chunk_idx_asc_null_last" + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 2.2 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Test Set 2.3 [Index(ASC, Null_Last), Compression(DESC,Null_First)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | f | t +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 2.3 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--DROP INDEX idx_asc_null_last +--Test Set 2.4 [Index(ASC, Null_Last), Compression(DESC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC NULLS LAST'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | f | f +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 2.4 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +DROP INDEX idx_asc_null_last; +--Test Set 3.1 [Index(DESC, Null_First), Compression(ASC,Null_First)] +CREATE INDEX idx_desc_null_first ON tab1(id, time DESC NULLS FIRST); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | t | t +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 3.1 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Test Set 3.2 [Index(DESC, Null_First), Compression(ASC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | t | f +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 3.2 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Test Set 3.3 [Index(DESC, Null_First), Compression(DESC,Null_First)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | f | t +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_indexscan_start matched index "_hyper_1_1_chunk_idx_desc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_2_chunk_idx_desc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_3_chunk_idx_desc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_4_chunk_idx_desc_null_first" + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 3.3 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--DROP INDEX idx_asc_null_last +--Test Set 3.4 [Index(DESC, Null_First), Compression(DESC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC NULLS LAST'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | f | f +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 3.4 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +DROP INDEX idx_desc_null_first; +--Test Set 4.1 [Index(DESC, Null_Last), Compression(ASC,Null_First)] +CREATE INDEX idx_desc_null_last ON tab1(id, time DESC); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | t | t +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 4.1 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Test Set 4.2 [Index(DESC, Null_Last), Compression(ASC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | t | f +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 4.2 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Test Set 4.3 [Index(DESC, Null_Last), Compression(DESC,Null_First)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | f | t +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_indexscan_start matched index "_hyper_1_1_chunk_idx_desc_null_last" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_2_chunk_idx_desc_null_last" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_3_chunk_idx_desc_null_last" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_4_chunk_idx_desc_null_last" + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 4.3 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Test Set 4.4 [Index(DESC, Null_Last), Compression(DESC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC NULLS LAST'); +SELECT * FROM timescaledb_information.compression_settings; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | tab1 | id | 1 | | | + public | tab1 | time | | 1 | f | f +(2 rows) + +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = '', timescaledb.compress_orderby = 'time'); +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_indexscan_start matched index "_hyper_1_1_chunk_tab1_time_idx" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_2_chunk_tab1_time_idx" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_3_chunk_tab1_time_idx" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_4_chunk_tab1_time_idx" + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Cleanup 4.4 +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +DROP INDEX idx_desc_null_last; +--Test Set 5 GUC SET timescaledb.enable_compression_indexscan +-- Default this flag will be true. +SET timescaledb.enable_compression_indexscan = 'OFF'; +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +SET timescaledb.enable_compression_indexscan = 'ON'; +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_indexscan_start matched index "_hyper_1_1_chunk_tab1_time_idx" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_2_chunk_tab1_time_idx" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_3_chunk_tab1_time_idx" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_4_chunk_tab1_time_idx" + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +--Test Set 6 Compare two compression paths +INSERT into tab2 SELECT * from tab1; +CREATE INDEX idx_asc_null_first ON tab1(id, time ASC NULLS FIRST); +CREATE INDEX idx2_asc_null_first ON tab2(id, time ASC NULLS FIRST); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +ALTER TABLE tab2 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +SET timescaledb.enable_compression_indexscan = 'OFF'; +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +SET timescaledb.enable_compression_indexscan = 'ON'; +SELECT compress_chunk(show_chunks('tab2')); +INFO: compress_chunk_indexscan_start matched index "_hyper_2_81_chunk_idx2_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_2_82_chunk_idx2_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_2_83_chunk_idx2_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_2_84_chunk_idx2_asc_null_first" + compress_chunk +----------------------------------------- + _timescaledb_internal._hyper_2_81_chunk + _timescaledb_internal._hyper_2_82_chunk + _timescaledb_internal._hyper_2_83_chunk + _timescaledb_internal._hyper_2_84_chunk +(4 rows) + +SELECT id, time from tab1 EXCEPT SELECT id, time from tab2; + id | time +----+------ +(0 rows) + +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab2')); + decompress_chunk +----------------------------------------- + _timescaledb_internal._hyper_2_81_chunk + _timescaledb_internal._hyper_2_82_chunk + _timescaledb_internal._hyper_2_83_chunk + _timescaledb_internal._hyper_2_84_chunk +(4 rows) + +DROP INDEX idx2_asc_null_first; +DROP INDEX idx_asc_null_first; +--Test Set 7.1 with Collation order_by +\set ON_ERROR_STOP 1 +CREATE TABLE tab3 ( + name text, + time timestamptz not null +); +SELECT FROM create_hypertable('tab3', 'time'); +-- +(1 row) + +--Data generation +INSERT INTO tab3 +SELECT +name, +time + (INTERVAL '1 minute' * random()) AS time +FROM +md5(random()::text) as name, +generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-28 1:00', '1 hour') AS g1(time) +ORDER BY +time; +CREATE INDEX idx_asc_null_first ON tab3(name COLLATE "C", time ASC NULLS FIRST); +ALTER TABLE tab3 SET(timescaledb.compress, timescaledb.compress_segmentby = '', timescaledb.compress_orderby = 'name, time NULLS FIRST'); +SELECT compress_chunk(show_chunks('tab3')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +------------------------------------------ + _timescaledb_internal._hyper_22_93_chunk + _timescaledb_internal._hyper_22_94_chunk + _timescaledb_internal._hyper_22_95_chunk + _timescaledb_internal._hyper_22_96_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab3')); + decompress_chunk +------------------------------------------ + _timescaledb_internal._hyper_22_93_chunk + _timescaledb_internal._hyper_22_94_chunk + _timescaledb_internal._hyper_22_95_chunk + _timescaledb_internal._hyper_22_96_chunk +(4 rows) + +CREATE INDEX idxcol_asc_null_first ON tab3(name, time ASC NULLS FIRST); +SELECT compress_chunk(show_chunks('tab3')); +INFO: compress_chunk_indexscan_start matched index "_hyper_22_93_chunk_idxcol_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_22_94_chunk_idxcol_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_22_95_chunk_idxcol_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_22_96_chunk_idxcol_asc_null_first" + compress_chunk +------------------------------------------ + _timescaledb_internal._hyper_22_93_chunk + _timescaledb_internal._hyper_22_94_chunk + _timescaledb_internal._hyper_22_95_chunk + _timescaledb_internal._hyper_22_96_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab3')); + decompress_chunk +------------------------------------------ + _timescaledb_internal._hyper_22_93_chunk + _timescaledb_internal._hyper_22_94_chunk + _timescaledb_internal._hyper_22_95_chunk + _timescaledb_internal._hyper_22_96_chunk +(4 rows) + +DROP INDEX idx_asc_null_first; +DROP INDEX idxcol_asc_null_first; +--Test Set 7.1 with Collation segment_by +CREATE INDEX idx_asc_null_first ON tab3(name COLLATE "ucs_basic", time ASC NULLS FIRST); +ALTER TABLE tab3 SET(timescaledb.compress, timescaledb.compress_segmentby = 'name', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT compress_chunk(show_chunks('tab3')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +------------------------------------------ + _timescaledb_internal._hyper_22_93_chunk + _timescaledb_internal._hyper_22_94_chunk + _timescaledb_internal._hyper_22_95_chunk + _timescaledb_internal._hyper_22_96_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab3')); + decompress_chunk +------------------------------------------ + _timescaledb_internal._hyper_22_93_chunk + _timescaledb_internal._hyper_22_94_chunk + _timescaledb_internal._hyper_22_95_chunk + _timescaledb_internal._hyper_22_96_chunk +(4 rows) + +CREATE INDEX idxcol_asc_null_first ON tab3(name, time ASC NULLS FIRST); +SELECT compress_chunk(show_chunks('tab3')); +INFO: compress_chunk_indexscan_start matched index "_hyper_22_93_chunk_idxcol_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_22_94_chunk_idxcol_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_22_95_chunk_idxcol_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_22_96_chunk_idxcol_asc_null_first" + compress_chunk +------------------------------------------ + _timescaledb_internal._hyper_22_93_chunk + _timescaledb_internal._hyper_22_94_chunk + _timescaledb_internal._hyper_22_95_chunk + _timescaledb_internal._hyper_22_96_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab3')); + decompress_chunk +------------------------------------------ + _timescaledb_internal._hyper_22_93_chunk + _timescaledb_internal._hyper_22_94_chunk + _timescaledb_internal._hyper_22_95_chunk + _timescaledb_internal._hyper_22_96_chunk +(4 rows) + +DROP INDEX idx_asc_null_first; +DROP INDEX idxcol_asc_null_first; +--Test Set 8 with multiple segment_by +CREATE INDEX idx_asc_null_first ON tab1(id, c1, time ASC NULLS FIRST); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id, c1', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_indexscan_start matched index "_hyper_1_1_chunk_idx_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_2_chunk_idx_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_3_chunk_idx_asc_null_first" +INFO: compress_chunk_indexscan_start matched index "_hyper_1_4_chunk_idx_asc_null_first" + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +DROP INDEX idx_asc_null_first; +CREATE INDEX idx_asc_null_first ON tab1(id, c1 DESC, time ASC NULLS FIRST); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id, c1', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +DROP INDEX idx_asc_null_first; +--Test Set 9 +--Last Column mismatch +CREATE INDEX idx_asc_null_first ON tab1(id, c1, c2); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id, c1', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +DROP INDEX idx_asc_null_first; +--Index Column out of order +CREATE INDEX idx_asc_null_first ON tab1(c1, id, time ASC NULLS FIRST); +SELECT compress_chunk(show_chunks('tab1')); +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start +INFO: compress_chunk_tuplesort_start + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +SELECT decompress_chunk(show_chunks('tab1')); + decompress_chunk +---------------------------------------- + _timescaledb_internal._hyper_1_1_chunk + _timescaledb_internal._hyper_1_2_chunk + _timescaledb_internal._hyper_1_3_chunk + _timescaledb_internal._hyper_1_4_chunk +(4 rows) + +DROP INDEX idx_asc_null_first; +--Tear down +DROP TABLE tab1; +DROP TABLE tab2; +DROP TABLE tab3; +SET timescaledb.show_compression_path_info = 'off'; diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index ff84328ff87..2edcaa3bddd 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -44,6 +44,7 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) compression_errors.sql compression_hypertable.sql compression_merge.sql + compression_indexscan.sql compression_segment_meta.sql compression.sql compress_table.sql diff --git a/tsl/test/sql/compression_indexscan.sql b/tsl/test/sql/compression_indexscan.sql new file mode 100644 index 00000000000..f9a8cb93e03 --- /dev/null +++ b/tsl/test/sql/compression_indexscan.sql @@ -0,0 +1,258 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +--Enable compression path info +SET timescaledb.show_compression_path_info= 'on'; +--Table creation +CREATE TABLE tab1 ( + time timestamptz not null, + id integer not null, + c1 double precision null, + c2 double precision null +); + +CREATE TABLE tab2 ( + time timestamptz not null, + id integer not null, + c1 double precision null, + c2 double precision null +); +--Hypertable creation +SELECT FROM create_hypertable('tab1', 'time'); +SELECT FROM create_hypertable('tab2', 'time'); + +--Data generation +INSERT INTO tab1 +SELECT +time + (INTERVAL '1 minute' * random()) AS time, +id, +random() AS c1, +random()* 100 AS c2 +FROM +generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-28 1:00', '1 hour') AS g1(time), +generate_series(1, 100, 1 ) AS g2(id) +ORDER BY +time; + +--Test Set 1.1 [ Index(ASC, Null_First), Compression(ASC, Null_First) ] +CREATE INDEX idx_asc_null_first ON tab1(id, time ASC NULLS FIRST); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 1.1 +SELECT decompress_chunk(show_chunks('tab1')); + +--Test Set 1.2 [Index(ASC, Null_First), Compression(ASC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 1.2 +SELECT decompress_chunk(show_chunks('tab1')); + +--Test Set 1.3 [Index(ASC, Null_First), Compression(DESC,Null_First)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 1.3 +SELECT decompress_chunk(show_chunks('tab1')); +--DROP INDEX idx_asc_null_last + +--Test Set 1.4 [Index(ASC, Null_First), Compression(DESC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC NULLS LAST'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 1.4 +SELECT decompress_chunk(show_chunks('tab1')); +DROP INDEX idx_asc_null_first; + +--Test Set 2.1 [Index(ASC, Null_Last), Compression(ASC,Null_First)] +CREATE INDEX idx_asc_null_last ON tab1(id, time); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 2.1 +SELECT decompress_chunk(show_chunks('tab1')); + +--Test Set 2.2 [Index(ASC, Null_Last), Compression(ASC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 2.2 +SELECT decompress_chunk(show_chunks('tab1')); + +--Test Set 2.3 [Index(ASC, Null_Last), Compression(DESC,Null_First)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 2.3 +SELECT decompress_chunk(show_chunks('tab1')); +--DROP INDEX idx_asc_null_last + +--Test Set 2.4 [Index(ASC, Null_Last), Compression(DESC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC NULLS LAST'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 2.4 +SELECT decompress_chunk(show_chunks('tab1')); +DROP INDEX idx_asc_null_last; + +--Test Set 3.1 [Index(DESC, Null_First), Compression(ASC,Null_First)] +CREATE INDEX idx_desc_null_first ON tab1(id, time DESC NULLS FIRST); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 3.1 +SELECT decompress_chunk(show_chunks('tab1')); + + +--Test Set 3.2 [Index(DESC, Null_First), Compression(ASC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 3.2 +SELECT decompress_chunk(show_chunks('tab1')); + +--Test Set 3.3 [Index(DESC, Null_First), Compression(DESC,Null_First)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 3.3 +SELECT decompress_chunk(show_chunks('tab1')); +--DROP INDEX idx_asc_null_last + +--Test Set 3.4 [Index(DESC, Null_First), Compression(DESC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC NULLS LAST'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 3.4 +SELECT decompress_chunk(show_chunks('tab1')); +DROP INDEX idx_desc_null_first; + +--Test Set 4.1 [Index(DESC, Null_Last), Compression(ASC,Null_First)] +CREATE INDEX idx_desc_null_last ON tab1(id, time DESC); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 4.1 +SELECT decompress_chunk(show_chunks('tab1')); + +--Test Set 4.2 [Index(DESC, Null_Last), Compression(ASC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 4.2 +SELECT decompress_chunk(show_chunks('tab1')); + +--Test Set 4.3 [Index(DESC, Null_Last), Compression(DESC,Null_First)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 4.3 +SELECT decompress_chunk(show_chunks('tab1')); + +--Test Set 4.4 [Index(DESC, Null_Last), Compression(DESC,Null_Last)] +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time DESC NULLS LAST'); +SELECT * FROM timescaledb_information.compression_settings; +SELECT compress_chunk(show_chunks('tab1')); +SELECT decompress_chunk(show_chunks('tab1')); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = '', timescaledb.compress_orderby = 'time'); +SELECT compress_chunk(show_chunks('tab1')); +--Cleanup 4.4 +SELECT decompress_chunk(show_chunks('tab1')); +DROP INDEX idx_desc_null_last; + +--Test Set 5 GUC SET timescaledb.enable_compression_indexscan +-- Default this flag will be true. +SET timescaledb.enable_compression_indexscan = 'OFF'; +SELECT compress_chunk(show_chunks('tab1')); +SELECT decompress_chunk(show_chunks('tab1')); +SET timescaledb.enable_compression_indexscan = 'ON'; +SELECT compress_chunk(show_chunks('tab1')); +SELECT decompress_chunk(show_chunks('tab1')); + +--Test Set 6 Compare two compression paths +INSERT into tab2 SELECT * from tab1; +CREATE INDEX idx_asc_null_first ON tab1(id, time ASC NULLS FIRST); +CREATE INDEX idx2_asc_null_first ON tab2(id, time ASC NULLS FIRST); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +ALTER TABLE tab2 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id', timescaledb.compress_orderby = 'time NULLS FIRST'); +SET timescaledb.enable_compression_indexscan = 'OFF'; +SELECT compress_chunk(show_chunks('tab1')); +SET timescaledb.enable_compression_indexscan = 'ON'; +SELECT compress_chunk(show_chunks('tab2')); +SELECT id, time from tab1 EXCEPT SELECT id, time from tab2; +SELECT decompress_chunk(show_chunks('tab1')); +SELECT decompress_chunk(show_chunks('tab2')); +DROP INDEX idx2_asc_null_first; +DROP INDEX idx_asc_null_first; + +--Test Set 7.1 with Collation order_by +\set ON_ERROR_STOP 1 +CREATE TABLE tab3 ( + name text, + time timestamptz not null +); + +SELECT FROM create_hypertable('tab3', 'time'); +--Data generation +INSERT INTO tab3 +SELECT +name, +time + (INTERVAL '1 minute' * random()) AS time +FROM +md5(random()::text) as name, +generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-28 1:00', '1 hour') AS g1(time) +ORDER BY +time; +CREATE INDEX idx_asc_null_first ON tab3(name COLLATE "C", time ASC NULLS FIRST); +ALTER TABLE tab3 SET(timescaledb.compress, timescaledb.compress_segmentby = '', timescaledb.compress_orderby = 'name, time NULLS FIRST'); +SELECT compress_chunk(show_chunks('tab3')); +SELECT decompress_chunk(show_chunks('tab3')); +CREATE INDEX idxcol_asc_null_first ON tab3(name, time ASC NULLS FIRST); +SELECT compress_chunk(show_chunks('tab3')); +SELECT decompress_chunk(show_chunks('tab3')); +DROP INDEX idx_asc_null_first; +DROP INDEX idxcol_asc_null_first; + +--Test Set 7.1 with Collation segment_by +CREATE INDEX idx_asc_null_first ON tab3(name COLLATE "ucs_basic", time ASC NULLS FIRST); +ALTER TABLE tab3 SET(timescaledb.compress, timescaledb.compress_segmentby = 'name', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT compress_chunk(show_chunks('tab3')); +SELECT decompress_chunk(show_chunks('tab3')); +CREATE INDEX idxcol_asc_null_first ON tab3(name, time ASC NULLS FIRST); +SELECT compress_chunk(show_chunks('tab3')); +SELECT decompress_chunk(show_chunks('tab3')); +DROP INDEX idx_asc_null_first; +DROP INDEX idxcol_asc_null_first; + +--Test Set 8 with multiple segment_by +CREATE INDEX idx_asc_null_first ON tab1(id, c1, time ASC NULLS FIRST); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id, c1', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT compress_chunk(show_chunks('tab1')); +SELECT decompress_chunk(show_chunks('tab1')); +DROP INDEX idx_asc_null_first; +CREATE INDEX idx_asc_null_first ON tab1(id, c1 DESC, time ASC NULLS FIRST); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id, c1', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT compress_chunk(show_chunks('tab1')); +SELECT decompress_chunk(show_chunks('tab1')); +DROP INDEX idx_asc_null_first; + +--Test Set 9 +--Last Column mismatch +CREATE INDEX idx_asc_null_first ON tab1(id, c1, c2); +ALTER TABLE tab1 SET(timescaledb.compress, timescaledb.compress_segmentby = 'id, c1', timescaledb.compress_orderby = 'time NULLS FIRST'); +SELECT compress_chunk(show_chunks('tab1')); +SELECT decompress_chunk(show_chunks('tab1')); +DROP INDEX idx_asc_null_first; +--Index Column out of order +CREATE INDEX idx_asc_null_first ON tab1(c1, id, time ASC NULLS FIRST); +SELECT compress_chunk(show_chunks('tab1')); +SELECT decompress_chunk(show_chunks('tab1')); +DROP INDEX idx_asc_null_first; + +--Tear down +DROP TABLE tab1; +DROP TABLE tab2; +DROP TABLE tab3; +SET timescaledb.show_compression_path_info = 'off'; \ No newline at end of file