From 358133fc027db9a8df9f273181747e56fcea5994 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 14 Nov 2022 17:10:40 +0100 Subject: [PATCH] Add a GUC variable to enable/disable OSM for SELECTs --- src/chunk_scan.c | 1 + src/guc.c | 13 ++++++ src/guc.h | 1 + src/hypertable_restrict_info.c | 36 ++++++++++++---- tsl/test/expected/chunk_utils_internal.out | 49 ++++++++++++++++++++++ tsl/test/sql/chunk_utils_internal.sql | 10 +++++ 6 files changed, 103 insertions(+), 7 deletions(-) diff --git a/src/chunk_scan.c b/src/chunk_scan.c index 914ef7ac8d6..740e59a43ec 100644 --- a/src/chunk_scan.c +++ b/src/chunk_scan.c @@ -11,6 +11,7 @@ #include "debug_point.h" #include "dimension_vector.h" +#include "guc.h" #include "hypertable.h" #include "hypercube.h" #include "scan_iterator.h" diff --git a/src/guc.c b/src/guc.c index 1bd009190ae..acbb380ceb1 100644 --- a/src/guc.c +++ b/src/guc.c @@ -73,6 +73,7 @@ bool ts_guc_enable_constraint_exclusion = true; bool ts_guc_enable_qual_propagation = true; bool ts_guc_enable_cagg_reorder_groupby = true; bool ts_guc_enable_now_constify = true; +bool ts_guc_enable_osm_reads = true; TSDLLEXPORT bool ts_guc_enable_transparent_decompression = true; bool ts_guc_enable_per_data_node_queries = true; bool ts_guc_enable_async_append = true; @@ -292,6 +293,18 @@ _guc_init(void) NULL, NULL); + DefineCustomBoolVariable("timescaledb.enable_tiered_reads", + "Enable tiered data reads", + "Enable reading of tiered data by including a foreign table " + "representing the data in the object storage into the query plan", + &ts_guc_enable_osm_reads, + true, + PGC_USERSET, + 0, + NULL, + NULL, + NULL); + DefineCustomIntVariable("timescaledb.max_insert_batch_size", "The max number of tuples to batch before sending to a data node", "When acting as a access node, TimescaleDB splits batches of " diff --git a/src/guc.h b/src/guc.h index 2aaed4af789..405b705abd3 100644 --- a/src/guc.h +++ b/src/guc.h @@ -25,6 +25,7 @@ extern bool ts_guc_enable_runtime_exclusion; extern bool ts_guc_enable_constraint_exclusion; extern bool ts_guc_enable_cagg_reorder_groupby; extern bool ts_guc_enable_now_constify; +extern bool ts_guc_enable_osm_reads; extern TSDLLEXPORT bool ts_guc_enable_transparent_decompression; extern TSDLLEXPORT bool ts_guc_enable_per_data_node_queries; extern TSDLLEXPORT bool ts_guc_enable_async_append; diff --git a/src/hypertable_restrict_info.c b/src/hypertable_restrict_info.c index 635c41d2a42..c561ec04f5e 100644 --- a/src/hypertable_restrict_info.c +++ b/src/hypertable_restrict_info.c @@ -20,6 +20,7 @@ #include "dimension.h" #include "dimension_slice.h" #include "dimension_vector.h" +#include "guc.h" #include "hypercube.h" #include "partitioning.h" #include "scan_iterator.h" @@ -664,6 +665,18 @@ ts_hypertable_restrict_info_get_chunks(HypertableRestrictInfo *hri, Hypertable * * No restrictions on hyperspace. Just enumerate all the chunks. */ chunk_ids = ts_chunk_get_chunk_ids_by_hypertable_id(ht->fd.id); + + /* + * If the hypertable has an OSM chunk it would end up in the list + * as well. We need to remove it when OSM reads are disabled via GUC + * variable. + */ + if (!ts_guc_enable_osm_reads) + { + int32 osm_chunk_id = ts_chunk_get_osm_chunk_id(ht->fd.id); + + chunk_ids = list_delete_int(chunk_ids, osm_chunk_id); + } } else { @@ -686,16 +699,25 @@ ts_hypertable_restrict_info_get_chunks(HypertableRestrictInfo *hri, Hypertable * } /* - * Always include the OSM chunk if we have one. It has some virtual - * dimension slices (at the moment, (+inf, +inf) slice for time, but it - * used to be different and might change again.) So sometimes it will - * match and sometimes it won't, so we have to check if it's already - * there not to add a duplicate. + * Always include the OSM chunk if we have one and OSM reads are + * enabled. It has some virtual dimension slices (at the moment, + * (+inf, +inf) slice for time, but it used to be different and might + * change again.) So sometimes it will match and sometimes it won't, + * so we have to check if it's already there not to add a duplicate. + * Similarly if OSM reads are disabled then we exclude the OSM chunk. */ int32 osm_chunk_id = ts_chunk_get_osm_chunk_id(ht->fd.id); - if (osm_chunk_id != 0 && !list_member_int(chunk_ids, osm_chunk_id)) + + if (osm_chunk_id != INVALID_CHUNK_ID) { - chunk_ids = lappend_int(chunk_ids, osm_chunk_id); + if (!ts_guc_enable_osm_reads) + { + chunk_ids = list_delete_int(chunk_ids, osm_chunk_id); + } + else if (!list_member_int(chunk_ids, osm_chunk_id)) + { + chunk_ids = lappend_int(chunk_ids, osm_chunk_id); + } } } diff --git a/tsl/test/expected/chunk_utils_internal.out b/tsl/test/expected/chunk_utils_internal.out index 99a21fe0525..e6502021b4e 100644 --- a/tsl/test/expected/chunk_utils_internal.out +++ b/tsl/test/expected/chunk_utils_internal.out @@ -569,6 +569,55 @@ SELECT * from ht_try WHERE timec > '2020-01-01 01:00' ORDER BY 1; Thu May 05 01:00:00 2022 PDT | 222 | 222 (1 row) +--TEST GUC variable to enable/disable OSM chunk +SET timescaledb.enable_tiered_reads=false; +EXPLAIN (COSTS OFF) SELECT * from ht_try; + QUERY PLAN +------------------------------- + Seq Scan on _hyper_5_10_chunk +(1 row) + +EXPLAIN (COSTS OFF) SELECT * from ht_try WHERE timec > '2022-01-01 01:00'; + QUERY PLAN +---------------------------------------------------------------------------------- + Index Scan using _hyper_5_10_chunk_ht_try_timec_idx on _hyper_5_10_chunk + Index Cond: (timec > 'Sat Jan 01 01:00:00 2022 PST'::timestamp with time zone) +(2 rows) + +EXPLAIN (COSTS OFF) SELECT * from ht_try WHERE timec < '2023-01-01 01:00'; + QUERY PLAN +---------------------------------------------------------------------------------- + Index Scan using _hyper_5_10_chunk_ht_try_timec_idx on _hyper_5_10_chunk + Index Cond: (timec < 'Sun Jan 01 01:00:00 2023 PST'::timestamp with time zone) +(2 rows) + +SET timescaledb.enable_tiered_reads=true; +EXPLAIN (COSTS OFF) SELECT * from ht_try; + QUERY PLAN +--------------------------------------- + Append + -> Foreign Scan on child_fdw_table + -> Seq Scan on _hyper_5_10_chunk +(3 rows) + +EXPLAIN (COSTS OFF) SELECT * from ht_try WHERE timec > '2022-01-01 01:00'; + QUERY PLAN +---------------------------------------------------------------------------------------- + Append + -> Foreign Scan on child_fdw_table + -> Index Scan using _hyper_5_10_chunk_ht_try_timec_idx on _hyper_5_10_chunk + Index Cond: (timec > 'Sat Jan 01 01:00:00 2022 PST'::timestamp with time zone) +(4 rows) + +EXPLAIN (COSTS OFF) SELECT * from ht_try WHERE timec < '2023-01-01 01:00'; + QUERY PLAN +---------------------------------------------------------------------------------------- + Append + -> Foreign Scan on child_fdw_table + -> Index Scan using _hyper_5_10_chunk_ht_try_timec_idx on _hyper_5_10_chunk + Index Cond: (timec < 'Sun Jan 01 01:00:00 2023 PST'::timestamp with time zone) +(4 rows) + --TEST insert into a OSM chunk fails. actually any insert will fail. But we just need -- to mock the hook and make sure the timescaledb code works correctly. SELECT ts_setup_osm_hook(); diff --git a/tsl/test/sql/chunk_utils_internal.sql b/tsl/test/sql/chunk_utils_internal.sql index 3fbf10b9424..e317cb7857f 100644 --- a/tsl/test/sql/chunk_utils_internal.sql +++ b/tsl/test/sql/chunk_utils_internal.sql @@ -344,6 +344,16 @@ SELECT * from ht_try WHERE timec > '2000-01-01 01:00' and timec < '2022-01-01 0 SELECT * from ht_try WHERE timec > '2020-01-01 01:00' ORDER BY 1; +--TEST GUC variable to enable/disable OSM chunk +SET timescaledb.enable_tiered_reads=false; +EXPLAIN (COSTS OFF) SELECT * from ht_try; +EXPLAIN (COSTS OFF) SELECT * from ht_try WHERE timec > '2022-01-01 01:00'; +EXPLAIN (COSTS OFF) SELECT * from ht_try WHERE timec < '2023-01-01 01:00'; +SET timescaledb.enable_tiered_reads=true; +EXPLAIN (COSTS OFF) SELECT * from ht_try; +EXPLAIN (COSTS OFF) SELECT * from ht_try WHERE timec > '2022-01-01 01:00'; +EXPLAIN (COSTS OFF) SELECT * from ht_try WHERE timec < '2023-01-01 01:00'; + --TEST insert into a OSM chunk fails. actually any insert will fail. But we just need -- to mock the hook and make sure the timescaledb code works correctly.