From 8bb0f1801970687a02bc2a6c4a4d28947b2db444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=BA=E5=8D=9A?= Date: Sun, 20 Aug 2023 22:57:53 +0800 Subject: [PATCH 01/11] bad size of mark skip idx --- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 26 ++++------------- .../MergeTree/MergeTreeDataSelectExecutor.h | 3 -- ...5_size_of_marks_skip_idx_explain.reference | 29 +++++++++++++++++++ .../02865_size_of_marks_skip_idx_explain.sql | 21 ++++++++++++++ 4 files changed, 55 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.reference create mode 100644 tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.sql diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ee515106591a..38bf376faadd 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -981,9 +981,9 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd const auto & index_and_condition = skip_indexes.useful_indices[idx]; auto & stat = useful_indices_stat[idx]; stat.total_parts.fetch_add(1, std::memory_order_relaxed); - stat.total_granules.fetch_add(ranges.ranges.getNumberOfMarks(), std::memory_order_relaxed); + size_t total_granules = ranges.ranges.getNumberOfMarks(); + stat.total_granules.fetch_add(total_granules, std::memory_order_relaxed); - size_t granules_dropped = 0; ranges.ranges = filterMarksUsingIndex( index_and_condition.index, index_and_condition.condition, @@ -991,12 +991,11 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd ranges.ranges, settings, reader_settings, - granules_dropped, mark_cache.get(), uncompressed_cache.get(), log); - stat.granules_dropped.fetch_add(granules_dropped, std::memory_order_relaxed); + stat.granules_dropped.fetch_add(total_granules - ranges.ranges.getNumberOfMarks(), std::memory_order_relaxed); if (ranges.ranges.empty()) stat.parts_dropped.fetch_add(1, std::memory_order_relaxed); } @@ -1010,17 +1009,15 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd auto & stat = merged_indices_stat[idx]; stat.total_parts.fetch_add(1, std::memory_order_relaxed); - size_t total_granules = 0; - size_t granules_dropped = 0; + size_t total_granules = ranges.ranges.getNumberOfMarks(); ranges.ranges = filterMarksUsingMergedIndex( indices_and_condition.indices, indices_and_condition.condition, part, ranges.ranges, settings, reader_settings, - total_granules, granules_dropped, mark_cache.get(), uncompressed_cache.get(), log); stat.total_granules.fetch_add(total_granules, std::memory_order_relaxed); - stat.granules_dropped.fetch_add(granules_dropped, std::memory_order_relaxed); + stat.granules_dropped.fetch_add(total_granules - ranges.ranges.getNumberOfMarks(), std::memory_order_relaxed); if (ranges.ranges.empty()) stat.parts_dropped.fetch_add(1, std::memory_order_relaxed); @@ -1574,7 +1571,6 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( const MarkRanges & ranges, const Settings & settings, const MergeTreeReaderSettings & reader_settings, - size_t & granules_dropped, MarkCache * mark_cache, UncompressedCache * uncompressed_cache, Poco::Logger * log) @@ -1644,8 +1640,6 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( { // vector of indexes of useful ranges auto result = ann_condition->getUsefulRanges(granule); - if (result.empty()) - ++granules_dropped; for (auto range : result) { @@ -1670,10 +1664,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( result = cache_in_store.store ? gin_filter_condition->mayBeTrueOnGranuleInPart(granule, cache_in_store) : true; if (!result) - { - ++granules_dropped; continue; - } MarkRange data_range( std::max(ranges[i].begin, index_mark * index_granularity), @@ -1698,8 +1689,6 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingMergedIndex( const MarkRanges & ranges, const Settings & settings, const MergeTreeReaderSettings & reader_settings, - size_t & total_granules, - size_t & granules_dropped, MarkCache * mark_cache, UncompressedCache * uncompressed_cache, Poco::Logger * log) @@ -1756,8 +1745,6 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingMergedIndex( for (auto & reader : readers) reader->seek(index_range.begin); - total_granules += index_range.end - index_range.begin; - for (size_t index_mark = index_range.begin; index_mark < index_range.end; ++index_mark) { if (index_mark != index_range.begin || !granules_filled || last_index_mark != index_range.begin) @@ -1770,10 +1757,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingMergedIndex( } if (!condition->mayBeTrueOnGranule(granules)) - { - ++granules_dropped; continue; - } MarkRange data_range( std::max(range.begin, index_mark * index_granularity), diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index a5dcbfe66508..74d8d8e3c8f7 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -93,7 +93,6 @@ class MergeTreeDataSelectExecutor const MarkRanges & ranges, const Settings & settings, const MergeTreeReaderSettings & reader_settings, - size_t & granules_dropped, MarkCache * mark_cache, UncompressedCache * uncompressed_cache, Poco::Logger * log); @@ -105,8 +104,6 @@ class MergeTreeDataSelectExecutor const MarkRanges & ranges, const Settings & settings, const MergeTreeReaderSettings & reader_settings, - size_t & total_granules, - size_t & granules_dropped, MarkCache * mark_cache, UncompressedCache * uncompressed_cache, Poco::Logger * log); diff --git a/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.reference b/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.reference new file mode 100644 index 000000000000..09a64329b05a --- /dev/null +++ b/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.reference @@ -0,0 +1,29 @@ +Expression ((Projection + Before ORDER BY)) + Filter (WHERE) + ReadFromMergeTree (default.test_skip_idx) + Indexes: + Skip + Name: name_idx_g2 + Description: minmax GRANULARITY 2 + Parts: 1/1 + Granules: 2/5 + Skip + Name: name_idx_g1 + Description: minmax GRANULARITY 1 + Parts: 1/1 + Granules: 1/2 +-- +Expression ((Projection + Before ORDER BY)) + Filter (WHERE) + ReadFromMergeTree (default.test_skip_idx) + Indexes: + Skip + Name: name_idx_g2 + Description: minmax GRANULARITY 2 + Parts: 1/1 + Granules: 2/5 + Skip + Name: name_idx_g1 + Description: minmax GRANULARITY 1 + Parts: 1/1 + Granules: 2/2 diff --git a/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.sql b/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.sql new file mode 100644 index 000000000000..2c66d28203b6 --- /dev/null +++ b/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS test_skip_idx; + +CREATE TABLE test_skip_idx +( + `id` UInt32, + INDEX name_idx_g2 id TYPE minmax GRANULARITY 2, + INDEX name_idx_g1 id TYPE minmax GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS index_granularity = 1, index_granularity_bytes = 0; + +INSERT INTO test_skip_idx SELECT number FROM system.numbers LIMIT 5 OFFSET 1; + +EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 2; + +SELECT '--'; + +EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 3; + +DROP TABLE IF EXISTS test_skip_idx; From 0e2b81596071ab01728cb62acb0fb9b9e1ba9944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=BA=E5=8D=9A?= Date: Mon, 21 Aug 2023 17:47:55 +0800 Subject: [PATCH 02/11] update function test --- .../01786_explain_merge_tree.reference | 158 +++++++++--------- .../02865_size_of_marks_skip_idx_explain.sql | 21 --- ..._size_of_marks_skip_idx_explain.reference} | 6 +- .../02866_size_of_marks_skip_idx_explain.sh | 23 +++ 4 files changed, 103 insertions(+), 105 deletions(-) delete mode 100644 tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.sql rename tests/queries/0_stateless/{02865_size_of_marks_skip_idx_explain.reference => 02866_size_of_marks_skip_idx_explain.reference} (82%) create mode 100755 tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh diff --git a/tests/queries/0_stateless/01786_explain_merge_tree.reference b/tests/queries/0_stateless/01786_explain_merge_tree.reference index 794acc310ce2..84ed3cbe32b9 100644 --- a/tests/queries/0_stateless/01786_explain_merge_tree.reference +++ b/tests/queries/0_stateless/01786_explain_merge_tree.reference @@ -1,83 +1,83 @@ - ReadFromMergeTree (default.test_index) - Indexes: - MinMax - Keys: - y - Condition: (y in [1, +Inf)) - Parts: 4/5 - Granules: 11/12 - Partition - Keys: - y - bitAnd(z, 3) - Condition: and((y in [1, +Inf)), (bitAnd(z, 3) not in [1, 1])) - Parts: 3/4 - Granules: 10/11 - PrimaryKey - Keys: - x - y - Condition: and((x in [11, +Inf)), (y in [1, +Inf))) - Parts: 2/3 - Granules: 6/10 - Skip - Name: t_minmax - Description: minmax GRANULARITY 2 - Parts: 1/2 - Granules: 4/6 - Skip - Name: t_set - Description: set GRANULARITY 2 - Parts: 1/1 - Granules: 2/4 + ReadFromMergeTree (default.test_index) + Indexes: + MinMax + Keys: + y + Condition: (y in [1, +Inf)) + Parts: 4/5 + Granules: 11/12 + Partition + Keys: + y + bitAnd(z, 3) + Condition: and((y in [1, +Inf)), (bitAnd(z, 3) not in [1, 1])) + Parts: 3/4 + Granules: 10/11 + PrimaryKey + Keys: + x + y + Condition: and((x in [11, +Inf)), (y in [1, +Inf))) + Parts: 2/3 + Granules: 6/10 + Skip + Name: t_minmax + Description: minmax GRANULARITY 2 + Parts: 1/2 + Granules: 3/6 + Skip + Name: t_set + Description: set GRANULARITY 2 + Parts: 1/1 + Granules: 2/3 ----------------- - "Node Type": "ReadFromMergeTree", - "Description": "default.test_index", - "Indexes": [ - { - "Type": "MinMax", - "Keys": ["y"], - "Condition": "(y in [1, +Inf))", - "Initial Parts": 5, - "Selected Parts": 4, - "Initial Granules": 12, - "Selected Granules": 11 - }, - { - "Type": "Partition", - "Keys": ["y", "bitAnd(z, 3)"], - "Condition": "and((y in [1, +Inf)), (bitAnd(z, 3) not in [1, 1]))", - "Initial Parts": 4, - "Selected Parts": 3, - "Initial Granules": 11, - "Selected Granules": 10 - }, - { - "Type": "PrimaryKey", - "Keys": ["x", "y"], - "Condition": "and((x in [11, +Inf)), (y in [1, +Inf)))", - "Initial Parts": 3, - "Selected Parts": 2, - "Initial Granules": 10, - "Selected Granules": 6 - }, - { - "Type": "Skip", - "Name": "t_minmax", - "Description": "minmax GRANULARITY 2", - "Initial Parts": 2, - "Selected Parts": 1, - "Initial Granules": 6, - "Selected Granules": 4 - }, - { - "Type": "Skip", - "Name": "t_set", - "Description": "set GRANULARITY 2", - "Initial Parts": 1, - "Selected Parts": 1, - "Initial Granules": 4, - "Selected Granules": 2 + "Node Type": "ReadFromMergeTree", + "Description": "default.test_index", + "Indexes": [ + { + "Type": "MinMax", + "Keys": ["y"], + "Condition": "(y in [1, +Inf))", + "Initial Parts": 5, + "Selected Parts": 4, + "Initial Granules": 12, + "Selected Granules": 11 + }, + { + "Type": "Partition", + "Keys": ["y", "bitAnd(z, 3)"], + "Condition": "and((y in [1, +Inf)), (bitAnd(z, 3) not in [1, 1]))", + "Initial Parts": 4, + "Selected Parts": 3, + "Initial Granules": 11, + "Selected Granules": 10 + }, + { + "Type": "PrimaryKey", + "Keys": ["x", "y"], + "Condition": "and((x in [11, +Inf)), (y in [1, +Inf)))", + "Initial Parts": 3, + "Selected Parts": 2, + "Initial Granules": 10, + "Selected Granules": 6 + }, + { + "Type": "Skip", + "Name": "t_minmax", + "Description": "minmax GRANULARITY 2", + "Initial Parts": 2, + "Selected Parts": 1, + "Initial Granules": 6, + "Selected Granules": 3 + }, + { + "Type": "Skip", + "Name": "t_set", + "Description": "set GRANULARITY 2", + "Initial Parts": 1, + "Selected Parts": 1, + "Initial Granules": 3, + "Selected Granules": 2 } ] } diff --git a/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.sql b/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.sql deleted file mode 100644 index 2c66d28203b6..000000000000 --- a/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.sql +++ /dev/null @@ -1,21 +0,0 @@ -DROP TABLE IF EXISTS test_skip_idx; - -CREATE TABLE test_skip_idx -( - `id` UInt32, - INDEX name_idx_g2 id TYPE minmax GRANULARITY 2, - INDEX name_idx_g1 id TYPE minmax GRANULARITY 1 -) -ENGINE = MergeTree -ORDER BY tuple() -SETTINGS index_granularity = 1, index_granularity_bytes = 0; - -INSERT INTO test_skip_idx SELECT number FROM system.numbers LIMIT 5 OFFSET 1; - -EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 2; - -SELECT '--'; - -EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 3; - -DROP TABLE IF EXISTS test_skip_idx; diff --git a/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.reference b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference similarity index 82% rename from tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.reference rename to tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference index 09a64329b05a..e7f08b4f0675 100644 --- a/tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.reference +++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference @@ -1,5 +1,3 @@ -Expression ((Projection + Before ORDER BY)) - Filter (WHERE) ReadFromMergeTree (default.test_skip_idx) Indexes: Skip @@ -12,9 +10,7 @@ Expression ((Projection + Before ORDER BY)) Description: minmax GRANULARITY 1 Parts: 1/1 Granules: 1/2 --- -Expression ((Projection + Before ORDER BY)) - Filter (WHERE) +----------------- ReadFromMergeTree (default.test_skip_idx) Indexes: Skip diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh new file mode 100755 index 000000000000..aa8878f07a7f --- /dev/null +++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Tags: no-random-merge-tree-settings + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --optimize_move_to_prewhere=1 --convert_query_to_cnf=0 --optimize_read_in_order=1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_skip_idx" + +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_skip_idx (id UInt32, INDEX name_idx_g2 id TYPE minmax GRANULARITY 2, INDEX name_idx_g1 id TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1, index_granularity_bytes = 0" +$CLICKHOUSE_CLIENT -q "INSERT INTO test_skip_idx SELECT number FROM system.numbers LIMIT 5 OFFSET 1" + +$CLICKHOUSE_CLIENT -q " + EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 2 + " | grep -A 100 "ReadFromMergeTree" # | grep -v "Description" + +echo "-----------------" + +$CLICKHOUSE_CLIENT -q " + EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 3 + " | grep -A 100 "ReadFromMergeTree" # | grep -v "Description" From 0688bdea90dcebcd9e1d59913279f31d5d5fd6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=BA=E5=8D=9A?= Date: Mon, 21 Aug 2023 17:59:04 +0800 Subject: [PATCH 03/11] format code --- .../01786_explain_merge_tree.reference | 158 +++++++++--------- ...6_size_of_marks_skip_idx_explain.reference | 48 +++--- 2 files changed, 103 insertions(+), 103 deletions(-) diff --git a/tests/queries/0_stateless/01786_explain_merge_tree.reference b/tests/queries/0_stateless/01786_explain_merge_tree.reference index 84ed3cbe32b9..46f89745ba32 100644 --- a/tests/queries/0_stateless/01786_explain_merge_tree.reference +++ b/tests/queries/0_stateless/01786_explain_merge_tree.reference @@ -1,83 +1,83 @@ - ReadFromMergeTree (default.test_index) - Indexes: - MinMax - Keys: - y - Condition: (y in [1, +Inf)) - Parts: 4/5 - Granules: 11/12 - Partition - Keys: - y - bitAnd(z, 3) - Condition: and((y in [1, +Inf)), (bitAnd(z, 3) not in [1, 1])) - Parts: 3/4 - Granules: 10/11 - PrimaryKey - Keys: - x - y - Condition: and((x in [11, +Inf)), (y in [1, +Inf))) - Parts: 2/3 - Granules: 6/10 - Skip - Name: t_minmax - Description: minmax GRANULARITY 2 - Parts: 1/2 - Granules: 3/6 - Skip - Name: t_set - Description: set GRANULARITY 2 - Parts: 1/1 - Granules: 2/3 + ReadFromMergeTree (default.test_index) + Indexes: + MinMax + Keys: + y + Condition: (y in [1, +Inf)) + Parts: 4/5 + Granules: 11/12 + Partition + Keys: + y + bitAnd(z, 3) + Condition: and((y in [1, +Inf)), (bitAnd(z, 3) not in [1, 1])) + Parts: 3/4 + Granules: 10/11 + PrimaryKey + Keys: + x + y + Condition: and((x in [11, +Inf)), (y in [1, +Inf))) + Parts: 2/3 + Granules: 6/10 + Skip + Name: t_minmax + Description: minmax GRANULARITY 2 + Parts: 1/2 + Granules: 3/6 + Skip + Name: t_set + Description: set GRANULARITY 2 + Parts: 1/1 + Granules: 2/3 ----------------- - "Node Type": "ReadFromMergeTree", - "Description": "default.test_index", - "Indexes": [ - { - "Type": "MinMax", - "Keys": ["y"], - "Condition": "(y in [1, +Inf))", - "Initial Parts": 5, - "Selected Parts": 4, - "Initial Granules": 12, - "Selected Granules": 11 - }, - { - "Type": "Partition", - "Keys": ["y", "bitAnd(z, 3)"], - "Condition": "and((y in [1, +Inf)), (bitAnd(z, 3) not in [1, 1]))", - "Initial Parts": 4, - "Selected Parts": 3, - "Initial Granules": 11, - "Selected Granules": 10 - }, - { - "Type": "PrimaryKey", - "Keys": ["x", "y"], - "Condition": "and((x in [11, +Inf)), (y in [1, +Inf)))", - "Initial Parts": 3, - "Selected Parts": 2, - "Initial Granules": 10, - "Selected Granules": 6 - }, - { - "Type": "Skip", - "Name": "t_minmax", - "Description": "minmax GRANULARITY 2", - "Initial Parts": 2, - "Selected Parts": 1, - "Initial Granules": 6, - "Selected Granules": 3 - }, - { - "Type": "Skip", - "Name": "t_set", - "Description": "set GRANULARITY 2", - "Initial Parts": 1, - "Selected Parts": 1, - "Initial Granules": 3, - "Selected Granules": 2 + "Node Type": "ReadFromMergeTree", + "Description": "default.test_index", + "Indexes": [ + { + "Type": "MinMax", + "Keys": ["y"], + "Condition": "(y in [1, +Inf))", + "Initial Parts": 5, + "Selected Parts": 4, + "Initial Granules": 12, + "Selected Granules": 11 + }, + { + "Type": "Partition", + "Keys": ["y", "bitAnd(z, 3)"], + "Condition": "and((y in [1, +Inf)), (bitAnd(z, 3) not in [1, 1]))", + "Initial Parts": 4, + "Selected Parts": 3, + "Initial Granules": 11, + "Selected Granules": 10 + }, + { + "Type": "PrimaryKey", + "Keys": ["x", "y"], + "Condition": "and((x in [11, +Inf)), (y in [1, +Inf)))", + "Initial Parts": 3, + "Selected Parts": 2, + "Initial Granules": 10, + "Selected Granules": 6 + }, + { + "Type": "Skip", + "Name": "t_minmax", + "Description": "minmax GRANULARITY 2", + "Initial Parts": 2, + "Selected Parts": 1, + "Initial Granules": 6, + "Selected Granules": 3 + }, + { + "Type": "Skip", + "Name": "t_set", + "Description": "set GRANULARITY 2", + "Initial Parts": 1, + "Selected Parts": 1, + "Initial Granules": 3, + "Selected Granules": 2 } ] } diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference index e7f08b4f0675..6c1d98af57f2 100644 --- a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference +++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference @@ -1,25 +1,25 @@ - ReadFromMergeTree (default.test_skip_idx) - Indexes: - Skip - Name: name_idx_g2 - Description: minmax GRANULARITY 2 - Parts: 1/1 - Granules: 2/5 - Skip - Name: name_idx_g1 - Description: minmax GRANULARITY 1 - Parts: 1/1 - Granules: 1/2 + ReadFromMergeTree (default.test_skip_idx) + Indexes: + Skip + Name: name_idx_g2 + Description: minmax GRANULARITY 2 + Parts: 1/1 + Granules: 2/5 + Skip + Name: name_idx_g1 + Description: minmax GRANULARITY 1 + Parts: 1/1 + Granules: 1/2 ----------------- - ReadFromMergeTree (default.test_skip_idx) - Indexes: - Skip - Name: name_idx_g2 - Description: minmax GRANULARITY 2 - Parts: 1/1 - Granules: 2/5 - Skip - Name: name_idx_g1 - Description: minmax GRANULARITY 1 - Parts: 1/1 - Granules: 2/2 + ReadFromMergeTree (default.test_skip_idx) + Indexes: + Skip + Name: name_idx_g2 + Description: minmax GRANULARITY 2 + Parts: 1/1 + Granules: 2/5 + Skip + Name: name_idx_g1 + Description: minmax GRANULARITY 1 + Parts: 1/1 + Granules: 2/2 From 928d507dd976c8e18d285a9d65c5272ce113a383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=BA=E5=8D=9A?= Date: Mon, 21 Aug 2023 20:03:26 +0800 Subject: [PATCH 04/11] update test --- .../queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh index aa8878f07a7f..e09586fbf099 100755 --- a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh +++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh @@ -9,7 +9,7 @@ CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --optimize_move_to_prewhere=1 --convert_qu $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_skip_idx" -$CLICKHOUSE_CLIENT -q "CREATE TABLE test_skip_idx (id UInt32, INDEX name_idx_g2 id TYPE minmax GRANULARITY 2, INDEX name_idx_g1 id TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1, index_granularity_bytes = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE test_skip_idx (id UInt32, INDEX name_idx_g2 id TYPE minmax GRANULARITY 2, INDEX name_idx_g1 id TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0" $CLICKHOUSE_CLIENT -q "INSERT INTO test_skip_idx SELECT number FROM system.numbers LIMIT 5 OFFSET 1" $CLICKHOUSE_CLIENT -q " From a0be72c29e1b3c980783abcfd6b923d5aa780cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=BA=E5=8D=9A?= Date: Mon, 21 Aug 2023 20:31:38 +0800 Subject: [PATCH 05/11] update test --- ...6_size_of_marks_skip_idx_explain.reference | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference index 6c1d98af57f2..e7f08b4f0675 100644 --- a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference +++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference @@ -1,25 +1,25 @@ - ReadFromMergeTree (default.test_skip_idx) - Indexes: - Skip - Name: name_idx_g2 - Description: minmax GRANULARITY 2 - Parts: 1/1 - Granules: 2/5 - Skip - Name: name_idx_g1 - Description: minmax GRANULARITY 1 - Parts: 1/1 - Granules: 1/2 + ReadFromMergeTree (default.test_skip_idx) + Indexes: + Skip + Name: name_idx_g2 + Description: minmax GRANULARITY 2 + Parts: 1/1 + Granules: 2/5 + Skip + Name: name_idx_g1 + Description: minmax GRANULARITY 1 + Parts: 1/1 + Granules: 1/2 ----------------- - ReadFromMergeTree (default.test_skip_idx) - Indexes: - Skip - Name: name_idx_g2 - Description: minmax GRANULARITY 2 - Parts: 1/1 - Granules: 2/5 - Skip - Name: name_idx_g1 - Description: minmax GRANULARITY 1 - Parts: 1/1 - Granules: 2/2 + ReadFromMergeTree (default.test_skip_idx) + Indexes: + Skip + Name: name_idx_g2 + Description: minmax GRANULARITY 2 + Parts: 1/1 + Granules: 2/5 + Skip + Name: name_idx_g1 + Description: minmax GRANULARITY 1 + Parts: 1/1 + Granules: 2/2 From bd5d27e9f7a300751c12aba79b78ea711f084a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=99=BA=E5=8D=9A?= Date: Mon, 21 Aug 2023 23:13:02 +0800 Subject: [PATCH 06/11] update test --- tests/queries/0_stateless/02354_annoy_index.reference | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02354_annoy_index.reference b/tests/queries/0_stateless/02354_annoy_index.reference index 5e01a6e566e4..330fe28305ae 100644 --- a/tests/queries/0_stateless/02354_annoy_index.reference +++ b/tests/queries/0_stateless/02354_annoy_index.reference @@ -94,7 +94,7 @@ Expression ((Projection + Before ORDER BY)) Name: annoy_index Description: annoy GRANULARITY 2 Parts: 0/1 - Granules: 2/4 + Granules: 0/4 ORDER BY type, L2Distance, check that index is used Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) @@ -110,7 +110,7 @@ Expression (Projection) Name: annoy_index Description: annoy GRANULARITY 2 Parts: 1/1 - Granules: 4/4 + Granules: 2/4 --- Test with Array, GRANULARITY = 4, index_granularity = 4 --- WHERE type, L2Distance, check that index is used Expression ((Projection + Before ORDER BY)) @@ -125,7 +125,7 @@ Expression ((Projection + Before ORDER BY)) Name: annoy_index Description: annoy GRANULARITY 4 Parts: 0/1 - Granules: 3/4 + Granules: 0/4 ORDER BY type, L2Distance, check that index is used Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) @@ -141,4 +141,4 @@ Expression (Projection) Name: annoy_index Description: annoy GRANULARITY 4 Parts: 1/1 - Granules: 4/4 + Granules: 1/4 From 243657b518683ce97030f6ba89befa9655e33af5 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 23 Aug 2023 11:15:29 +0000 Subject: [PATCH 07/11] Convert to sql test, fix expected results of usearch test --- .../0_stateless/02354_usearch_index.reference | 8 +++---- ...6_size_of_marks_skip_idx_explain.reference | 5 +++- .../02866_size_of_marks_skip_idx_explain.sh | 23 ------------------- .../02866_size_of_marks_skip_idx_explain.sql | 20 ++++++++++++++++ 4 files changed, 28 insertions(+), 28 deletions(-) delete mode 100755 tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh create mode 100755 tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql diff --git a/tests/queries/0_stateless/02354_usearch_index.reference b/tests/queries/0_stateless/02354_usearch_index.reference index 9896f149d453..a93209f6ba8c 100644 --- a/tests/queries/0_stateless/02354_usearch_index.reference +++ b/tests/queries/0_stateless/02354_usearch_index.reference @@ -93,7 +93,7 @@ Expression ((Projection + Before ORDER BY)) Name: usearch_index Description: usearch GRANULARITY 2 Parts: 0/1 - Granules: 2/4 + Granules: 0/4 ORDER BY type, L2Distance, check that index is used Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) @@ -109,7 +109,7 @@ Expression (Projection) Name: usearch_index Description: usearch GRANULARITY 2 Parts: 1/1 - Granules: 4/4 + Granules: 2/4 --- Test with Array, GRANULARITY = 4, index_granularity = 4 --- WHERE type, L2Distance, check that index is used Expression ((Projection + Before ORDER BY)) @@ -124,7 +124,7 @@ Expression ((Projection + Before ORDER BY)) Name: usearch_index Description: usearch GRANULARITY 4 Parts: 0/1 - Granules: 3/4 + Granules: 0/4 ORDER BY type, L2Distance, check that index is used Expression (Projection) Limit (preliminary LIMIT (without OFFSET)) @@ -140,4 +140,4 @@ Expression (Projection) Name: usearch_index Description: usearch GRANULARITY 4 Parts: 1/1 - Granules: 4/4 + Granules: 1/4 diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference index e7f08b4f0675..9bbc59f4002f 100644 --- a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference +++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference @@ -1,3 +1,5 @@ +Expression ((Projection + Before ORDER BY)) + Filter (WHERE) ReadFromMergeTree (default.test_skip_idx) Indexes: Skip @@ -10,7 +12,8 @@ Description: minmax GRANULARITY 1 Parts: 1/1 Granules: 1/2 ------------------ +Expression ((Projection + Before ORDER BY)) + Filter (WHERE) ReadFromMergeTree (default.test_skip_idx) Indexes: Skip diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh deleted file mode 100755 index e09586fbf099..000000000000 --- a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-random-merge-tree-settings - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --optimize_move_to_prewhere=1 --convert_query_to_cnf=0 --optimize_read_in_order=1" - -$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_skip_idx" - -$CLICKHOUSE_CLIENT -q "CREATE TABLE test_skip_idx (id UInt32, INDEX name_idx_g2 id TYPE minmax GRANULARITY 2, INDEX name_idx_g1 id TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0" -$CLICKHOUSE_CLIENT -q "INSERT INTO test_skip_idx SELECT number FROM system.numbers LIMIT 5 OFFSET 1" - -$CLICKHOUSE_CLIENT -q " - EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 2 - " | grep -A 100 "ReadFromMergeTree" # | grep -v "Description" - -echo "-----------------" - -$CLICKHOUSE_CLIENT -q " - EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 3 - " | grep -A 100 "ReadFromMergeTree" # | grep -v "Description" diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql new file mode 100755 index 000000000000..1aa17d563932 --- /dev/null +++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql @@ -0,0 +1,20 @@ +-- Tags: no-random-merge-tree-settings + +SET optimize_move_to_prewhere = 1; +SET convert_query_to_cnf = 0; +SET optimize_read_in_order = 1; + +DROP TABLE IF EXISTS test_skip_idx; + +CREATE TABLE test_skip_idx ( + id UInt32, + INDEX name_idx_g2 id TYPE minmax GRANULARITY 2, + INDEX name_idx_g1 id TYPE minmax GRANULARITY 1) +ENGINE = MergeTree +ORDER BY tuple() +SETTINGS index_granularity = 1, index_granularity_bytes = 0, min_bytes_for_wide_part = 0; + +INSERT INTO test_skip_idx SELECT number FROM system.numbers LIMIT 5 OFFSET 1; + +EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 2; +EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 3; From 2c7a01525147b66001b6554a92df91243927960b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 23 Aug 2023 11:17:01 +0000 Subject: [PATCH 08/11] Drop table at end of test --- .../0_stateless/02866_size_of_marks_skip_idx_explain.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql index 1aa17d563932..1eac6df60e03 100755 --- a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql +++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql @@ -18,3 +18,5 @@ INSERT INTO test_skip_idx SELECT number FROM system.numbers LIMIT 5 OFFSET 1; EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 2; EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 3; + +DROP TABLE test_skip_idx; From bfa4ee75999e3edaf0cb4ddc07e4d2dc73ca8ceb Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 23 Aug 2023 12:16:22 +0000 Subject: [PATCH 09/11] Fix style check --- .../queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql old mode 100755 new mode 100644 From b4dbde89aa94962ef564222eaab18d4c6ff49b67 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 23 Aug 2023 12:44:21 +0000 Subject: [PATCH 10/11] Fix expected results --- tests/queries/0_stateless/02354_annoy_index.reference | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/02354_annoy_index.reference b/tests/queries/0_stateless/02354_annoy_index.reference index 74aaec3dcf98..a0ffb1e1f7ff 100644 --- a/tests/queries/0_stateless/02354_annoy_index.reference +++ b/tests/queries/0_stateless/02354_annoy_index.reference @@ -141,3 +141,9 @@ Expression (Projection) Name: annoy_index Description: annoy GRANULARITY 4 Parts: 1/1 + Granules: 1/4 +--- Test correctness of Annoy index with > 1 mark +1 [1,0,0,0] +9000 [9000,0,0,0] +1 (1,0,0,0) +9000 (9000,0,0,0) From 8365a3623c3539531b7323755152f57b2cd4e97e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 24 Aug 2023 10:23:17 +0000 Subject: [PATCH 11/11] Force-enable analyzer --- .../02866_size_of_marks_skip_idx_explain.reference | 8 ++++---- .../0_stateless/02866_size_of_marks_skip_idx_explain.sql | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference index 9bbc59f4002f..d3502e9893cf 100644 --- a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference +++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.reference @@ -1,5 +1,5 @@ -Expression ((Projection + Before ORDER BY)) - Filter (WHERE) +Expression ((Project names + Projection)) + Filter ((WHERE + Change column names to column identifiers)) ReadFromMergeTree (default.test_skip_idx) Indexes: Skip @@ -12,8 +12,8 @@ Expression ((Projection + Before ORDER BY)) Description: minmax GRANULARITY 1 Parts: 1/1 Granules: 1/2 -Expression ((Projection + Before ORDER BY)) - Filter (WHERE) +Expression ((Project names + Projection)) + Filter ((WHERE + Change column names to column identifiers)) ReadFromMergeTree (default.test_skip_idx) Indexes: Skip diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql index 1eac6df60e03..b916c5ca13da 100644 --- a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql +++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql @@ -4,6 +4,8 @@ SET optimize_move_to_prewhere = 1; SET convert_query_to_cnf = 0; SET optimize_read_in_order = 1; +SET allow_experimental_analyzer = 1; -- slightly different operator names than w/o + DROP TABLE IF EXISTS test_skip_idx; CREATE TABLE test_skip_idx (