Skip to content

25.3.8 Backport of #79969 - Fix match(col, '^…') index analysis with escaped metacharacters to avoid wrong results and crashes #1122

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 5, 2025
Merged

25.3.8 Backport of #79969 - Fix match(col, '^…') index analysis with escaped metacharacters to avoid wrong results and crashes #1122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions src/Storages/MergeTree/KeyCondition.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ static String extractFixedPrefixFromRegularExpression(const String & regexp)
const char * pos = begin;
const char * end = regexp.data() + regexp.size();

while (pos != end)
while (pos < end)
{
switch (*pos)
{
Expand All @@ -149,19 +149,22 @@ static String extractFixedPrefixFromRegularExpression(const String & regexp)
case '$':
case '.':
case '[':
case ']':
case '?':
case '*':
case '+':
case '\\':
case '{':
case '}':
case '-':
fixed_prefix += *pos;
++pos;
break;
default:
/// all other escape sequences are not supported
pos = end;
break;
}

++pos;
break;
}

Expand Down
52 changes: 52 additions & 0 deletions tests/queries/0_stateless/03456_match_index_prefix_extraction.reference
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
Condition: (path in [\'xxx(zzz\', \'xxx(zz{\'))
1
Condition: (path in [\'xxx)zzz\', \'xxx)zz{\'))
1
Condition: (path in [\'xxx^zzz\', \'xxx^zz{\'))
1
Condition: (path in [\'xxx$zzz\', \'xxx$zz{\'))
1
Condition: (path in [\'xxx.zzz\', \'xxx.zz{\'))
1
Condition: (path in [\'xxx[zzz\', \'xxx[zz{\'))
1
Condition: (path in [\'xxx]zzz\', \'xxx]zz{\'))
1
Condition: (path in [\'xxx?zzz\', \'xxx?zz{\'))
1
Condition: (path in [\'xxx*zzz\', \'xxx*zz{\'))
1
Condition: (path in [\'xxx+zzz\', \'xxx+zz{\'))
1
Condition: (path in [\'xxx\\\\zzz\', \'xxx\\\\zz{\'))
1
Condition: (path in [\'xxx{zzz\', \'xxx{zz{\'))
1
Condition: (path in [\'xxx}zzz\', \'xxx}zz{\'))
1
Condition: (path in [\'xxx-zzz\', \'xxx-zz{\'))
1
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
15
Condition: (path in [\'xxx\', \'xxy\'))
15
Condition: (path in [\'xxx\', \'xxy\'))
15
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: true
Condition: true
111 changes: 111 additions & 0 deletions tests/queries/0_stateless/03456_match_index_prefix_extraction.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
SET parallel_replicas_local_plan=1;

drop table if exists foo;

CREATE TABLE foo (id UInt8, path String) engine = MergeTree ORDER BY (path) SETTINGS index_granularity=1;

INSERT INTO foo VALUES (1, 'xxx|yyy'),
(2, 'xxx(zzz'),
(3, 'xxx)zzz'),
(4, 'xxx^zzz'),
(5, 'xxx$zzz'),
(6, 'xxx.zzz'),
(7, 'xxx[zzz'),
(8, 'xxx]zzz'),
(9, 'xxx?zzz'),
(10, 'xxx*zzz'),
(11, 'xxx+zzz'),
(12, 'xxx\\zzz'),
(13, 'xxx{zzz'),
(14, 'xxx}zzz'),
(15, 'xxx-zzz');


-- check if also escaped sequence are properly extracted
SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\(zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\(zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\)zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\)zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\^zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\^zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\$zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\$zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\.zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\.zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\[zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\[zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\]zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\]zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\?zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\?zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\*zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\*zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\+zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\+zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\\\zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\\\zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\{zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\{zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\}zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\}zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\-zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\-zzz') SETTINGS force_primary_key = 1;


-- those regex chars prevent the index use (only 3 first chars used during index scan)
SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\0bla')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\0bla') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx(bla)')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx(bla)') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx[bla]')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx[bla]') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx^bla')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx^bla') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx.bla')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx.bla') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx+bla')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx+bla') SETTINGS force_primary_key = 1;


-- here the forth char is not used during index, because it has 0+ quantifier
SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxxx{0,1}')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxxx{0,1}') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxxx?')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxxx?') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxxx*')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxxx*') SETTINGS force_primary_key = 1;

-- some unsupported regex chars - only 3 first chars used during index scan
SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\d+')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\d+') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\w+')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\w+') SETTINGS force_primary_key = 1;


-- fully disabled for pipes - see https://github.com/ClickHouse/ClickHouse/pull/54696
SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\|zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\|zzz') SETTINGS force_primary_key = 1; -- { serverError INDEX_NOT_USED }

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxxx|foo')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxxx|foo') SETTINGS force_primary_key = 1; -- { serverError INDEX_NOT_USED }
Loading