Skip to content

/ manticoresearch

Permalink
Browse files

fixed #463 crash on JSON field IN with large values; added regression… 

…s to test 361
  • Loading branch information
@tomatolog
tomatolog committed Sep 12, 2018
1 parent 5fb8d2a commit 5ed92c4b66da6423fa28f11b1fd59103ed1e1fb9
Unified Split
Showing with 58 additions and 16 deletions.
  1. +16 −12 src/sphinxexpr.cpp
  2. +3 −3 src/sphinxexpr.y
  3. +1 −1 test/test_361/model.bin
  4. +38 −0 test/test_361/test.xml
28 src/sphinxexpr.cpp
View file
@@ -2675,6 +2675,7 @@ class ConstList_c
CSphVector<float> m_dFloats; ///< float storage
ESphAttr m_eRetType { SPH_ATTR_INTEGER }; ///< SPH_ATTR_INTEGER, SPH_ATTR_BIGINT, SPH_ATTR_STRING, or SPH_ATTR_FLOAT
CSphString m_sExpr; ///< m_sExpr copy for TOK_CONST_STRING evaluation
bool m_bPackedStrings = false;

public:

@@ -2803,7 +2804,7 @@ class ExprParser_t
int AddNodeRand ( int iArg );
int AddNodeUdf ( int iCall, int iArg );
int AddNodePF ( int iFunc, int iArg );
int AddNodeConstlist ( int64_t iValue );
int AddNodeConstlist ( int64_t iValue, bool bPackedString );
int AddNodeConstlist ( float iValue );
void AppendToConstlist ( int iNode, int64_t iValue );
void AppendToConstlist ( int iNode, float iValue );
@@ -5496,19 +5497,21 @@ class Expr_JsonFieldIn_c : public Expr_ArgVsConstSet_c<int64_t>
const char * sExpr = pConsts->m_sExpr.cstr();
int iExprLen = pConsts->m_sExpr.Length();

for ( int64_t iVal : m_dValues )
if ( pConsts->m_bPackedStrings )
{
auto iOfs = (int)( iVal>>32 );
auto iLen = (int)( iVal & 0xffffffffUL );
if ( iOfs>0 && iOfs+iLen<=iExprLen )
for ( int64_t iVal : m_dValues )
{
CSphString sRes;
SqlUnescape ( sRes, sExpr + iOfs, iLen );
m_dHashes.Add ( sphFNV64 ( sRes.cstr(), sRes.Length() ) );
auto iOfs = (int)( iVal>>32 );
auto iLen = (int)( iVal & 0xffffffffUL );
if ( iOfs>0 && iLen>0 && iOfs+iLen<=iExprLen )
{
CSphString sRes;
SqlUnescape ( sRes, sExpr + iOfs, iLen );
m_dHashes.Add ( sphFNV64 ( sRes.cstr(), sRes.Length() ) );
}
}
m_dHashes.Sort();
}

m_dHashes.Sort();
}

Expr_JsonFieldIn_c ( UservarIntSet_c * pUserVar, ISphExpr * pArg )
@@ -5706,7 +5709,7 @@ class Expr_StrIn_c : public Expr_ArgVsConstSet_c<int64_t>, public ExprLocatorTra
, m_pUservar ( pUservar )
{
assert ( tLoc.m_iBitOffset>=0 && tLoc.m_iBitCount>0 );
assert ( !pConsts || !pUservar );
assert ( !pConsts || !pUservar || !pConsts->m_bPackedStrings );

m_fnStrCmp = GetCollationFn ( eCollation );

@@ -7335,13 +7338,14 @@ int ExprParser_t::AddNodePF ( int iFunc, int iArg )
return m_dNodes.GetLength()-1;
}

int ExprParser_t::AddNodeConstlist ( int64_t iValue )
int ExprParser_t::AddNodeConstlist ( int64_t iValue, bool bPackedString )
{
ExprNode_t & tNode = m_dNodes.Add();
tNode.m_iToken = TOK_CONST_LIST;
tNode.m_pConsts = new ConstList_c();
tNode.m_pConsts->Add ( iValue );
tNode.m_pConsts->m_sExpr = m_sExpr;
tNode.m_pConsts->m_bPackedStrings = bPackedString;
return m_dNodes.GetLength()-1;
}

6 src/sphinxexpr.y
View file
@@ -171,8 +171,8 @@ arglist:
;

constlist:
TOK_CONST_INT { $$ = pParser->AddNodeConstlist ( $1 ); }
| '-' TOK_CONST_INT { $$ = pParser->AddNodeConstlist ( -$2 );}
TOK_CONST_INT { $$ = pParser->AddNodeConstlist ( $1, false ); }
| '-' TOK_CONST_INT { $$ = pParser->AddNodeConstlist ( -$2, false );}
| TOK_CONST_FLOAT { $$ = pParser->AddNodeConstlist ( $1 ); }
| '-' TOK_CONST_FLOAT { $$ = pParser->AddNodeConstlist ( -$2 );}
| constlist ',' TOK_CONST_INT { pParser->AppendToConstlist ( $$, $3 ); }
@@ -182,7 +182,7 @@ constlist:
;

stringlist:
TOK_CONST_STRING { $$ = pParser->AddNodeConstlist ( $1 ); }
TOK_CONST_STRING { $$ = pParser->AddNodeConstlist ( $1, true ); }
| stringlist ',' TOK_CONST_STRING { pParser->AppendToConstlist ( $$, $3 ); }
;

2 test/test_361/model.bin
View file
@@ -1 +1 @@
a:1:{i:0;a:15:{i:0;a:3:{s:8:"sphinxql";s:18:"select * from test";s:10:"total_rows";i:2;s:4:"rows";a:2:{i:0;a:3:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";s:1:"j";s:15:"{"a":5,"b":123}";}i:1;a:3:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:1:"j";s:17:"{"a":101,"b":134}";}}}i:1;a:3:{s:8:"sphinxql";s:39:"select * from test where j.a IN (5,100)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:3:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";s:1:"j";s:15:"{"a":5,"b":123}";}}}i:2;a:3:{s:8:"sphinxql";s:57:"select *, IN(j.b, 100, 134) as cnd from test where cnd<>0";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:1:"j";s:17:"{"a":101,"b":134}";s:3:"cnd";s:1:"1";}}}i:3;a:3:{s:8:"sphinxql";s:56:"select *, j.b IN(100, 134) as cnd from test where cnd<>0";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:1:"j";s:17:"{"a":101,"b":134}";s:3:"cnd";s:1:"1";}}}i:4;a:3:{s:8:"sphinxql";s:39:"select * from dist where j.a IN (5,100)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:3:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";s:1:"j";s:15:"{"a":5,"b":123}";}}}i:5;a:3:{s:8:"sphinxql";s:57:"select *, IN(j.b, 100, 134) as cnd from dist where cnd<>0";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:1:"j";s:17:"{"a":101,"b":134}";s:3:"cnd";s:1:"1";}}}i:6;a:3:{s:8:"sphinxql";s:56:"select *, j.b IN(100, 134) as cnd from dist where cnd<>0";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:1:"j";s:17:"{"a":101,"b":134}";s:3:"cnd";s:1:"1";}}}i:7;a:2:{s:8:"sphinxql";s:62:"INSERT INTO pq (query, filters) VALUES ('', 'j.a IN (5, 100)')";s:14:"total_affected";i:1;}i:8;a:2:{s:8:"sphinxql";s:63:"INSERT INTO pq (query, filters) VALUES ('', 'IN (j.a, 5, 100)')";s:14:"total_affected";i:1;}i:9;a:2:{s:8:"sphinxql";s:82:"INSERT INTO pq (query, filters) VALUES ('', 'j.a IN (5, 100) and CRC32(\'test\')')";s:14:"total_affected";i:1;}i:10;a:2:{s:8:"sphinxql";s:83:"INSERT INTO pq (query, filters) VALUES ('', 'IN (j.a, 5, 100) and CRC32(\'test\')')";s:14:"total_affected";i:1;}i:11;a:2:{s:8:"sphinxql";s:84:"INSERT INTO pq (query, filters) VALUES ('', 'j.a IN (5, 100) or IN (j.b, 134, 200)')";s:14:"total_affected";i:1;}i:12;a:2:{s:8:"sphinxql";s:84:"INSERT INTO pq (query, filters) VALUES ('', 'IN (j.a, 5, 100) or j.b IN (134, 200)')";s:14:"total_affected";i:1;}i:13;a:3:{s:8:"sphinxql";s:93:"CALL PQ ('pq', ('{"j":{"a":5, "b":123}}', '{"j":{"a":101, "b":134}}'), 1 as docs, 1 as query)";s:10:"total_rows";i:6;s:4:"rows";a:6:{i:0;a:5:{s:3:"UID";s:1:"1";s:9:"Documents";s:1:"1";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:15:" j.a IN (5,100)";}i:1;a:5:{s:3:"UID";s:1:"2";s:9:"Documents";s:1:"1";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:17:" IN (j.a, 5, 100)";}i:2;a:5:{s:3:"UID";s:1:"3";s:9:"Documents";s:1:"1";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:34:" j.a IN (5, 100) and CRC32('test')";}i:3;a:5:{s:3:"UID";s:1:"4";s:9:"Documents";s:1:"1";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:35:" IN (j.a, 5, 100) and CRC32('test')";}i:4;a:5:{s:3:"UID";s:1:"5";s:9:"Documents";s:3:"1,2";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:38:" j.a IN (5, 100) or IN (j.b, 134, 200)";}i:5;a:5:{s:3:"UID";s:1:"6";s:9:"Documents";s:3:"1,2";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:38:" IN (j.a, 5, 100) or j.b IN (134, 200)";}}}i:14;a:3:{s:8:"sphinxql";s:39:"select *, IN (missed.a,5,100) from test";s:5:"error";s:47:"index test: parse error: unknown column: missed";s:5:"errno";i:1064;}}}
a:1:{i:0;a:18:{i:0;a:3:{s:8:"sphinxql";s:18:"select * from test";s:10:"total_rows";i:2;s:4:"rows";a:2:{i:0;a:3:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";s:1:"j";s:15:"{"a":5,"b":123}";}i:1;a:3:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:1:"j";s:17:"{"a":101,"b":134}";}}}i:1;a:3:{s:8:"sphinxql";s:39:"select * from test where j.a IN (5,100)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:3:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";s:1:"j";s:15:"{"a":5,"b":123}";}}}i:2;a:3:{s:8:"sphinxql";s:57:"select *, IN(j.b, 100, 134) as cnd from test where cnd<>0";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:1:"j";s:17:"{"a":101,"b":134}";s:3:"cnd";s:1:"1";}}}i:3;a:3:{s:8:"sphinxql";s:56:"select *, j.b IN(100, 134) as cnd from test where cnd<>0";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:1:"j";s:17:"{"a":101,"b":134}";s:3:"cnd";s:1:"1";}}}i:4;a:3:{s:8:"sphinxql";s:39:"select * from dist where j.a IN (5,100)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:3:{s:2:"id";s:1:"1";s:3:"gid";s:1:"1";s:1:"j";s:15:"{"a":5,"b":123}";}}}i:5;a:3:{s:8:"sphinxql";s:57:"select *, IN(j.b, 100, 134) as cnd from dist where cnd<>0";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:1:"j";s:17:"{"a":101,"b":134}";s:3:"cnd";s:1:"1";}}}i:6;a:3:{s:8:"sphinxql";s:56:"select *, j.b IN(100, 134) as cnd from dist where cnd<>0";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:1:"1";s:1:"j";s:17:"{"a":101,"b":134}";s:3:"cnd";s:1:"1";}}}i:7;a:2:{s:8:"sphinxql";s:62:"INSERT INTO pq (query, filters) VALUES ('', 'j.a IN (5, 100)')";s:14:"total_affected";i:1;}i:8;a:2:{s:8:"sphinxql";s:63:"INSERT INTO pq (query, filters) VALUES ('', 'IN (j.a, 5, 100)')";s:14:"total_affected";i:1;}i:9;a:2:{s:8:"sphinxql";s:82:"INSERT INTO pq (query, filters) VALUES ('', 'j.a IN (5, 100) and CRC32(\'test\')')";s:14:"total_affected";i:1;}i:10;a:2:{s:8:"sphinxql";s:83:"INSERT INTO pq (query, filters) VALUES ('', 'IN (j.a, 5, 100) and CRC32(\'test\')')";s:14:"total_affected";i:1;}i:11;a:2:{s:8:"sphinxql";s:84:"INSERT INTO pq (query, filters) VALUES ('', 'j.a IN (5, 100) or IN (j.b, 134, 200)')";s:14:"total_affected";i:1;}i:12;a:2:{s:8:"sphinxql";s:84:"INSERT INTO pq (query, filters) VALUES ('', 'IN (j.a, 5, 100) or j.b IN (134, 200)')";s:14:"total_affected";i:1;}i:13;a:3:{s:8:"sphinxql";s:93:"CALL PQ ('pq', ('{"j":{"a":5, "b":123}}', '{"j":{"a":101, "b":134}}'), 1 as docs, 1 as query)";s:10:"total_rows";i:6;s:4:"rows";a:6:{i:0;a:5:{s:3:"UID";s:1:"1";s:9:"Documents";s:1:"1";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:15:" j.a IN (5,100)";}i:1;a:5:{s:3:"UID";s:1:"2";s:9:"Documents";s:1:"1";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:17:" IN (j.a, 5, 100)";}i:2;a:5:{s:3:"UID";s:1:"3";s:9:"Documents";s:1:"1";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:34:" j.a IN (5, 100) and CRC32('test')";}i:3;a:5:{s:3:"UID";s:1:"4";s:9:"Documents";s:1:"1";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:35:" IN (j.a, 5, 100) and CRC32('test')";}i:4;a:5:{s:3:"UID";s:1:"5";s:9:"Documents";s:3:"1,2";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:38:" j.a IN (5, 100) or IN (j.b, 134, 200)";}i:5;a:5:{s:3:"UID";s:1:"6";s:9:"Documents";s:3:"1,2";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:38:" IN (j.a, 5, 100) or j.b IN (134, 200)";}}}i:14;a:3:{s:8:"sphinxql";s:39:"select *, IN (missed.a,5,100) from test";s:5:"error";s:47:"index test: parse error: unknown column: missed";s:5:"errno";i:1064;}i:15;a:2:{s:8:"sphinxql";s:107:"INSERT INTO pq_in (query, filters) VALUES ('', '(j.a=5 OR (j.a=6 AND NOT IN(j.b, 1220431865,6663093570)))')";s:14:"total_affected";i:1;}i:16;a:3:{s:8:"sphinxql";s:94:"CALL PQ ('pq_in', ('{"j":{"a":5, "b":123}}', '{"j":{"a":6, "b":134}}'), 1 as docs, 1 as query)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:3:"UID";s:1:"1";s:9:"Documents";s:3:"1,2";s:5:"Query";s:0:"";s:4:"Tags";s:0:"";s:7:"Filters";s:58:" (j.a=5 OR (j.a=6 AND NOT IN(j.b, 1220431865,6663093570)))";}}}i:17;a:3:{s:8:"sphinxql";s:56:"select *, in(j.a,1220431865,6663093570) as p from arrays";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:3:{s:2:"id";s:3:"123";s:1:"j";s:36:"{"a":[1,2,3,4],"t":["t1","t2","t3"]}";s:1:"p";s:1:"0";}i:1;a:3:{s:2:"id";s:3:"234";s:1:"j";s:36:"{"a":[2,3,4,5],"t":["t2","t3","t4"]}";s:1:"p";s:1:"0";}i:2;a:3:{s:2:"id";s:3:"345";s:1:"j";s:36:"{"a":[3,4,5,1],"t":["t3","t4","t5"]}";s:1:"p";s:1:"0";}i:3;a:3:{s:2:"id";s:3:"456";s:1:"j";s:44:"{"a":["4","5","6","2"],"t":["t4","t5","t6"]}";s:1:"p";s:1:"0";}}}}}
38 test/test_361/test.xml
View file
@@ -43,6 +43,28 @@ index dist
type = distributed
agent = <my_address/>:test
}

index pq_in
{
type = percolate
path = <data_path/>/pq_in
rt_field = title
rt_attr_json = j
}

source j_arrays
{
type = mysql
<sql_settings/>
sql_query = SELECT * from j_arrays
sql_attr_json = j
}

index arrays
{
source = j_arrays
path = <data_path/>/j_arrays
}
</config>

<db_create>
@@ -61,6 +83,16 @@ INSERT INTO `test_table` VALUES
(2,1,'test two','{"a":101,"b":134}')
</db_insert>

<db_create>CREATE TABLE j_arrays ( id INTEGER NOT NULL, gid INTEGER NOT NULL, j VARCHAR(255) NOT NULL )</db_create>
<db_drop>DROP TABLE IF EXISTS `j_arrays`;</db_drop>
<db_insert>
INSERT INTO j_arrays ( id, gid, j ) VALUES
( 123, 1, '{"a":[1,2,3,4], "t":["t1", "t2", "t3"]}' ),
( 234, 1, '{"a":[2,3,4,5], "t":["t2", "t3", "t4"]}' ),
( 345, 2, '{"a":[3,4,5,1], "t":["t3", "t4", "t5"]}' ),
( 456, 2, '{"a":["4","5","6","2"], "t":["t4", "t5", "t6"]}' )
</db_insert>

<sphqueries>
<!-- IN for plain indexes -->
<sphinxql>select * from test</sphinxql>
@@ -85,5 +117,11 @@ INSERT INTO `test_table` VALUES

<!-- crash on IN for plain indexes -->
<sphinxql>select *, IN (missed.a,5,100) from test</sphinxql>

<!-- crash on IN with large values -->
<sphinxql>INSERT INTO pq_in (query, filters) VALUES ('', '(j.a=5 OR (j.a=6 AND NOT IN(j.b, 1220431865,6663093570)))')</sphinxql>
<sphinxql>CALL PQ ('pq_in', ('{"j":{"a":5, "b":123}}', '{"j":{"a":6, "b":134}}'), 1 as docs, 1 as query)</sphinxql>
<sphinxql>select *, in(j.a,1220431865,6663093570) as p from arrays</sphinxql>

</sphqueries>
</test>

0 comments on commit 5ed92c4

Please sign in to comment.
You can’t perform that action at this time.