Skip to content
Permalink
Browse files

fixed #700 field at JSON document got altered by index stripper cause…

…s wrong match from sibling field; added regressions to test 359
  • Loading branch information...
tomatolog committed Feb 20, 2019
1 parent 645d9fd commit 75cd1342d05cdc352e3dd145cdbc6f79394a165b
Showing with 51 additions and 3 deletions.
  1. +31 −2 src/searchd.cpp
  2. +1 −1 test/test_359/model.bin
  3. +19 −0 test/test_359/test.xml
@@ -12588,7 +12588,25 @@ static bool ParseBsonDocument ( const VecTraits_T<BYTE>& dDoc, const CSphHash<Sc
{
if ( pItem->m_iField!=-1 && dChild.IsString () )
{
dFields[pItem->m_iField] = Vector<const char> ( dChild );
// stripper prior to build hits does not preserve field length
// but works with \0 strings and could walk all document and modifies it and alter field length
const VecTraits_T<const char> tField = Vector<const char> ( dChild );
if ( tField.GetLength() )
{
int64_t iOff = tStrings.m_dPackedData.GetLength();

// copy field content with tail zeroes
BYTE * pDst = tStrings.m_dPackedData.AddN ( tField.GetLength() + 1 + CSphString::GetGap() );
memcpy ( pDst, tField.Begin(), tField.GetLength() );
memset ( pDst + tField.GetLength(), 0, 1 + CSphString::GetGap() );

// pack offset into pointer then restore pointer after m_dPackedData filed
dFields[pItem->m_iField] = VecTraits_T<const char> ( (const char *)iOff, tField.GetLength() );
} else
{
dFields[pItem->m_iField] = tField;
}

if ( pItem==pId )
sMsg.Warn ( "field '%s' requested as docs_id identifier, but it is field!", sName.cstr() );
} else
@@ -13077,7 +13095,7 @@ static void PQLocalMatch ( const BlobVec_t &dDocs, const CSphString& sIndex, con
}
} else
{
// even withoout JSON docs MVA should match to schema definition on inserting data into accumulator
// even without JSON docs MVA should match to schema definition on inserting data into accumulator
for ( int i = 0; i<iAttrsCount; ++i )
{
const CSphColumnInfo &tCol = tSchema.GetAttr ( i );
@@ -13135,6 +13153,17 @@ static void PQLocalMatch ( const BlobVec_t &dDocs, const CSphString& sIndex, con
}

tStrings.SavePointersTo ( dStrings, false );

// convert back offset into tStrings buffer into pointers
for ( VecTraits_T<const char> & tField : dFields )
{
if ( !tField.GetLength() )
continue;

int64_t iOff = int64_t( tField.Begin() );
int iLen = tField.GetLength();
tField = VecTraits_T<const char> ( (const char *)( tStrings.m_dPackedData.Begin()+iOff ), iLen );
}
}
FixParsedMva ( dMvaParsed, dMva, iMvaCounter );

@@ -1 +1 @@
a:1:{i:0;a:26:{i:0;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:1;a:2:{s:8:"sphinxql";s:76:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:2;a:2:{s:8:"sphinxql";s:88:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:3;a:3:{s:8:"sphinxql";s:113:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"1";s:9:"documents";s:2:"21";s:5:"query";s:0:"";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:4;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:5;a:2:{s:8:"sphinxql";s:88:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:6;a:2:{s:8:"sphinxql";s:76:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:7;a:3:{s:8:"sphinxql";s:113:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"2";s:9:"documents";s:2:"21";s:5:"query";s:0:"";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:8;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:9;a:2:{s:8:"sphinxql";s:88:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:10;a:2:{s:8:"sphinxql";s:87:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.staff=\'got some\'')";s:14:"total_affected";i:1;}i:11;a:2:{s:8:"sphinxql";s:76:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:12;a:3:{s:8:"sphinxql";s:113:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"3";s:9:"documents";s:2:"21";s:5:"query";s:0:"";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:13;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:14;a:2:{s:8:"sphinxql";s:80:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:15;a:2:{s:8:"sphinxql";s:92:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:16;a:3:{s:8:"sphinxql";s:132:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}, "title":"test me"}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"1";s:9:"documents";s:2:"21";s:5:"query";s:4:"test";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:17;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:18;a:2:{s:8:"sphinxql";s:92:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:19;a:2:{s:8:"sphinxql";s:80:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:20;a:3:{s:8:"sphinxql";s:132:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}, "title":"test me"}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"2";s:9:"documents";s:2:"21";s:5:"query";s:4:"test";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:21;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:22;a:2:{s:8:"sphinxql";s:92:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:23;a:2:{s:8:"sphinxql";s:91:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.staff=\'got some\'')";s:14:"total_affected";i:1;}i:24;a:2:{s:8:"sphinxql";s:80:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:25;a:3:{s:8:"sphinxql";s:132:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}, "title":"test me"}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"3";s:9:"documents";s:2:"21";s:5:"query";s:4:"test";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}}}
a:1:{i:0;a:30:{i:0;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:1;a:2:{s:8:"sphinxql";s:76:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:2;a:2:{s:8:"sphinxql";s:88:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:3;a:3:{s:8:"sphinxql";s:113:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"1";s:9:"documents";s:2:"21";s:5:"query";s:0:"";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:4;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:5;a:2:{s:8:"sphinxql";s:88:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:6;a:2:{s:8:"sphinxql";s:76:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:7;a:3:{s:8:"sphinxql";s:113:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"2";s:9:"documents";s:2:"21";s:5:"query";s:0:"";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:8;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:9;a:2:{s:8:"sphinxql";s:88:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:10;a:2:{s:8:"sphinxql";s:87:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.staff=\'got some\'')";s:14:"total_affected";i:1;}i:11;a:2:{s:8:"sphinxql";s:76:"INSERT INTO pq (query, filters) VALUES ('', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:12;a:3:{s:8:"sphinxql";s:113:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"3";s:9:"documents";s:2:"21";s:5:"query";s:0:"";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:13;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:14;a:2:{s:8:"sphinxql";s:80:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:15;a:2:{s:8:"sphinxql";s:92:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:16;a:3:{s:8:"sphinxql";s:132:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}, "title":"test me"}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"1";s:9:"documents";s:2:"21";s:5:"query";s:4:"test";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:17;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:18;a:2:{s:8:"sphinxql";s:92:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:19;a:2:{s:8:"sphinxql";s:80:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:20;a:3:{s:8:"sphinxql";s:132:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}, "title":"test me"}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"2";s:9:"documents";s:2:"21";s:5:"query";s:4:"test";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:21;a:2:{s:8:"sphinxql";s:19:"TRUNCATE rtindex pq";s:14:"total_affected";i:0;}i:22;a:2:{s:8:"sphinxql";s:92:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.favorited=\'false\'')";s:14:"total_affected";i:1;}i:23;a:2:{s:8:"sphinxql";s:91:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.staff=\'got some\'')";s:14:"total_affected";i:1;}i:24;a:2:{s:8:"sphinxql";s:80:"INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.state=0')";s:14:"total_affected";i:1;}i:25;a:3:{s:8:"sphinxql";s:132:"CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}, "title":"test me"}'), 1 as docs, 1 as query, 'id' as docs_id)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"3";s:9:"documents";s:2:"21";s:5:"query";s:4:"test";s:4:"tags";s:0:"";s:7:"filters";s:29:"json_data.text.status.state=0";}}}i:26;a:2:{s:8:"sphinxql";s:43:"INSERT INTO pq1 (query) VALUES ('@title c')";s:14:"total_affected";i:1;}i:27;a:2:{s:8:"sphinxql";s:76:"CALL PQ('pq1', ('{\"title\":\"a\",\"content\":\"c\"}'),1 as docs,1 as query)";s:10:"total_rows";i:0;}i:28;a:2:{s:8:"sphinxql";s:60:"CALL PQ('pq1', ('{\"content\":\"c\"}'),1 as docs,1 as query)";s:10:"total_rows";i:0;}i:29;a:3:{s:8:"sphinxql";s:58:"CALL PQ('pq1', ('{\"title\":\"c\"}'),1 as docs,1 as query)";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:5:{s:2:"id";s:1:"1";s:9:"documents";s:1:"1";s:5:"query";s:8:"@title c";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}}}}}
@@ -20,6 +20,19 @@ index pq
rt_attr_json = json_data
}

index pq1
{
type = percolate
path = <data_path/>/pq_1
rt_field = title
rt_field = content
rt_attr_json = json_data

html_strip = 1
html_remove_elements = style, script
charset_table = english
}

</config>

<sphqueries>
@@ -57,6 +70,12 @@ index pq
<sphinxql>INSERT INTO pq (query, filters) VALUES ('test', 'json_data.text.status.state=0')</sphinxql>
<sphinxql>CALL PQ ('pq', ('{"id":21,"json_data":{"text":{"status":{"state":0}}}, "title":"test me"}'), 1 as docs, 1 as query, 'id' as docs_id)</sphinxql>

<!-- regression - stripper alter field -->
<sphinxql>INSERT INTO pq1 (query) VALUES ('@title c')</sphinxql>
<sphinxql>CALL PQ('pq1', ('{\"title\":\"a\",\"content\":\"c\"}'),1 as docs,1 as query)</sphinxql>
<sphinxql>CALL PQ('pq1', ('{\"content\":\"c\"}'),1 as docs,1 as query)</sphinxql>
<sphinxql>CALL PQ('pq1', ('{\"title\":\"c\"}'),1 as docs,1 as query)</sphinxql>

</sphqueries>

</test>

0 comments on commit 75cd134

Please sign in to comment.
You can’t perform that action at this time.