Skip to content
Permalink
Browse files

fixed #425 invalid filter at PQ matches query; added warning of PQ ma…

…tching errors and counter to meta; added regressions to test 321
  • Loading branch information
tomatolog committed Sep 6, 2018
1 parent ca09490 commit 552646bb6fefde4e2525298e43e628839b421d3d
Showing with 70 additions and 8 deletions.
  1. +44 −2 src/searchd.cpp
  2. +18 −5 src/sphinxrt.cpp
  3. +1 −0 src/sphinxrt.h
  4. +1 −1 test/test_321/model.bin
  5. +6 −0 test/test_321/test.xml
@@ -12328,6 +12328,45 @@ bool PercolateParseFilters ( const char * sFilters, ESphCollation eCollation, co
dFilterTree.SwapData ( tQuery.m_dFilterTree );
}

// maybe its better to create real filter instead of just checking column name
if ( iRes==0 && dFilters.GetLength() )
{
ARRAY_FOREACH ( i, dFilters )
{
const CSphFilterSettings & tFilter = dFilters[i];
if ( tFilter.m_sAttrName.IsEmpty() )
{
sError.SetSprintf ( "bad filter %d name", i );
return false;
}

if ( tFilter.m_sAttrName.Begins ( "@" ) )
{
sError.SetSprintf ( "unsupported filter column '%s'", tFilter.m_sAttrName.cstr() );
return false;
}

const char * sAttrName = tFilter.m_sAttrName.cstr();

// might be a JSON.field
CSphString sJsonField;
const char * sJsonDot = strchr ( sAttrName, '.' );
if ( sJsonDot )
{
assert ( sJsonDot>sAttrName );
sJsonField.SetBinary ( sAttrName, sJsonDot - sAttrName );
sAttrName = sJsonField.cstr();
}

int iCol = tSchema.GetAttrIndex ( sAttrName );
if ( iCol==-1 )
{
sError.SetSprintf ( "no such filter attribute '%s'", sAttrName );
return false;
}
}
}

// TODO: change way of filter -> expression create: produce single error, share parser code
// try expression
if ( iRes!=0 && !dFilters.GetLength() && sError.Begins ( "percolate filters: syntax error" ) )
@@ -12343,7 +12382,7 @@ bool PercolateParseFilters ( const char * sFilters, ESphCollation eCollation, co
tExpr.m_sAttrName = sFilters;
} else
{
iRes = 1;
return false;
}
}

@@ -13074,6 +13113,8 @@ static void PercolateMatchDocuments ( const StrVec_t & dDocs, const PercolateOpt

if ( iDocsNoIdCount )
sWarning.SetSprintf ( "skipped %d documents without id field '%s'", iDocsNoIdCount, sIdAlias.cstr() );
if ( tRes.m_iQueriesFailed )
sWarning.SetSprintf ( "%d queries failed", tRes.m_iQueriesFailed );

SendPercolateReply ( tRes, sWarning, sError, dDocids, tOut );
tMeta.Swap ( tRes );
@@ -13186,6 +13227,7 @@ void HandleMysqlPercolateMeta ( const PercolateMatchResult_t & tMeta, const CSph
tOut.DataTuplet ( "Setup", sTmp.cstr() );
}
tOut.DataTuplet ( "Queries matched", tMeta.m_iQueriesMatched );
tOut.DataTuplet ( "Queries failed", tMeta.m_iQueriesFailed );
tOut.DataTuplet ( "Document matches", tMeta.m_iDocsMatched );
tOut.DataTuplet ( "Total queries stored", tMeta.m_iTotalQueries );
tOut.DataTuplet ( "Term only queries", tMeta.m_iOnlyTerms );
@@ -13196,7 +13238,7 @@ void HandleMysqlPercolateMeta ( const PercolateMatchResult_t & tMeta, const CSph
uint64_t tmMatched = 0;
const char * sDelimiter = "";
sTmp.Clear();
assert ( tMeta.m_dQueryDesc.GetLength()==tMeta.m_dQueryDT.GetLength() );
assert ( tMeta.m_iQueriesMatched==tMeta.m_dQueryDT.GetLength() );
ARRAY_FOREACH ( i, tMeta.m_dQueryDT )
{
int tmQuery = tMeta.m_dQueryDT[i];
@@ -11500,9 +11500,9 @@ struct PercolateMatchContext_t
bool m_bGetQuery = false;
bool m_bGetFilters = false;
bool m_bVerbose = false;
int m_iQueriesFailed = 0;

StringBuilder_c m_tFilterBuf;
CSphString m_sWarning;
KillListVector m_dKillist;

PercolateDictProxy_c m_tDictMap;
@@ -11557,7 +11557,15 @@ static void MatchingWork ( const StoredQuery_t * pStored, PercolateMatchContext_

tMatchCtx.m_iEarlyPassed++;
tMatchCtx.m_pCtx->ResetFilters();
tMatchCtx.m_pCtx->CreateFilters ( false, &pStored->m_dFilters, tMatchCtx.m_tSchema, pMva, pStrings, tMatchCtx.m_sWarning, tMatchCtx.m_sWarning, SPH_COLLATION_DEFAULT, true, tMatchCtx.m_dKillist, &pStored->m_dFilterTree );

// FIXME!!! collect and show all errors and warnings somehow
CSphString sError;
CSphString sWarning;
if ( !tMatchCtx.m_pCtx->CreateFilters ( false, &pStored->m_dFilters, tMatchCtx.m_tSchema, pMva, pStrings, sError, sWarning, SPH_COLLATION_DEFAULT, true, tMatchCtx.m_dKillist, &pStored->m_dFilterTree ) )
{
tMatchCtx.m_iQueriesFailed++;
return;
}


const bool bCollectDocs = tMatchCtx.m_bGetDocs;
@@ -11636,13 +11644,14 @@ static void MatchingWork ( const StoredQuery_t * pStored, PercolateMatchContext_
tDesc.m_sFilters = tMatchCtx.m_tFilterBuf.cstr();
}
}

if ( tMatchCtx.m_bVerbose )
tMatchCtx.m_dDt.Add ( (int)( sphMicroTimer() - tmQueryStart ) );

} else if ( bCollectDocs ) // pop's up reserved but not used matched counter
{
tMatchCtx.m_dDocsMatched.Resize ( iDocsOff );
}

if ( tMatchCtx.m_bVerbose )
tMatchCtx.m_dDt.Add ( (int)( sphMicroTimer() - tmQueryStart ) );
}


@@ -11717,6 +11726,7 @@ static void PercolateGetResult ( int iTotalQueries, CSphFixedVector<PercolateMat
tRes.m_iTotalQueries = iTotalQueries;
tRes.m_iEarlyOutQueries = ( iTotalQueries - pMatch->m_iEarlyPassed );
tRes.m_iOnlyTerms = pMatch->m_iOnlyTerms;
tRes.m_iQueriesFailed = pMatch->m_iQueriesFailed;
if ( tRes.m_bVerbose )
tRes.m_dQueryDT.CopyFrom ( pMatch->m_dDt );

@@ -11753,6 +11763,7 @@ static void PercolateGetResult ( int iTotalQueries, CSphFixedVector<PercolateMat
tRes.m_iDocsMatched += pMatch->m_iDocsMatched;
tRes.m_iEarlyOutQueries += pMatch->m_iEarlyPassed;
tRes.m_iOnlyTerms += pMatch->m_iOnlyTerms;
tRes.m_iQueriesFailed += pMatch->m_iQueriesFailed;
}
tRes.m_iTotalQueries = iTotalQueries;
tRes.m_iEarlyOutQueries = ( iTotalQueries - tRes.m_iEarlyOutQueries );
@@ -11866,6 +11877,7 @@ bool PercolateIndex_c::MatchDocuments ( ISphRtAccum * pAccExt, PercolateMatchRes
MEMORY ( MEM_INDEX_RT );

int64_t tmStart = sphMicroTimer();
tRes.m_tmSetup = tmStart;
m_sLastWarning = "";

RtAccum_t * pAcc = AcquireAccum ( NULL, pAccExt, true, this, m_pDict, true );
@@ -12559,6 +12571,7 @@ void PercolateMatchResult_t::Swap ( PercolateMatchResult_t & tOther )
m_iQueriesMatched = tOther.m_iQueriesMatched;
m_iDocsMatched = tOther.m_iDocsMatched;
m_tmTotal = tOther.m_tmTotal;
m_iQueriesFailed = tOther.m_iQueriesFailed;

m_bVerbose = tOther.m_bVerbose;
m_dQueryDT.SwapData ( tOther.m_dQueryDT );
@@ -128,6 +128,7 @@ struct PercolateMatchResult_t
CSphFixedVector<PercolateQueryDesc> m_dQueryDesc;
CSphFixedVector<int> m_dDocs;
int m_iQueriesMatched;
int m_iQueriesFailed = 0;
int m_iDocsMatched;
int64_t m_tmTotal;

Large diffs are not rendered by default.

@@ -214,6 +214,12 @@ CREATE TABLE `test_table` (
<sphinxql>insert into pq_mva (query, tags) values('', 'q_all')</sphinxql>
<sphinxql>CALL PQ ('pq_mva', ('{"title":"black", "mva3":[1, 12]}', '{"mva2":[15], "mva3":[8,8,1]}', '{"mva3":[13]}', '{"mva2":[15], "mva3":[8,13,1]}'), 1 as docs, 1 as docs_json, 1 as query )</sphinxql>

<!-- bad filters -->
<sphinxql>insert into pq_filter1 (query,filters) values('orange', 'test > 3')</sphinxql>
<sphinxql>insert into pq_filter1 (query,filters) values('orange', 'test.price > 3')</sphinxql>
<sphinxql>insert into pq_filter1 (query,filters) values('orange', '@weight > 3')</sphinxql>
<sphinxql>select * from pq_filter1</sphinxql>

</sphqueries>

</test>

0 comments on commit 552646b

Please sign in to comment.
You can’t perform that action at this time.