Permalink
Browse files

fixed effyis#13; added document id field to JSON document CALL PQ sta…

…tement; added regressions to test 321
  • Loading branch information...
tomatolog committed May 16, 2018
1 parent 91da8df commit 9ebc58916cd515eaa88da66d0895aebf0d1f2b5f
Showing with 76 additions and 22 deletions.
  1. +34 −10 src/searchd.cpp
  2. +32 −9 src/searchdhttp.cpp
  3. +2 −1 src/sphinxrt.cpp
  4. +1 −1 src/sphinxrt.h
  5. +1 −1 test/test_321/model.bin
  6. +6 −0 test/test_321/test.xml
@@ -12816,7 +12816,7 @@ static void PercolateQuery ( const SqlStmt_t & tStmt, bool bReplace, ESphCollati
tOut.Ok ( 1, iWarnings );
}
static void SendPercolateReply ( const PercolateMatchResult_t & tRes, const CSphString & sWarning, const CSphString & sError, SqlRowBuffer_c & tOut )
static void SendPercolateReply ( const PercolateMatchResult_t & tRes, const CSphString & sWarning, const CSphString & sError, const CSphFixedVector<SphDocID_t> & dDocids, SqlRowBuffer_c & tOut )
{
if ( !sError.IsEmpty() )
{
@@ -12860,7 +12860,9 @@ static void SendPercolateReply ( const PercolateMatchResult_t & tRes, const CSph
const char * sSep = "";
for ( int iDoc = 0; iDoc<iCount; iDoc++ )
{
sDocs.Appendf ( "%s" DOCID_FMT, sSep, tRes.m_dDocs[iDocOff + 1 + iDoc] );
int iRow = tRes.m_dDocs[iDocOff + 1 + iDoc];
SphDocID_t uDocid = ( dDocids.GetLength() ? dDocids[iRow] : iRow );
sDocs.Appendf ( "%s" DOCID_FMT, sSep, uDocid );
sSep = ",";
}
iDocOff += iCount + 1;
@@ -13049,6 +13051,9 @@ static bool ParseJsonDocument ( const char * sDoc, const CSphHash<SchemaItemVari
tAttr.m_sVal = pChild->valuestring;
tDoc.SetAttr ( pItem->m_tLoc, tAttr, pItem->m_eType );
}
} else if ( sName && strlen ( sName )==2 && strncmp ( sName, "id", 2 )==0 )
{
tDoc.m_uDocID = pChild->valueint;
}
pChild = pChild->next;
@@ -13109,14 +13114,13 @@ static void PercolateMatchDocuments ( const CSphVector<CSphString> & dDocs, cons
}
CSphString sTokenFilterOpts;
SphDocID_t uDocID = 1;
ARRAY_FOREACH ( iRow, dDocs )
CSphFixedVector<SphDocID_t> dDocids ( 0 );
SphDocID_t uSeqDocid = 1;
ARRAY_FOREACH ( iDoc, dDocs )
{
// doc-id
tDoc.m_uDocID = uDocID;
uDocID++;
dFields[0] = dDocs[iRow].cstr();
tDoc.m_uDocID = 0;
dFields[0] = dDocs[iDoc].cstr();
cJSON * pJsonStorage = NULL;
if ( tOpts.m_bJsonDocs )
@@ -13130,7 +13134,7 @@ static void PercolateMatchDocuments ( const CSphVector<CSphString> & dDocs, cons
tLoc.m_bDynamic = true;
tDoc.SetDefaultAttr ( tLoc, tCol.m_eAttrType );
}
if ( !ParseJsonDocument ( dDocs[iRow].cstr(), hSchemaLocators, iRow, dFields, tDoc, &pJsonStorage, sError ) )
if ( !ParseJsonDocument ( dDocs[iDoc].cstr(), hSchemaLocators, iDoc, dFields, tDoc, &pJsonStorage, sError ) )
break;
}
@@ -13141,6 +13145,26 @@ static void PercolateMatchDocuments ( const CSphVector<CSphString> & dDocs, cons
break;
}
// assign proper docids
bool bGotDocid = ( tDoc.m_uDocID!=0 );
if ( bGotDocid && !dDocids.GetLength() )
{
dDocids.Reset ( dDocs.GetLength()+1 );
dDocids[0] = 0; // 0 element unused
for ( int iInit=0; iInit<=iDoc; iInit++ )
dDocids[iInit] = iInit;
}
if ( bGotDocid )
{
dDocids[iDoc+1] = tDoc.m_uDocID;
uSeqDocid = Max ( uSeqDocid, tDoc.m_uDocID );
} else if ( dDocids.GetLength() )
{
dDocids[iDoc+1] = uSeqDocid;
}
tDoc.m_uDocID = iDoc+1; // PQ work with sequential document numbers, 0 element unused
uSeqDocid++;
// add document
pIndex->AddDocument ( pIndex->CloneIndexingTokenizer (), iFieldsCount, dFields.Begin(), tDoc, true, sTokenFilterOpts, NULL, CSphVector<DWORD>(), sError, sWarning, pAccum );
@@ -13166,7 +13190,7 @@ static void PercolateMatchDocuments ( const CSphVector<CSphString> & dDocs, cons
pIndex->MatchDocuments ( pAccum, tRes );
SendPercolateReply ( tRes, sWarning, sError, tOut );
SendPercolateReply ( tRes, sWarning, sError, dDocids, tOut );
tMeta.Swap ( tRes );
}
@@ -1460,7 +1460,7 @@ struct SourceMatch_c : public CSphMatch
}
};
static void EncodePercolateMatchResult ( const PercolateMatchResult_t & tRes, const CSphString & sIndex, JsonEscapedBuilder & tOut )
static void EncodePercolateMatchResult ( const PercolateMatchResult_t & tRes, const CSphFixedVector<SphDocID_t> & dDocids, const CSphString & sIndex, JsonEscapedBuilder & tOut )
{
tOut += "{";
@@ -1525,7 +1525,9 @@ static void EncodePercolateMatchResult ( const PercolateMatchResult_t & tRes, co
const char * sSep = "";
for ( int iDoc = 0; iDoc<iCount; iDoc++ )
{
tOut.Appendf ( "%s" DOCID_FMT, sSep, tRes.m_dDocs[iDocOff + 1 + iDoc] );
int iRow = tRes.m_dDocs[iDocOff + 1 + iDoc];
SphDocID_t uDocid = ( dDocids.GetLength() ? dDocids[iRow] : iRow );
tOut.Appendf ( "%s" DOCID_FMT, sSep, uDocid );
sSep = ",";
}
tOut += "] }";
@@ -1617,10 +1619,13 @@ bool HttpHandlerPQ_c::GotDocuments ( PercolateIndex_i * pIndex, const CSphString
ISphRtAccum * pAccum = tAcc.GetAcc ( pIndex, sError );
CSphString sTokenFilterOpts;
SphDocID_t uDocID = 1;
CSphFixedVector<SphDocID_t> dDocids ( 0 );
SphDocID_t uSeqDocid = 1;
int iDoc = 0;
for ( const cJSON * pDoc : dDocs )
{
// reset all back to defaults
tDoc.m_uDocID = 0;
dFields.Fill ( sTmp.scstr() );
for ( int i=0; i<iAttrsCount; i++ )
{
@@ -1639,7 +1644,7 @@ bool HttpHandlerPQ_c::GotDocuments ( PercolateIndex_i * pIndex, const CSphString
if ( !pItem )
{
if ( sName && ( strncmp ( sName, "id", 2 )==0 || strncmp ( sName, "uid", 3 )==0 ) )
uDocID = pChild->valueint;
tDoc.m_uDocID = pChild->valueint;
continue;
}
@@ -1652,8 +1657,26 @@ bool HttpHandlerPQ_c::GotDocuments ( PercolateIndex_i * pIndex, const CSphString
}
}
tDoc.m_uDocID = uDocID;
uDocID++;
// assign proper docids
bool bGotDocid = ( tDoc.m_uDocID!=0 );
if ( bGotDocid && !dDocids.GetLength() )
{
dDocids.Reset ( dDocs.GetLength()+1 );
dDocids[0] = 0; // 0 element unused
for ( int iInit=0; iInit<=iDoc; iInit++ )
dDocids[iInit] = iInit;
}
if ( bGotDocid )
{
dDocids[iDoc+1] = tDoc.m_uDocID;
uSeqDocid = Max ( uSeqDocid, tDoc.m_uDocID );
} else if ( dDocids.GetLength() )
{
dDocids[iDoc+1] = uSeqDocid;
}
tDoc.m_uDocID = iDoc+1; // PQ work with sequential document numbers, 0 element unused
uSeqDocid++;
iDoc++;
// add document
pIndex->AddDocument ( pIndex->CloneIndexingTokenizer (), iFieldsCount, dFields.Begin(), tDoc, true, sTokenFilterOpts, NULL, CSphVector<DWORD>(), sError, sWarning, pAccum );
@@ -1678,7 +1701,7 @@ bool HttpHandlerPQ_c::GotDocuments ( PercolateIndex_i * pIndex, const CSphString
bool bRes = pIndex->MatchDocuments ( pAccum, tRes );
JsonEscapedBuilder sRes;
EncodePercolateMatchResult ( tRes, sIndex, sRes );
EncodePercolateMatchResult ( tRes, dDocids, sIndex, sRes );
BuildReply ( sRes, SPH_HTTP_STATUS_200 );
return bRes;
@@ -1821,9 +1844,9 @@ bool HttpHandlerPQ_c::ListQueries ( PercolateIndex_i * pIndex, const CSphString
tRes.m_tmTotal = sphMicroTimer() - tmStart;
CSphFixedVector<SphDocID_t> dTmpids ( 0 );
JsonEscapedBuilder sRes;
EncodePercolateMatchResult ( tRes, sIndex, sRes );
EncodePercolateMatchResult ( tRes, dTmpids, sIndex, sRes );
BuildReply ( sRes, SPH_HTTP_STATUS_200 );
return true;
@@ -10845,6 +10845,7 @@ static void SegmentGetRejects ( const RtSegment_t * pSeg, SegmentReject_t & tRej
if ( !pDoc )
break;
// Docid - should be row-based started from 1
assert ( pDoc->m_uDocID>=1 && (int)pDoc->m_uDocID<pSeg->m_iRows+1 );
int iDoc = (int)pDoc->m_uDocID - 1;
tReject.m_dPerDocTerms[iDoc].Add ( uHash );
@@ -11644,7 +11645,7 @@ class PercolateDictProxy_c : public CSphDict
struct PercolateMatchContext_t
{
CSphVector<PercolateQueryDesc> m_dQueryMatched;
CSphVector<SphDocID_t> m_dDocsMatched;
CSphVector<int> m_dDocsMatched;
CSphVector<int> m_dDt;
int m_iQueriesMatched = 0;
int m_iDocsMatched = 0;
@@ -127,7 +127,7 @@ struct PercolateMatchResult_t
bool m_bGetFilters;
CSphFixedVector<PercolateQueryDesc> m_dQueryDesc;
CSphFixedVector<SphDocID_t> m_dDocs;
CSphFixedVector<int> m_dDocs;
int m_iQueriesMatched;
int m_iDocsMatched;
int64_t m_tmTotal;
Oops, something went wrong.

0 comments on commit 9ebc589

Please sign in to comment.