Skip to content

/ manticoresearch

Permalink
Browse files

fixed #8 broken CALL KEYWORDS to distributed index via API or to remo… 

…te agent; added regressions to test 365
  • Loading branch information
@tomatolog
tomatolog committed Oct 12, 2018
1 parent f974f20 commit 092083282ea8ae0b2e72fb6989c57ccec81e74ac
Unified Split
Showing with 116 additions and 85 deletions.
  1. +92 −81 src/searchd.cpp
  2. +1 −1 test/test_365/model.bin
  3. +23 −3 test/test_365/test.xml
173 src/searchd.cpp
View file
@@ -10525,6 +10525,8 @@ void HandleCommandExcerpt ( ISphOutputBuffer & tOut, int iVer, InputBuffer_c & t
// KEYWORDS HANDLER
/////////////////////////////////////////////////////////////////////////////

static bool DoGetKeywords ( const CSphString & sIndex, const CSphString & sQuery, const GetKeywordsSettings_t & tSettings, CSphVector <CSphKeywordInfo> & dKeywords, CSphString & sError, SearchFailuresLog_c & tFailureLog );

void HandleCommandKeywords ( ISphOutputBuffer & tOut, WORD uVer, InputBuffer_c & tReq )
{
if ( !CheckCommandVersion ( uVer, VER_COMMAND_KEYWORDS, tOut ) )
@@ -10541,22 +10543,22 @@ void HandleCommandKeywords ( ISphOutputBuffer & tOut, WORD uVer, InputBuffer_c &
tSettings.m_bFoldWildcards = !!tReq.GetInt ();
tSettings.m_iExpansionLimit = tReq.GetInt ();
}
auto pServed = GetServed ( sIndex );
if ( !pServed )
{
SendErrorReply ( tOut, "unknown local index '%s' in search request", sIndex.cstr() );
return;
}

CSphString sError;
SearchFailuresLog_c tFailureLog;
CSphVector < CSphKeywordInfo > dKeywords;
bool bOk = DoGetKeywords ( sIndex, sQuery, tSettings, dKeywords, sError, tFailureLog );
if ( !bOk )
{
ServedDescRPtr_c pIndex ( pServed );
if ( !pIndex->m_pIndex->GetKeywords ( dKeywords, sQuery.cstr (), tSettings, &sError ) )
{
SendErrorReply ( tOut, "error generating keywords: %s", sError.cstr () );
return;
}
SendErrorReply ( tOut, "%s", sError.cstr() );
return;
}
// just log distribute index error as command has no warning filed to pass such error into
if ( !tFailureLog.IsEmpty() )
{
StringBuilder_c sErrorBuf;
tFailureLog.BuildReport ( sErrorBuf );
sphWarning ( "%s", sErrorBuf.cstr() );
}

int iRespLen = 4;
@@ -13839,6 +13841,81 @@ struct KeywordsReplyParser_t : public IReplyParser_t

static void MergeKeywords ( CSphVector<CSphKeywordInfo> & dKeywords );
static void SortKeywords ( const GetKeywordsSettings_t & tSettings, CSphVector<CSphKeywordInfo> & dKeywords );
bool DoGetKeywords ( const CSphString & sIndex, const CSphString & sQuery, const GetKeywordsSettings_t & tSettings, CSphVector <CSphKeywordInfo> & dKeywords, CSphString & sError, SearchFailuresLog_c & tFailureLog )
{
auto pLocal = GetServed ( sIndex );
auto pDistributed = GetDistr ( sIndex );

if ( !pLocal && !pDistributed )
{
sError.SetSprintf ( "no such index %s", sIndex.cstr() );
return false;
}

bool bOk = false;
// just local plain or template index
if ( pLocal )
{
ServedDescRPtr_c pIndex ( pLocal );
if ( !pIndex->m_pIndex )
{
sError.SetSprintf ( "missed index '%s'", sIndex.cstr() );
} else
{
bOk = pIndex->m_pIndex->GetKeywords ( dKeywords, sQuery.cstr(), tSettings, &sError );
}
} else
{
// FIXME!!! g_iDistThreads thread pool for locals.
// locals
const StrVec_t & dLocals = pDistributed->m_dLocal;
CSphVector<CSphKeywordInfo> dKeywordsLocal;
for ( const CSphString &sLocal : dLocals )
{
ServedDescRPtr_c pServed ( GetServed ( sLocal ) );
if ( !pServed || !pServed->m_pIndex )
{
tFailureLog.Submit ( sLocal.cstr(), sIndex.cstr(), "missed index" );
continue;
}

dKeywordsLocal.Resize(0);
if ( pServed->m_pIndex->GetKeywords ( dKeywordsLocal, sQuery.cstr(), tSettings, &sError ) )
dKeywords.Append ( dKeywordsLocal );
else
tFailureLog.SubmitEx ( sLocal.cstr (), sIndex.cstr (), "keyword extraction failed: %s", sError.cstr () );
}

// remote agents requests send off thread
VecRefPtrsAgentConn_t dAgents;
// fixme! We don't need all hosts here, only usual selected mirrors
pDistributed->GetAllHosts ( dAgents );

int iAgentsReply = 0;
if ( !dAgents.IsEmpty() )
{
// connect to remote agents and query them
KeywordsRequestBuilder_t tReqBuilder ( tSettings, sQuery );
KeywordsReplyParser_t tParser ( tSettings.m_bStats, dKeywords );
iAgentsReply = PerformRemoteTasks ( dAgents, &tReqBuilder, &tParser );

for ( const AgentConn_t * pAgent : dAgents )
if ( !pAgent->m_bSuccess && !pAgent->m_sFailure.IsEmpty() )
tFailureLog.SubmitEx ( pAgent->m_tDesc.m_sIndexes.cstr(), sIndex.cstr(),
"agent %s: %s", pAgent->m_tDesc.GetMyUrl().cstr(), pAgent->m_sFailure.cstr() );
}

// process result sets
if ( dLocals.GetLength() + iAgentsReply>1 )
MergeKeywords ( dKeywords );

bOk = true;
}

SortKeywords ( tSettings, dKeywords );

return bOk;
}

void HandleMysqlCallKeywords ( SqlRowBuffer_c & tOut, SqlStmt_t & tStmt, CSphString & sWarning )
{
@@ -13905,81 +13982,15 @@ void HandleMysqlCallKeywords ( SqlRowBuffer_c & tOut, SqlStmt_t & tStmt, CSphStr
}
const CSphString & sTerm = tStmt.m_dInsertValues[0].m_sVal;
const CSphString & sIndex = tStmt.m_dInsertValues[1].m_sVal;
CSphVector<CSphKeywordInfo> dKeywords;
SearchFailuresLog_c tFailureLog;

auto pLocal = GetServed ( sIndex );
auto pDistributed = GetDistr ( sIndex );

if ( !pLocal && !pDistributed )
if ( !DoGetKeywords ( sIndex, sTerm, tSettings, dKeywords, sError, tFailureLog ) )
{
sError.SetSprintf ( "no such index %s", sIndex.cstr() );
tOut.Error ( tStmt.m_sStmt, sError.cstr() );
return;
}

SearchFailuresLog_c tFailureLog;
CSphVector<CSphKeywordInfo> dKeywords;

// just local plain or template index
if ( pLocal )
{
ServedDescRPtr_c pIndex ( pLocal );
if ( !pIndex->m_pIndex )
{
tFailureLog.Submit ( sIndex.cstr(), NULL, "missed index" );
} else
{
bool bRes = pIndex->m_pIndex->GetKeywords ( dKeywords, sTerm.cstr(), tSettings, &sError );

if ( !bRes )
tFailureLog.SubmitEx ( sIndex.cstr(), NULL, "keyword extraction failed: %s", sError.cstr() );
}
} else
{
// FIXME!!! g_iDistThreads thread pool for locals.
// locals
const StrVec_t & dLocals = pDistributed->m_dLocal;
CSphVector<CSphKeywordInfo> dKeywordsLocal;
for ( const CSphString &sLocal : dLocals )
{
ServedDescRPtr_c pServed ( GetServed ( sLocal ) );
if ( !pServed || !pServed->m_pIndex )
{
tFailureLog.Submit ( sLocal.cstr(), sIndex.cstr(), "missed index" );
continue;
}

dKeywordsLocal.Resize(0);
if ( pServed->m_pIndex->GetKeywords ( dKeywordsLocal, sTerm.cstr(), tSettings, &sError ) )
dKeywords.Append ( dKeywordsLocal );
else
tFailureLog.SubmitEx ( sLocal.cstr (), sIndex.cstr (), "keyword extraction failed: %s", sError.cstr () );
}

// remote agents requests send off thread
VecRefPtrsAgentConn_t dAgents;
// fixme! We don't need all hosts here, only usual selected mirrors
pDistributed->GetAllHosts ( dAgents );

int iAgentsReply = 0;
if ( !dAgents.IsEmpty() )
{
// connect to remote agents and query them
KeywordsRequestBuilder_t tReqBuilder ( tSettings, sTerm );
KeywordsReplyParser_t tParser ( tSettings.m_bStats, dKeywords );
iAgentsReply = PerformRemoteTasks ( dAgents, &tReqBuilder, &tParser );

for ( const AgentConn_t * pAgent : dAgents )
if ( !pAgent->m_bSuccess && !pAgent->m_sFailure.IsEmpty() )
tFailureLog.SubmitEx ( pAgent->m_tDesc.m_sIndexes.cstr(), sIndex.cstr(),
"agent %s: %s", pAgent->m_tDesc.GetMyUrl().cstr(), pAgent->m_sFailure.cstr() );
}

// process result sets
if ( dLocals.GetLength() + iAgentsReply>1 )
MergeKeywords ( dKeywords );
}

SortKeywords ( tSettings, dKeywords );

// result set header packet
tOut.HeadBegin ( tSettings.m_bStats ? 5 : 3 );
2 test/test_365/model.bin
View file
@@ -1 +1 @@
a:1:{i:0;a:6:{i:0;a:2:{s:8:"sphinxql";s:118:"INSERT INTO rt_bi (id, gid, title) VALUES ( 1, 1, 'manticore search is the best'), (2, 2, 'manticore is more better' )";s:14:"total_affected";i:2;}i:1;a:3:{s:8:"sphinxql";s:69:"CALL KEYWORDS ( 'manti*', 'idx_bi', 1 AS stats, 0 AS fold_wildcards )";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:9:"manticore";s:4:"docs";s:1:"3";s:4:"hits";s:1:"3";}i:1;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:12:"manticore is";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:2;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:14:"manticore like";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:3;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:16:"manticore search";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}}}i:2;a:3:{s:8:"sphinxql";s:68:"CALL KEYWORDS ( 'manti*', 'rt_bi', 1 AS stats, 0 AS fold_wildcards )";s:10:"total_rows";i:3;s:4:"rows";a:3:{i:0;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:16:"manticore search";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:1;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:9:"manticore";s:4:"docs";s:1:"2";s:4:"hits";s:1:"2";}i:2;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:12:"manticore is";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}}}i:3;a:2:{s:8:"sphinxql";s:20:"FLUSH RAMCHUNK rt_bi";s:14:"total_affected";i:0;}i:4;a:2:{s:8:"sphinxql";s:84:"INSERT INTO rt_bi (id, gid, title) VALUES (3, 3, 'search with manticore like many' )";s:14:"total_affected";i:1;}i:5;a:3:{s:8:"sphinxql";s:68:"CALL KEYWORDS ( 'manti*', 'rt_bi', 1 AS stats, 0 AS fold_wildcards )";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:14:"manticore like";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:1;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:16:"manticore search";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:2;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:12:"manticore is";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:3;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:9:"manticore";s:4:"docs";s:1:"3";s:4:"hits";s:1:"3";}}}}}
a:1:{i:0;a:8:{i:0;a:2:{s:8:"sphinxql";s:118:"INSERT INTO rt_bi (id, gid, title) VALUES ( 1, 1, 'manticore search is the best'), (2, 2, 'manticore is more better' )";s:14:"total_affected";i:2;}i:1;a:3:{s:8:"sphinxql";s:90:"CALL KEYWORDS ( 'manti*', 'idx_bi', 1 AS stats, 0 AS fold_wildcards, 'hits' as sort_mode )";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:9:"manticore";s:4:"docs";s:1:"3";s:4:"hits";s:1:"3";}i:1;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:12:"manticore is";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:2;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:14:"manticore like";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:3;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:16:"manticore search";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}}}i:2;a:3:{s:8:"sphinxql";s:89:"CALL KEYWORDS ( 'manti*', 'rt_bi', 1 AS stats, 0 AS fold_wildcards, 'hits' as sort_mode )";s:10:"total_rows";i:3;s:4:"rows";a:3:{i:0;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:9:"manticore";s:4:"docs";s:1:"2";s:4:"hits";s:1:"2";}i:1;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:12:"manticore is";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:2;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:16:"manticore search";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}}}i:3;a:2:{s:8:"sphinxql";s:20:"FLUSH RAMCHUNK rt_bi";s:14:"total_affected";i:0;}i:4;a:2:{s:8:"sphinxql";s:84:"INSERT INTO rt_bi (id, gid, title) VALUES (3, 3, 'search with manticore like many' )";s:14:"total_affected";i:1;}i:5;a:3:{s:8:"sphinxql";s:89:"CALL KEYWORDS ( 'manti*', 'rt_bi', 1 AS stats, 0 AS fold_wildcards, 'hits' as sort_mode )";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:9:"manticore";s:4:"docs";s:1:"3";s:4:"hits";s:1:"3";}i:1;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:12:"manticore is";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:2;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:14:"manticore like";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}i:3;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:16:"manticore search";s:4:"docs";s:1:"1";s:4:"hits";s:1:"1";}}}i:6;a:3:{s:8:"sphinxql";s:94:"CALL KEYWORDS ( 'manti*', 'dist_local', 1 AS stats, 0 AS fold_wildcards, 'hits' as sort_mode )";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:9:"manticore";s:4:"docs";s:1:"6";s:4:"hits";s:1:"6";}i:1;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:12:"manticore is";s:4:"docs";s:1:"2";s:4:"hits";s:1:"2";}i:2;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:14:"manticore like";s:4:"docs";s:1:"2";s:4:"hits";s:1:"2";}i:3;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:16:"manticore search";s:4:"docs";s:1:"2";s:4:"hits";s:1:"2";}}}i:7;a:3:{s:8:"sphinxql";s:95:"CALL KEYWORDS ( 'manti*', 'dist_remote', 1 AS stats, 0 AS fold_wildcards, 'hits' as sort_mode )";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:9:"manticore";s:4:"docs";s:1:"6";s:4:"hits";s:1:"6";}i:1;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:12:"manticore is";s:4:"docs";s:1:"2";s:4:"hits";s:1:"2";}i:2;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:14:"manticore like";s:4:"docs";s:1:"2";s:4:"hits";s:1:"2";}i:3;a:5:{s:4:"qpos";s:1:"1";s:9:"tokenized";s:6:"manti*";s:10:"normalized";s:16:"manticore search";s:4:"docs";s:1:"2";s:4:"hits";s:1:"2";}}}}}
26 test/test_365/test.xml
View file
@@ -46,6 +46,22 @@ index rt_bi
dict = keywords
min_prefix_len = 2
}

index dist_local
{
type = distributed
local = idx_bi
local = rt_bi
}

index dist_remote
{
type = distributed
agent = <my_address/>:dist_local
agent_connect_timeout = 1000
agent_query_timeout = 3000
}

</config>
<db_create>
CREATE TABLE `test_table` (
@@ -61,12 +77,16 @@ CREATE TABLE `test_table` (
<sphqueries>
<sphinxql>INSERT INTO rt_bi (id, gid, title) VALUES ( 1, 1, 'manticore search is the best'), (2, 2, 'manticore is more better' )</sphinxql>

<sphinxql>CALL KEYWORDS ( 'manti*', 'idx_bi', 1 AS stats, 0 AS fold_wildcards )</sphinxql>
<sphinxql>CALL KEYWORDS ( 'manti*', 'rt_bi', 1 AS stats, 0 AS fold_wildcards )</sphinxql>
<sphinxql>CALL KEYWORDS ( 'manti*', 'idx_bi', 1 AS stats, 0 AS fold_wildcards, 'hits' as sort_mode )</sphinxql>
<sphinxql>CALL KEYWORDS ( 'manti*', 'rt_bi', 1 AS stats, 0 AS fold_wildcards, 'hits' as sort_mode )</sphinxql>

<sphinxql>FLUSH RAMCHUNK rt_bi</sphinxql>
<sphinxql>INSERT INTO rt_bi (id, gid, title) VALUES (3, 3, 'search with manticore like many' )</sphinxql>
<sphinxql>CALL KEYWORDS ( 'manti*', 'rt_bi', 1 AS stats, 0 AS fold_wildcards )</sphinxql>
<sphinxql>CALL KEYWORDS ( 'manti*', 'rt_bi', 1 AS stats, 0 AS fold_wildcards, 'hits' as sort_mode )</sphinxql>

<!-- regression keywords to distributed index via API -->
<sphinxql>CALL KEYWORDS ( 'manti*', 'dist_local', 1 AS stats, 0 AS fold_wildcards, 'hits' as sort_mode )</sphinxql>
<sphinxql>CALL KEYWORDS ( 'manti*', 'dist_remote', 1 AS stats, 0 AS fold_wildcards, 'hits' as sort_mode )</sphinxql>
</sphqueries>

</test>

0 comments on commit 0920832

Please sign in to comment.