Permalink
Browse files

fixed #29 optimize rt index loose new settings; fixed #30 optimize wi…

…th sync option lock leaks; added regressions to test 317
  • Loading branch information...
tomatolog authored and klirichek committed Jun 2, 2017
1 parent cb99164 commit e406761d507ff2b96749922864ac026c13665dd6
Showing with 259 additions and 81 deletions.
  1. +4 −4 src/indexer.cpp
  2. +3 −0 src/searchd.cpp
  3. +100 −61 src/sphinx.cpp
  4. +5 −5 src/sphinx.h
  5. +9 −9 src/sphinxint.h
  6. +2 −2 src/sphinxrt.cpp
  7. +32 −0 test/test_317/model.bin
  8. +104 −0 test/test_317/test.xml
@@ -204,15 +204,15 @@ class CSphStopwordBuilderDict : public CSphDict
virtual void LoadStopwords ( const char *, const ISphTokenizer * ) {}
virtual void LoadStopwords ( const CSphVector<SphWordID_t> & ) {}
virtual void WriteStopwords ( CSphWriter & ) {}
virtual void WriteStopwords ( CSphWriter & ) const {}
virtual bool LoadWordforms ( const CSphVector<CSphString> &, const CSphEmbeddedFiles *, const ISphTokenizer *, const char * ) { return true; }
virtual void WriteWordforms ( CSphWriter & ) {}
virtual void WriteWordforms ( CSphWriter & ) const {}
virtual int SetMorphology ( const char *, CSphString & ) { return ST_OK; }
virtual void Setup ( const CSphDictSettings & tSettings ) { m_tSettings = tSettings; }
virtual const CSphDictSettings & GetSettings () const { return m_tSettings; }
virtual const CSphVector <CSphSavedFile> & GetStopwordsFileInfos () { return m_dSWFileInfos; }
virtual const CSphVector <CSphSavedFile> & GetWordformsFileInfos () { return m_dWFFileInfos; }
virtual const CSphVector <CSphSavedFile> & GetStopwordsFileInfos () const { return m_dSWFileInfos; }
virtual const CSphVector <CSphSavedFile> & GetWordformsFileInfos () const { return m_dWFFileInfos; }
virtual const CSphMultiformContainer * GetMultiWordforms () const { return NULL; }
virtual uint64_t GetSettingsFNV () const { return 0; }
@@ -15472,6 +15472,9 @@ void HandleMysqlOptimize ( SqlRowBuffer_c & tOut, const SqlStmt_t & tStmt )
const ServedIndex_c * pServed = g_pLocalIndexes->GetRlockedEntry ( tStmt.m_sIndex );
if ( pServed && pServed->m_pIndex && pServed->m_bEnabled )
static_cast<ISphRtIndex *>( pServed->m_pIndex )->Optimize ( &g_bShutdown, &g_tRtThrottle );
if ( pServed )
pServed->Unlock();
return;
}

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -569,7 +569,7 @@ class ISphTokenizer
virtual bool LoadSynonyms ( const char * sFilename, const CSphEmbeddedFiles * pFiles, CSphString & sError ) = 0;
/// write synonyms to file
virtual void WriteSynonyms ( CSphWriter & tWriter ) = 0;
virtual void WriteSynonyms ( CSphWriter & tWriter ) const = 0;
/// set phrase boundary chars
virtual bool SetBoundary ( const char * sConfig, CSphString & sError );
@@ -857,13 +857,13 @@ class CSphDict
virtual void LoadStopwords ( const CSphVector<SphWordID_t> & dStopwords ) = 0;
/// write stopwords to a file
virtual void WriteStopwords ( CSphWriter & tWriter ) = 0;
virtual void WriteStopwords ( CSphWriter & tWriter ) const = 0;
/// load wordforms from a given list of files
virtual bool LoadWordforms ( const CSphVector<CSphString> &, const CSphEmbeddedFiles * pEmbedded, const ISphTokenizer * pTokenizer, const char * sIndex ) = 0;
/// write wordforms to a file
virtual void WriteWordforms ( CSphWriter & tWriter ) = 0;
virtual void WriteWordforms ( CSphWriter & tWriter ) const = 0;
/// get wordforms
virtual const CSphWordforms * GetWordforms() { return NULL; }
@@ -888,10 +888,10 @@ class CSphDict
virtual const CSphDictSettings & GetSettings () const = 0;
/// stopwords file infos
virtual const CSphVector <CSphSavedFile> & GetStopwordsFileInfos () = 0;
virtual const CSphVector <CSphSavedFile> & GetStopwordsFileInfos () const = 0;
/// wordforms file infos
virtual const CSphVector <CSphSavedFile> & GetWordformsFileInfos () = 0;
virtual const CSphVector <CSphSavedFile> & GetWordformsFileInfos () const = 0;
/// get multiwordforms
virtual const CSphMultiformContainer * GetMultiWordforms () const = 0;
@@ -1582,9 +1582,9 @@ class CSphDictTraits : public CSphDict
virtual void LoadStopwords ( const char * sFiles, const ISphTokenizer * pTokenizer ) { m_pDict->LoadStopwords ( sFiles, pTokenizer ); }
virtual void LoadStopwords ( const CSphVector<SphWordID_t> & dStopwords ) { m_pDict->LoadStopwords ( dStopwords ); }
virtual void WriteStopwords ( CSphWriter & tWriter ) { m_pDict->WriteStopwords ( tWriter ); }
virtual void WriteStopwords ( CSphWriter & tWriter ) const { m_pDict->WriteStopwords ( tWriter ); }
virtual bool LoadWordforms ( const CSphVector<CSphString> & dFiles, const CSphEmbeddedFiles * pEmbedded, const ISphTokenizer * pTokenizer, const char * sIndex ) { return m_pDict->LoadWordforms ( dFiles, pEmbedded, pTokenizer, sIndex ); }
virtual void WriteWordforms ( CSphWriter & tWriter ) { m_pDict->WriteWordforms ( tWriter ); }
virtual void WriteWordforms ( CSphWriter & tWriter ) const { m_pDict->WriteWordforms ( tWriter ); }
virtual int SetMorphology ( const char * szMorph, CSphString & sMessage ) { return m_pDict->SetMorphology ( szMorph, sMessage ); }
virtual SphWordID_t GetWordID ( const BYTE * pWord, int iLen, bool bFilterStops ) { return m_pDict->GetWordID ( pWord, iLen, bFilterStops ); }
@@ -1593,8 +1593,8 @@ class CSphDictTraits : public CSphDict
virtual void Setup ( const CSphDictSettings & ) {}
virtual const CSphDictSettings & GetSettings () const { return m_pDict->GetSettings (); }
virtual const CSphVector <CSphSavedFile> & GetStopwordsFileInfos () { return m_pDict->GetStopwordsFileInfos (); }
virtual const CSphVector <CSphSavedFile> & GetWordformsFileInfos () { return m_pDict->GetWordformsFileInfos (); }
virtual const CSphVector <CSphSavedFile> & GetStopwordsFileInfos () const { return m_pDict->GetStopwordsFileInfos (); }
virtual const CSphVector <CSphSavedFile> & GetWordformsFileInfos () const { return m_pDict->GetWordformsFileInfos (); }
virtual const CSphMultiformContainer * GetMultiWordforms () const { return m_pDict->GetMultiWordforms (); }
virtual const CSphWordforms * GetWordforms () { return m_pDict->GetWordforms(); }
@@ -1661,7 +1661,7 @@ class CSphTokenFilter : public ISphTokenizer
virtual bool SetNgramChars ( const char * sConfig, CSphString & sError ) { return m_pTokenizer->SetNgramChars ( sConfig, sError ); }
virtual void SetNgramLen ( int iLen ) { m_pTokenizer->SetNgramLen ( iLen ); }
virtual bool LoadSynonyms ( const char * sFilename, const CSphEmbeddedFiles * pFiles, CSphString & sError ) { return m_pTokenizer->LoadSynonyms ( sFilename, pFiles, sError ); }
virtual void WriteSynonyms ( CSphWriter & tWriter ) { return m_pTokenizer->WriteSynonyms ( tWriter ); }
virtual void WriteSynonyms ( CSphWriter & tWriter ) const { return m_pTokenizer->WriteSynonyms ( tWriter ); }
virtual bool SetBoundary ( const char * sConfig, CSphString & sError ) { return m_pTokenizer->SetBoundary ( sConfig, sError ); }
virtual void Setup ( const CSphTokenizerSettings & tSettings ) { m_pTokenizer->Setup ( tSettings ); }
virtual const CSphTokenizerSettings & GetSettings () const { return m_pTokenizer->GetSettings (); }
@@ -1767,7 +1767,7 @@ void sphColumnToLowercase ( char * sVal );
bool sphCheckQueryHeight ( const struct XQNode_t * pRoot, CSphString & sError );
void sphTransformExtendedQuery ( XQNode_t ** ppNode, const CSphIndexSettings & tSettings, bool bHasBooleanOptimization, const ISphKeywordsStat * pKeywords );
void TransformAotFilter ( XQNode_t * pNode, const CSphWordforms * pWordforms, const CSphIndexSettings& tSettings );
bool sphMerge ( const CSphIndex * pDst, const CSphIndex * pSrc, const CSphVector<SphDocID_t> & dKillList, CSphString & sError, CSphIndexProgress & tProgress, ThrottleState_t * pThrottle, volatile bool * pGlobalStop, volatile bool * pLocalStop );
bool sphMerge ( const CSphIndex * pDst, const CSphIndex * pSrc, const CSphVector<SphDocID_t> & dKillList, CSphString & sError, CSphIndexProgress & tProgress, ThrottleState_t * pThrottle, volatile bool * pGlobalStop, volatile bool * pLocalStop, bool bSrcSettings );
CSphString sphReconstructNode ( const XQNode_t * pNode, const CSphSchema * pSchema );
void sphSetUnlinkOld ( bool bUnlink );
@@ -1777,12 +1777,12 @@ void WriteSchema ( CSphWriter & fdInfo, const CSphSchema & tSchema );
void ReadSchema ( CSphReader & rdInfo, CSphSchema & m_tSchema, DWORD uVersion, bool bDynamic );
void SaveIndexSettings ( CSphWriter & tWriter, const CSphIndexSettings & tSettings );
void LoadIndexSettings ( CSphIndexSettings & tSettings, CSphReader & tReader, DWORD uVersion );
void SaveTokenizerSettings ( CSphWriter & tWriter, ISphTokenizer * pTokenizer, int iEmbeddedLimit );
void SaveTokenizerSettings ( CSphWriter & tWriter, const ISphTokenizer * pTokenizer, int iEmbeddedLimit );
bool LoadTokenizerSettings ( CSphReader & tReader, CSphTokenizerSettings & tSettings, CSphEmbeddedFiles & tEmbeddedFiles, DWORD uVersion, CSphString & sWarning );
void SaveDictionarySettings ( CSphWriter & tWriter, CSphDict * pDict, bool bForceWordDict, int iEmbeddedLimit );
void SaveDictionarySettings ( CSphWriter & tWriter, const CSphDict * pDict, bool bForceWordDict, int iEmbeddedLimit );
void LoadDictionarySettings ( CSphReader & tReader, CSphDictSettings & tSettings, CSphEmbeddedFiles & tEmbeddedFiles, DWORD uVersion, CSphString & sWarning );
void LoadFieldFilterSettings ( CSphReader & tReader, CSphFieldFilterSettings & tFieldFilterSettings );
void SaveFieldFilterSettings ( CSphWriter & tWriter, ISphFieldFilter * pFieldFilter );
void SaveFieldFilterSettings ( CSphWriter & tWriter, const ISphFieldFilter * pFieldFilter );
DWORD ReadVersion ( const char * sPath, CSphString & sError );
bool AddFieldLens ( CSphSchema & tSchema, bool bDynamic, CSphString & sError );
@@ -8727,7 +8727,7 @@ void RtIndex_t::Optimize ( volatile bool * pForceTerminate, ThrottleState_t * pT
// merge data to disk ( data is constant during that phase )
CSphIndexProgress tProgress;
bool bMerged = sphMerge ( pOldest, pOlder, dKlist, sError, tProgress, pThrottle, pForceTerminate, &m_bOptimizeStop );
bool bMerged = sphMerge ( pOldest, pOlder, dKlist, sError, tProgress, pThrottle, pForceTerminate, &m_bOptimizeStop, true );
if ( !bMerged )
{
sphWarning ( "rt optimize: index %s: failed to merge %s to %s (error %s)",
@@ -8959,7 +8959,7 @@ void RtIndex_t::ProgressiveMerge ( volatile bool * pForceTerminate, ThrottleStat
// merge data to disk ( data is constant during that phase )
CSphIndexProgress tProgress;
bool bMerged = sphMerge ( pDst, pSrc, dKlist, sError, tProgress, pThrottle, pForceTerminate, &m_bOptimizeStop );
bool bMerged = sphMerge ( pDst, pSrc, dKlist, sError, tProgress, pThrottle, pForceTerminate, &m_bOptimizeStop, true );
if ( !bMerged )
{
sphWarning ( "rt optimize: index %s: failed to merge %s to %s (error %s)",
@@ -0,0 +1,32 @@
a:1:{i:0;a:1:{i:0;a:22:{i:0;s:14:"initial commit";i:1;s:3:"
OK";i:2;s:3:"
OK";i:3;s:10:"query test";i:4;s:19:"
1 | 1
2 | 2
2 rows";i:5;s:10:"query tes*";i:6;s:7:"
0 rows";i:7;s:3:"
OK";i:8;s:3:"
OK";i:9;s:3:"
OK";i:10;s:10:"query test";i:11;s:35:"
11 | 11
12 | 12
1 | 1
2 | 2
4 rows";i:12;s:10:"query tes*";i:13;s:23:"
11 | 11
12 | 12
2 rows";i:14;s:3:"
OK";i:15;s:3:"
OK";i:16;s:10:"query test";i:17;s:35:"
1 | 1
2 | 2
11 | 11
12 | 12
4 rows";i:18;s:10:"query tes*";i:19;s:35:"
1 | 1
2 | 2
11 | 11
12 | 12
4 rows";i:20;s:3:"
OK";i:21;s:7:"
0 rows";}}}
@@ -0,0 +1,104 @@
<?xml version="1.0" encoding="utf-8"?>
<test>
<name>RECONFIGURE in RT indexes (chunk merge order)</name>
<config>
searchd
{
<searchd_settings/>
binlog_path =
workers = threads
}
index rt
{
type = rt
path = <data_path/>/rt
charset_table = a..z, _, A..Z->a..z
dict = keywords
rt_field = name
rt_attr_uint = idd
#min_prefix_len = 3
}
source src_dummy
{
type = mysql
<sql_settings/>
sql_query = SELECT id, gid, text FROM test_table
sql_attr_uint = gid
}
index test_dummy
{
source = src_dummy
path = <data_path/>/test_dummy
}
</config>
<db_drop>drop table if exists test_table</db_drop>
<db_create>
create table test_table
(
id int not null,
text varchar(255) not null,
gid int
);
</db_create>
<db_insert>
insert into test_table values ( 1, 'first', 1 )
</db_insert>
<custom_test><![CDATA[
$results = array();
$ql->Reconnect();
$results[] = 'initial commit';
$results[] = "\n" . $ql->Query ( "REPLACE INTO rt (id, name, idd) VALUES (1, 'test', 1)" );
$results[] = "\n" . $ql->Query ( "REPLACE INTO rt (id, name, idd) VALUES (2, 'test', 2)" );
$results[] = 'query test';
$results[] = "\n" . $ql->Query ( "SELECT * FROM rt WHERE MATCH ( 'test' )" );
$results[] = 'query tes*';
$results[] = "\n" . $ql->Query ( "SELECT * FROM rt WHERE MATCH ( 'tes*' )" );
$orig_conf = file('config.conf');
$new_conf = '';
foreach ( $orig_conf as $line )
{
if ( strstr ( $line, '#min_prefix_len' ) )
{
$line = 'min_prefix_len = 3' . "\n";
}
$new_conf .= $line;
}
file_put_contents('config.conf', $new_conf);
$results[] = "\n" . $ql->Query ( "ALTER RTINDEX rt RECONFIGURE" );
$results[] = "\n" . $ql->Query ( "REPLACE INTO rt (id, name, idd) VALUES (11, 'test', 11)" );
$results[] = "\n" . $ql->Query ( "REPLACE INTO rt (id, name, idd) VALUES (12, 'test', 12)" );
$results[] = 'query test';
$results[] = "\n" . $ql->Query ( "SELECT * FROM rt WHERE MATCH ( 'test' )" );
$results[] = 'query tes*';
$results[] = "\n" . $ql->Query ( "SELECT * FROM rt WHERE MATCH ( 'tes*' )" );
$results[] = "\n" . $ql->Query ( "FLUSH RAMCHUNK rt" );
$results[] = "\n" . $ql->Query ( "OPTIMIZE INDEX rt option sync=1" );
$results[] = 'query test';
$results[] = "\n" . $ql->Query ( "SELECT * FROM rt WHERE MATCH ( 'test' )" );
$results[] = 'query tes*';
$results[] = "\n" . $ql->Query ( "SELECT * FROM rt WHERE MATCH ( 'tes*' )" );
// regression rt index lock leaks after optimize command
$results[] = "\n" . $ql->Query ( "truncate RTINDEX rt" );
$results[] = "\n" . $ql->Query ( "SELECT * FROM rt" );
]]></custom_test>
</test>

0 comments on commit e406761

Please sign in to comment.