Skip to content
Permalink
Browse files

fixed #979 replication of RT index into node where same RT index exis…

…ts and has different path; added regression to test 385
  • Loading branch information...
tomatolog committed Aug 14, 2019
1 parent 0caa694 commit 0a1a2c81038794983b4e30c8aefc7cc20e89b03f
Showing with 108 additions and 9 deletions.
  1. +16 −9 src/searchdreplication.cpp
  2. +12 −0 src/sphinxutils.cpp
  3. +1 −0 src/sphinxutils.h
  4. +1 −0 test/test_385/model.bin
  5. +78 −0 test/test_385/test.xml
@@ -3924,25 +3924,32 @@ bool RemoteFileReserve ( const PQRemoteData_t & tCmd, PQRemoteReply_t & tRes, CS
}
}

tRes.m_pDst->m_dRemotePaths.Reset ( tCmd.m_pChunks->m_dBaseNames.GetLength() );

// use cluster path as head of index path or existed index path
CSphString sIndexPath;
if ( tRes.m_bIndexActive )
{
sIndexPath = GetPathOnly ( tRes.m_sRemoteIndexPath );
// set index files names into existing index files
ARRAY_FOREACH ( iFile, tCmd.m_pChunks->m_dBaseNames )
{
const CSphString & sFile = tCmd.m_pChunks->m_dBaseNames[iFile];
tRes.m_pDst->m_dRemotePaths[iFile].SetSprintf ( "%s.%s", tRes.m_sRemoteIndexPath.cstr(), GetExtension ( sFile.cstr() ) );
}
} else
{
CSphString sIndexPath;
if ( !GetClusterPath ( tCmd.m_sCluster, sIndexPath, sError ) )
return false;
tRes.m_sRemoteIndexPath.SetSprintf ( "%s/%s", sIndexPath.cstr(), tCmd.m_sIndexFileName.cstr() );
}

// set index files names
tRes.m_pDst->m_dRemotePaths.Reset ( tCmd.m_pChunks->m_dBaseNames.GetLength() );
ARRAY_FOREACH ( iFile, tCmd.m_pChunks->m_dBaseNames )
{
const CSphString & sFile = tCmd.m_pChunks->m_dBaseNames[iFile];
tRes.m_pDst->m_dRemotePaths[iFile].SetSprintf ( "%s/%s", sIndexPath.cstr(), sFile.cstr() );
// set index files names into cluster folder
ARRAY_FOREACH ( iFile, tCmd.m_pChunks->m_dBaseNames )
{
const CSphString & sFile = tCmd.m_pChunks->m_dBaseNames[iFile];
tRes.m_pDst->m_dRemotePaths[iFile].SetSprintf ( "%s/%s", sIndexPath.cstr(), sFile.cstr() );
}
}

int iBits = tCmd.m_pChunks->m_dChunks.Last().m_iHashStartItem + tCmd.m_pChunks->m_dChunks.Last().GetChunksCount();
tRes.m_pDst->m_dNodeChunks.Init ( iBits );

@@ -3888,6 +3888,18 @@ CSphString GetPathOnly ( const CSphString & sFullPath )

}

const char * GetExtension ( const CSphString & sFullPath )
{
if ( sFullPath.IsEmpty() )
return nullptr;

const char * pDot = strchr ( sFullPath.cstr(), '.' );
if ( !pDot || pDot[1]=='\0' )
return nullptr;

return pDot+1;
}

static CSphAtomic_T<int64_t> g_iUID { 1 };
static int64_t g_iUidBase = 0;

@@ -496,6 +496,7 @@ namespace TlsMsg
// extract basename from path
const char * GetBaseName ( const CSphString & sFullPath );
CSphString GetPathOnly ( const CSphString & sFullPath );
const char * GetExtension ( const CSphString & sFullPath );
bool HasMvaUpdated ( const CSphString & sIndexPath );

// uuid short generator - static across daemon
@@ -0,0 +1 @@
a:1:{i:0;a:13:{i:0;a:2:{s:8:"sphinxql";s:189:"insert into rt1 (id,title,mva1,gid,j1) values (1, 'test 1', (11,12,13), 11, '{"data":"test 1", fd:11}'), (2, 'test 2', (21,22,23), 21, ''), (3, 'test 3', (), 31, '{"data":"test 3", fd:31}')";s:14:"total_affected";i:3;}i:1;a:3:{s:8:"sphinxql";s:22:"SELECT * FROM test:rt1";s:10:"total_rows";i:3;s:4:"rows";a:3:{i:0;a:4:{s:2:"id";s:1:"1";s:3:"gid";s:2:"11";s:2:"j1";s:25:"{"data":"test 1","fd":11}";s:4:"mva1";s:8:"11,12,13";}i:1;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:2:"21";s:2:"j1";N;s:4:"mva1";s:8:"21,22,23";}i:2;a:4:{s:2:"id";s:1:"3";s:3:"gid";s:2:"31";s:2:"j1";s:25:"{"data":"test 3","fd":31}";s:4:"mva1";s:0:"";}}}i:2;a:2:{s:8:"sphinxql";s:190:"replace into rt1 (id,title,mva1,gid,j1) values (5, 'test 5', (51,52,53), 51, '{"data":"test 5", fd:51}'), (1, 'test 6', (61,62,63), 61, '{"data":"test 6", fd:16}'), (7, 'test 7', (), 71, '')";s:14:"total_affected";i:3;}i:3;a:3:{s:8:"sphinxql";s:22:"SELECT * FROM test:rt1";s:10:"total_rows";i:5;s:4:"rows";a:5:{i:0;a:4:{s:2:"id";s:1:"1";s:3:"gid";s:2:"61";s:2:"j1";s:25:"{"data":"test 6","fd":16}";s:4:"mva1";s:8:"61,62,63";}i:1;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:2:"21";s:2:"j1";N;s:4:"mva1";s:8:"21,22,23";}i:2;a:4:{s:2:"id";s:1:"3";s:3:"gid";s:2:"31";s:2:"j1";s:25:"{"data":"test 3","fd":31}";s:4:"mva1";s:0:"";}i:3;a:4:{s:2:"id";s:1:"5";s:3:"gid";s:2:"51";s:2:"j1";s:25:"{"data":"test 5","fd":51}";s:4:"mva1";s:8:"51,52,53";}i:4;a:4:{s:2:"id";s:1:"7";s:3:"gid";s:2:"71";s:2:"j1";N;s:4:"mva1";s:0:"";}}}i:4;a:2:{s:8:"sphinxql";s:19:"create cluster test";s:14:"total_affected";i:0;}i:5;a:2:{s:8:"sphinxql";s:26:"ALTER CLUSTER test ADD rt1";s:14:"total_affected";i:0;}i:6;a:3:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:69:"insert into rt1 (id,title,mva1) values (10, 'test 10', (110,120,130))";s:14:"total_affected";i:1;}i:7;a:4:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:17:"SELECT * FROM rt1";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:4:{s:2:"id";s:2:"10";s:3:"gid";s:1:"0";s:2:"j1";N;s:4:"mva1";s:11:"110,120,130";}}}i:8;a:3:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:37:"join cluster test at '%addr_connect%'";s:14:"total_affected";i:0;}i:9;a:4:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:22:"SELECT * FROM test:rt1";s:10:"total_rows";i:5;s:4:"rows";a:5:{i:0;a:4:{s:2:"id";s:1:"1";s:3:"gid";s:2:"61";s:2:"j1";s:25:"{"data":"test 6","fd":16}";s:4:"mva1";s:8:"61,62,63";}i:1;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:2:"21";s:2:"j1";N;s:4:"mva1";s:8:"21,22,23";}i:2;a:4:{s:2:"id";s:1:"3";s:3:"gid";s:2:"31";s:2:"j1";s:25:"{"data":"test 3","fd":31}";s:4:"mva1";s:0:"";}i:3;a:4:{s:2:"id";s:1:"5";s:3:"gid";s:2:"51";s:2:"j1";s:25:"{"data":"test 5","fd":51}";s:4:"mva1";s:8:"51,52,53";}i:4;a:4:{s:2:"id";s:1:"7";s:3:"gid";s:2:"71";s:2:"j1";N;s:4:"mva1";s:0:"";}}}i:10;a:3:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:75:"replace into test:rt1 (id,title,mva1) values (20, 'test 20', (151,152,153))";s:14:"total_affected";i:1;}i:11;a:4:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:22:"SELECT * FROM test:rt1";s:10:"total_rows";i:6;s:4:"rows";a:6:{i:0;a:4:{s:2:"id";s:1:"1";s:3:"gid";s:2:"61";s:2:"j1";s:25:"{"data":"test 6","fd":16}";s:4:"mva1";s:8:"61,62,63";}i:1;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:2:"21";s:2:"j1";N;s:4:"mva1";s:8:"21,22,23";}i:2;a:4:{s:2:"id";s:1:"3";s:3:"gid";s:2:"31";s:2:"j1";s:25:"{"data":"test 3","fd":31}";s:4:"mva1";s:0:"";}i:3;a:4:{s:2:"id";s:1:"5";s:3:"gid";s:2:"51";s:2:"j1";s:25:"{"data":"test 5","fd":51}";s:4:"mva1";s:8:"51,52,53";}i:4;a:4:{s:2:"id";s:1:"7";s:3:"gid";s:2:"71";s:2:"j1";N;s:4:"mva1";s:0:"";}i:5;a:4:{s:2:"id";s:2:"20";s:3:"gid";s:1:"0";s:2:"j1";N;s:4:"mva1";s:11:"151,152,153";}}}i:12;a:3:{s:8:"sphinxql";s:22:"SELECT * FROM test:rt1";s:10:"total_rows";i:6;s:4:"rows";a:6:{i:0;a:4:{s:2:"id";s:1:"1";s:3:"gid";s:2:"61";s:2:"j1";s:25:"{"data":"test 6","fd":16}";s:4:"mva1";s:8:"61,62,63";}i:1;a:4:{s:2:"id";s:1:"2";s:3:"gid";s:2:"21";s:2:"j1";N;s:4:"mva1";s:8:"21,22,23";}i:2;a:4:{s:2:"id";s:1:"3";s:3:"gid";s:2:"31";s:2:"j1";s:25:"{"data":"test 3","fd":31}";s:4:"mva1";s:0:"";}i:3;a:4:{s:2:"id";s:1:"5";s:3:"gid";s:2:"51";s:2:"j1";s:25:"{"data":"test 5","fd":51}";s:4:"mva1";s:8:"51,52,53";}i:4;a:4:{s:2:"id";s:1:"7";s:3:"gid";s:2:"71";s:2:"j1";N;s:4:"mva1";s:0:"";}i:5;a:4:{s:2:"id";s:2:"20";s:3:"gid";s:1:"0";s:2:"j1";N;s:4:"mva1";s:11:"151,152,153";}}}}}
@@ -0,0 +1,78 @@
<?xml version="1.0" encoding="utf-8"?>
<test>
<name>replication for RT with different local index files</name>

<requires><replication/><non-windows/></requires>
<skip_indexer/>
<num_agents>2</num_agents>

<config>

<agent0>
searchd
{
<searchd_settings/>
data_dir = <data_path path="data0"></data_path>
server_id = <agent_id/>
}

index rt1
{
type = rt
path = <data_path/>/rt1
rt_mem_limit = 128k

rt_field = title
rt_attr_multi = mva1
rt_attr_uint = gid
rt_attr_json = j1
}
</agent0>
<agent1>
searchd
{
<searchd_settings/>
data_dir = <data_path path="data1"></data_path>
server_id = <agent_id/>
}

index rt1
{
type = rt
path = <data_path/>/rt_one
rt_mem_limit = 128k

rt_field = title
rt_attr_multi = mva1
rt_attr_uint = gid
rt_attr_json = j1
}

</agent1>

</config>


<sphqueries>
<!-- request to node 0 -->
<sphinxql d="0">insert into rt1 (id,title,mva1,gid,j1) values (1, 'test 1', (11,12,13), 11, '{"data":"test 1", fd:11}'), (2, 'test 2', (21,22,23), 21, ''), (3, 'test 3', (), 31, '{"data":"test 3", fd:31}')</sphinxql>
<sphinxql d="0">SELECT * FROM test:rt1</sphinxql>
<sphinxql d="0">replace into rt1 (id,title,mva1,gid,j1) values (5, 'test 5', (51,52,53), 51, '{"data":"test 5", fd:51}'), (1, 'test 6', (61,62,63), 61, '{"data":"test 6", fd:16}'), (7, 'test 7', (), 71, '')</sphinxql>
<sphinxql d="0">SELECT * FROM test:rt1</sphinxql>

<sphinxql d="0">create cluster test</sphinxql>
<sphinxql d="0">ALTER CLUSTER test ADD rt1</sphinxql>

<!-- request to node 1 -->
<sphinxql d="1">insert into rt1 (id,title,mva1) values (10, 'test 10', (110,120,130))</sphinxql>
<sphinxql d="1">SELECT * FROM rt1</sphinxql>
<sphinxql d="1" cluster_connect="0">join cluster test at '%addr_connect%'</sphinxql>
<sphinxql d="1">SELECT * FROM test:rt1</sphinxql>
<sphinxql d="1">replace into test:rt1 (id,title,mva1) values (20, 'test 20', (151,152,153))</sphinxql>
<sphinxql d="1">SELECT * FROM test:rt1</sphinxql>

<sphinxql d="0">SELECT * FROM test:rt1</sphinxql>

</sphqueries>

</test>

0 comments on commit 0a1a2c8

Please sign in to comment.
You can’t perform that action at this time.