From 71f81d8f338af0cf5d5e77a53992bfce5619ce7c Mon Sep 17 00:00:00 2001 From: Stas Klinov Date: Thu, 10 Aug 2023 17:07:00 +0200 Subject: [PATCH] fixed replication nodes retry after some nodes are missed and name resolution of these nodes failed (should fixed kubernetes and docker nodes issues with replication); fixed error message at replication start failure; fixed model at test 376; added error code and readable message on name resolution failure; fixed dev#530 --- src/searchdaemon.cpp | 6 +- src/searchdreplication.cpp | 163 +++++++++++++++++++++---------------- test/test_376/model.bin | 2 +- 3 files changed, 96 insertions(+), 75 deletions(-) diff --git a/src/searchdaemon.cpp b/src/searchdaemon.cpp index 4b9b1daa31..9815b74e8d 100644 --- a/src/searchdaemon.cpp +++ b/src/searchdaemon.cpp @@ -677,11 +677,11 @@ DWORD sphGetAddress ( const char * sHost, bool bFatal, bool bIP, CSphString * pF if ( iResult!=0 || !pResult ) { if ( pFatal ) - pFatal->SetSprintf ( "no AF_INET address found for: %s", sHost ); + pFatal->SetSprintf ( "no AF_INET address found for: %s, error %d: %s", sHost, iResult, gai_strerror(iResult) ); else if ( bFatal ) - sphFatal( "no AF_INET address found for: %s", sHost ); + sphFatal( "no AF_INET address found for: %s, error %d: %s", sHost, iResult, gai_strerror(iResult) ); else - sphLogDebugv( "no AF_INET address found for: %s", sHost ); + sphLogDebugv( "no AF_INET address found for: %s, error %d: %s", sHost, iResult, gai_strerror(iResult) ); return 0; } diff --git a/src/searchdreplication.cpp b/src/searchdreplication.cpp index ee08e640ac..9990eb1692 100644 --- a/src/searchdreplication.cpp +++ b/src/searchdreplication.cpp @@ -294,9 +294,6 @@ static bool ClusterGetNodes ( const CSphString & sClusterNodes, const CSphString // callback at remote node for CLUSTER_GET_NODES to return actual nodes list at cluster static bool RemoteClusterGetNodes ( const CSphString & sCluster, const CSphString & sGTID, CSphString & sError, CSphString & sNodes ); -// utility function to filter nodes list provided at string by specific protocol -static bool ClusterFilterNodes ( Str_t sSrcNodes, Proto_e eProto, CSphString & sDstNodes, CSphString & sError ); - // callback at remote node for CLUSTER_UPDATE_NODES to update nodes list at cluster from actual nodes list static bool RemoteClusterUpdateNodes ( const CSphString & sCluster, CSphString * pNodes, bool bSaveConf, bool bMergeNodes, CSphString & sError ); @@ -3133,49 +3130,45 @@ struct VecAgentDesc_t : public ISphNoncopyable, public CSphVector // get nodes of specific type from string class ISphDescIterator { -protected: - CSphString & m_sError; - virtual void ResetNodes () = 0; - public: - ISphDescIterator ( CSphString & sError ) - : m_sError ( sError ) - {} - bool ProcessNodes ( const CSphString & sNodes ) + bool ProcessNodes ( const char * sNodes, CSphString & sError, CSphString & sWarning ) { - bool bOk = false; - int iRetry = 0; - const int iRetryMax = g_iNodeRetry; - while ( true ) + if ( !sNodes || !*sNodes ) { - bOk = SplitNodes ( sNodes ); - iRetry++; + m_sError += "empty nodes list"; + return false; + } - if ( !bOk && iRetry=g_iNodeRetry ) break; - } - return bOk; - } + m_sError.Clear(); + ResetNodes(); + // should wait and retry for DNS set + Threads::Coro::SleepMsec ( g_iAnyNodesTimeout ); + } -private: - bool SplitNodes ( const CSphString & sNodes ) - { - if ( sNodes.IsEmpty () ) + if ( IsValid() ) + { + sWarning = m_sError.cstr(); + return true; + } else { - m_sError = "empty nodes list"; + sError = m_sError.cstr(); return false; } + } +private: + bool SplitNodes ( const char * sNodes ) + { + bool bOk = true; CSphString sTmp = sNodes; char * sCur = const_cast( sTmp.cstr() ); while ( *sCur ) @@ -3195,14 +3188,25 @@ class ISphDescIterator sCur++; } - if ( *sAddrs && !SetNode ( sAddrs ) ) - return false; + if ( !*sAddrs ) + continue; + + CSphString sError; + if ( !SetNode ( sAddrs, sError ) ) + { + bOk = false; + m_sError += sError.cstr(); + } } - return true; + return bOk; } - virtual bool SetNode ( const char * sAddrs ) = 0; + virtual bool SetNode ( const char * sAddrs, CSphString & sError ) = 0; + virtual void ResetNodes () = 0; + virtual bool IsValid() const = 0; + + StringBuilder_c m_sError { ";" }; }; CSphString GetAddr ( const ListenerDesc_t & tListen ) @@ -3225,16 +3229,20 @@ class AgentDescIterator_t : public ISphDescIterator VecAgentDesc_t & m_dNodes; void ResetNodes () override { m_dNodes.Reset(); } + bool IsValid() const override { return !m_dNodes.IsEmpty(); } public: - AgentDescIterator_t ( VecAgentDesc_t & dNodes, CSphString & sError ) - : ISphDescIterator ( sError ) - , m_dNodes ( dNodes ) + AgentDescIterator_t ( VecAgentDesc_t & dNodes ) + : m_dNodes ( dNodes ) {} - bool SetNode ( const char * sListen ) override + bool SetNode ( const char * sListen, CSphString & sError ) override { - ListenerDesc_t tListen = ParseListener ( sListen, &m_sError ); + // filter out own address to do not query itself + if ( g_sIncomingProto.Begins ( sListen ) ) + return true; + + ListenerDesc_t tListen = ParseListener ( sListen, &sError ); if ( tListen.m_eProto==Proto_e::UNKNOWN ) return false; @@ -3245,10 +3253,6 @@ class AgentDescIterator_t : public ISphDescIterator if ( tListen.m_uIP==0 ) return true; - // filter out own address to do not query itself - if ( g_sIncomingProto.Begins ( sListen ) ) - return true; - AgentDesc_t * pDesc = new AgentDesc_t; m_dNodes.Add( pDesc ); pDesc->m_sAddr = GetAddr ( tListen ); @@ -3262,10 +3266,16 @@ class AgentDescIterator_t : public ISphDescIterator } }; -static bool GetNodes ( const CSphString & sNodes, VecAgentDesc_t & dNodes, CSphString & sError ) +static bool GetNodes ( const char * sNodes, VecAgentDesc_t & dNodes, CSphString & sError ) { - AgentDescIterator_t tIt ( dNodes, sError ); - return tIt.ProcessNodes ( sNodes ); + CSphString sWarning; + AgentDescIterator_t tIt ( dNodes ); + bool bOk = tIt.ProcessNodes ( sNodes, sError, sWarning ); + + if ( bOk && !sWarning.IsEmpty() ) + sphLogDebugRpl ( "node %s parse error: %s", sNodes, sWarning.cstr() ); + + return bOk; } static AgentConn_t * CreateAgent ( const AgentDesc_t & tDesc, const PQRemoteData_t & tReq, int64_t iTimeoutMs ) @@ -3285,7 +3295,7 @@ static AgentConn_t * CreateAgent ( const AgentDesc_t & tDesc, const PQRemoteData return pAgent; } -static bool GetNodes ( const CSphString & sNodes, VecRefPtrs_t & dNodes, const PQRemoteData_t & tReq, CSphString & sError ) +static bool GetNodes ( const char * sNodes, VecRefPtrs_t & dNodes, const PQRemoteData_t & tReq, CSphString & sError ) { VecAgentDesc_t dDesc; if ( !GetNodes ( sNodes, dDesc, sError ) ) @@ -3312,16 +3322,16 @@ class ListenerProtocolIterator_t : public ISphDescIterator sph::StringSet m_hNodes; void ResetNodes () override { m_hNodes.Reset(); } + bool IsValid() const override { return !m_hNodes.IsEmpty(); } public: - ListenerProtocolIterator_t ( Proto_e eProto, CSphString & sError ) - : ISphDescIterator ( sError ) - , m_eProto ( eProto ) + ListenerProtocolIterator_t ( Proto_e eProto ) + : m_eProto ( eProto ) {} - bool SetNode ( const char * sListen ) override + bool SetNode ( const char * sListen, CSphString & sError ) override { - ListenerDesc_t tListen = ParseListener ( sListen, &m_sError ); + ListenerDesc_t tListen = ParseListener ( sListen, &sError ); if ( tListen.m_eProto==Proto_e::UNKNOWN ) return false; @@ -3348,15 +3358,20 @@ class ListenerProtocolIterator_t : public ISphDescIterator }; // utility function to filter nodes list provided at string by specific protocol -bool ClusterFilterNodes ( Str_t sSrcNodes, Proto_e eProto, CSphString & sDstNodes, CSphString & sError ) +static bool ClusterFilterNodes ( const char * sSrcNodes, Proto_e eProto, CSphString & sDstNodes, CSphString & sError ) { - ListenerProtocolIterator_t tIt ( eProto, sError ); - if ( !tIt.ProcessNodes ( sSrcNodes ) ) - return false; + CSphString sWarning; + ListenerProtocolIterator_t tIt ( eProto ); + bool bOk = tIt.ProcessNodes ( sSrcNodes, sError, sWarning ); - sDstNodes = tIt.DumpNodes(); + if ( bOk ) + { + sDstNodes = tIt.DumpNodes(); + if ( !sWarning.IsEmpty() ) + sphLogDebugRpl ( "node %s parse error: %s", sSrcNodes, sWarning.cstr() ); + } - return ( sError.IsEmpty() ); + return bOk; } // base of API commands request and reply builders @@ -3894,7 +3909,7 @@ bool ClusterDelete ( const CSphString & sCluster, CSphString & sError, CSphStrin PQRemoteData_t tCmd; tCmd.m_sCluster = sCluster; VecRefPtrs_t dNodes; - if ( !GetNodes ( sNodes, dNodes, tCmd, sError ) ) + if ( !GetNodes ( sNodes.cstr(), dNodes, tCmd, sError ) ) return false; PQRemoteDelete_c tReq; @@ -5371,7 +5386,7 @@ static bool ClusterAlterAdd ( const CSphString & sCluster, const CSphString & sI if ( !sNodes.IsEmpty() ) { VecAgentDesc_t dDesc; - if ( !GetNodes ( sNodes, dDesc, sError ) ) + if ( !GetNodes ( sNodes.cstr(), dDesc, sError ) ) return false; if ( dDesc.GetLength() && !NodesReplicateIndex ( sCluster, sIndex, dDesc, pServed, sError ) ) @@ -5500,9 +5515,12 @@ bool SendClusterIndexes ( const ReplicationCluster_t * pCluster, const CSphStrin const wsrep_gtid_t & tStateID, CSphString & sError ) EXCLUDES ( g_tClustersLock ) { VecAgentDesc_t dDesc; - if ( !GetNodes ( sNode, dDesc, sError ) || !dDesc.GetLength() ) + if ( !GetNodes ( sNode.cstr(), dDesc, sError ) || !dDesc.GetLength() ) { - sError.SetSprintf ( "%s invalid node", sNode.cstr() ); + if ( sError.IsEmpty() ) + sError.SetSprintf ( "%s invalid node", sNode.cstr() ); + else + sError.SetSprintf ( "%s invalid node, error: %s", sNode.cstr(), sError.cstr() ); return false; } @@ -5660,9 +5678,12 @@ std::optional IsPartOfCluster ( const ServedDesc_t * pDesc ) bool ClusterGetNodes ( const CSphString & sClusterNodes, const CSphString & sCluster, const CSphString & sGTID, Proto_e eProto, CSphString & sError, CSphString & sNodes ) { VecAgentDesc_t dDesc; - if ( !GetNodes ( sClusterNodes, dDesc, sError ) || !dDesc.GetLength() || !sError.IsEmpty() ) + if ( !GetNodes ( sClusterNodes.cstr(), dDesc, sError ) || !dDesc.GetLength() || !sError.IsEmpty() ) { - sError.SetSprintf ( "%s invalid node, %s", sClusterNodes.cstr(), sError.cstr() ); + if ( sError.IsEmpty() ) + sError.SetSprintf ( "%s invalid node", sClusterNodes.cstr() ); + else + sError.SetSprintf ( "%s invalid node, error: %s", sClusterNodes.cstr(), sError.cstr() ); return false; } @@ -5720,7 +5741,7 @@ bool ClusterGetNodes ( const CSphString & sClusterNodes, const CSphString & sClu return false; } - if ( !ClusterFilterNodes ( Str_t ( sAllNodes ), eProto, sNodes, sError ) ) + if ( !ClusterFilterNodes ( sAllNodes.cstr(), eProto, sNodes, sError ) ) return false; if ( sNodes.IsEmpty() ) @@ -5815,7 +5836,7 @@ bool DoClusterAlterUpdate ( const CSphString & sCluster, const CSphString & sUpd tReqData.m_bJoinUpdate = bJoinUpdate; VecRefPtrs_t dNodes; - if ( !GetNodes ( sNodes, dNodes, tReqData, sError ) ) + if ( !GetNodes ( sNodes.cstr(), dNodes, tReqData, sError ) ) return false; PQRemoteClusterUpdateNodes_c tReq; @@ -5863,9 +5884,9 @@ bool RemoteClusterUpdateNodes ( const CSphString & sCluster, CSphString * pNodes sNodes << pCluster->m_sViewNodes.cstr(); } - if ( !ClusterFilterNodes ( Str_t ( sNodes ), Proto_e::SPHINX, pCluster->m_sClusterNodes, sError ) ) + if ( !ClusterFilterNodes ( sNodes.cstr(), Proto_e::SPHINX, pCluster->m_sClusterNodes, sError ) ) { - sError.SetSprintf ( "cluster '%s', invalid nodes(%s), error: %s", sCluster.cstr(), sNodes.cstr(), sError.cstr() ); + sError.SetSprintf ( "cluster '%s', invalid nodes(%s), error: %s", sCluster.cstr(), sNodes.cstr(), sError.scstr() ); return false; } bClusterChanged = ( uWasNodes!=sphFNV64 ( pCluster->m_sClusterNodes.cstr() ) ); diff --git a/test/test_376/model.bin b/test/test_376/model.bin index 2a7a95cc0c..39a4f74041 100644 --- a/test/test_376/model.bin +++ b/test/test_376/model.bin @@ -1 +1 @@ -a:1:{i:0;a:31:{i:0;a:2:{s:8:"sphinxql";s:26:"create table pq1 type='pq'";s:14:"total_affected";i:0;}i:1;a:2:{s:8:"sphinxql";s:26:"create table pq2 type='pq'";s:14:"total_affected";i:0;}i:2;a:2:{s:8:"sphinxql";s:26:"create table pq3 type='pq'";s:14:"total_affected";i:0;}i:3;a:2:{s:8:"sphinxql";s:19:"create cluster test";s:14:"total_affected";i:0;}i:4;a:2:{s:8:"sphinxql";s:26:"ALTER CLUSTER test ADD pq1";s:14:"total_affected";i:0;}i:5;a:2:{s:8:"sphinxql";s:45:"INSERT INTO test:pq1 (query) VALUES('test 1')";s:14:"total_affected";i:1;}i:6;a:2:{s:8:"sphinxql";s:45:"INSERT INTO test:pq1 (query) VALUES('test 2')";s:14:"total_affected";i:1;}i:7;a:2:{s:8:"sphinxql";s:45:"INSERT INTO test:pq1 (query) VALUES('test 3')";s:14:"total_affected";i:1;}i:8;a:3:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:43:"join cluster test '%addr_connect%' as nodes";s:14:"total_affected";i:0;}i:9;a:3:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:37:"join cluster test at '%addr_connect%'";s:14:"total_affected";i:0;}i:10;a:2:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:42:"/* stop-agent => stop=ok, return code=0 */";}i:11;a:2:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:42:"/* stop-agent => stop=ok, return code=0 */";}i:12;a:2:{s:8:"sphinxql";s:46:"INSERT INTO test:pq1 (query) VALUES('test 11')";s:14:"total_affected";i:1;}i:13;a:2:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:46:"/* start-agent => ; start=ok, return code=0 */";}i:14;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:48:"debug wait test like 'state' option 'timeout'=30";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:4:"name";s:5:"state";s:5:"Value";s:6:"synced";}}}i:15;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:22:"SELECT * FROM test:pq1";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:4:{s:2:"id";s:13:"1677721600002";s:5:"query";s:6:"test 2";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:1;a:4:{s:2:"id";s:13:"1677721600003";s:5:"query";s:6:"test 3";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:2;a:4:{s:2:"id";s:13:"1677721600004";s:5:"query";s:7:"test 11";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:3;a:4:{s:2:"id";s:13:"1677721600001";s:5:"query";s:6:"test 1";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}}}i:16;a:2:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:46:"/* start-agent => ; start=ok, return code=0 */";}i:17;a:4:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:48:"debug wait test like 'state' option 'timeout'=30";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:4:"name";s:5:"state";s:5:"Value";s:6:"synced";}}}i:18;a:4:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:22:"SELECT * FROM test:pq1";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:4:{s:2:"id";s:13:"1677721600002";s:5:"query";s:6:"test 2";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:1;a:4:{s:2:"id";s:13:"1677721600003";s:5:"query";s:6:"test 3";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:2;a:4:{s:2:"id";s:13:"1677721600004";s:5:"query";s:7:"test 11";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:3;a:4:{s:2:"id";s:13:"1677721600001";s:5:"query";s:6:"test 1";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}}}i:19;a:3:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:46:"INSERT INTO test:pq1 (query) VALUES('test 12')";s:14:"total_affected";i:1;}i:20;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:57:"debug wait test status 6 like 'state' option 'timeout'=30";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:4:"name";s:5:"state";s:5:"Value";s:1:"6";}}}i:21;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:22:"SELECT * FROM test:pq1";s:10:"total_rows";i:5;s:4:"rows";a:5:{i:0;a:4:{s:2:"id";s:13:"1677721600002";s:5:"query";s:6:"test 2";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:1;a:4:{s:2:"id";s:13:"1677721600003";s:5:"query";s:6:"test 3";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:2;a:4:{s:2:"id";s:13:"1677721600004";s:5:"query";s:7:"test 11";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:3;a:4:{s:2:"id";s:17:"72059271759527937";s:5:"query";s:7:"test 12";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:4;a:4:{s:2:"id";s:13:"1677721600001";s:5:"query";s:6:"test 1";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}}}i:22;a:2:{s:8:"sphinxql";s:27:"ALTER CLUSTER test DROP pq1";s:14:"total_affected";i:0;}i:23;a:2:{s:8:"sphinxql";s:14:"drop table pq1";s:14:"total_affected";i:0;}i:24;a:2:{s:8:"sphinxql";s:14:"drop table pq2";s:14:"total_affected";i:0;}i:25;a:2:{s:8:"sphinxql";s:14:"drop table pq3";s:14:"total_affected";i:0;}i:26;a:2:{s:8:"sphinxql";s:19:"DELETE cluster test";s:14:"total_affected";i:0;}i:27;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:42:"join cluster test2 at 'test_tmg_890123765'";s:5:"errno";i:1064;s:5:"error";s:145:"cluster 'test2', no nodes available(test_tmg_890123765), error: test_tmg_890123765 invalid node, no AF_INET address found for: test_tmg_890123765";}i:28;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:47:"join cluster test2 at 'test_tmg_890123765:9312'";s:5:"errno";i:1064;s:5:"error";s:155:"cluster 'test2', no nodes available(test_tmg_890123765:9312), error: test_tmg_890123765:9312 invalid node, no AF_INET address found for: test_tmg_890123765";}i:29;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:52:"join cluster test2 at 'test_tmg_890123765:9312:test'";s:5:"errno";i:1064;s:5:"error";s:152:"cluster 'test2', no nodes available(test_tmg_890123765:9312:test), error: test_tmg_890123765:9312:test invalid node, unknown listen protocol type 'test'";}i:30;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:52:"join cluster test2 at 'test_tmg_890123765:9312:9325'";s:5:"errno";i:1064;s:5:"error";s:152:"cluster 'test2', no nodes available(test_tmg_890123765:9312:9325), error: test_tmg_890123765:9312:9325 invalid node, unknown listen protocol type '9325'";}}} \ No newline at end of file +a:1:{i:0;a:31:{i:0;a:2:{s:8:"sphinxql";s:26:"create table pq1 type='pq'";s:14:"total_affected";i:0;}i:1;a:2:{s:8:"sphinxql";s:26:"create table pq2 type='pq'";s:14:"total_affected";i:0;}i:2;a:2:{s:8:"sphinxql";s:26:"create table pq3 type='pq'";s:14:"total_affected";i:0;}i:3;a:2:{s:8:"sphinxql";s:19:"create cluster test";s:14:"total_affected";i:0;}i:4;a:2:{s:8:"sphinxql";s:26:"ALTER CLUSTER test ADD pq1";s:14:"total_affected";i:0;}i:5;a:2:{s:8:"sphinxql";s:45:"INSERT INTO test:pq1 (query) VALUES('test 1')";s:14:"total_affected";i:1;}i:6;a:2:{s:8:"sphinxql";s:45:"INSERT INTO test:pq1 (query) VALUES('test 2')";s:14:"total_affected";i:1;}i:7;a:2:{s:8:"sphinxql";s:45:"INSERT INTO test:pq1 (query) VALUES('test 3')";s:14:"total_affected";i:1;}i:8;a:3:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:43:"join cluster test '%addr_connect%' as nodes";s:14:"total_affected";i:0;}i:9;a:3:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:37:"join cluster test at '%addr_connect%'";s:14:"total_affected";i:0;}i:10;a:2:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:42:"/* stop-agent => stop=ok, return code=0 */";}i:11;a:2:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:42:"/* stop-agent => stop=ok, return code=0 */";}i:12;a:2:{s:8:"sphinxql";s:46:"INSERT INTO test:pq1 (query) VALUES('test 11')";s:14:"total_affected";i:1;}i:13;a:2:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:46:"/* start-agent => ; start=ok, return code=0 */";}i:14;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:48:"debug wait test like 'state' option 'timeout'=30";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:4:"name";s:5:"state";s:5:"Value";s:6:"synced";}}}i:15;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:22:"SELECT * FROM test:pq1";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:4:{s:2:"id";s:13:"1677721600002";s:5:"query";s:6:"test 2";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:1;a:4:{s:2:"id";s:13:"1677721600003";s:5:"query";s:6:"test 3";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:2;a:4:{s:2:"id";s:13:"1677721600004";s:5:"query";s:7:"test 11";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:3;a:4:{s:2:"id";s:13:"1677721600001";s:5:"query";s:6:"test 1";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}}}i:16;a:2:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:46:"/* start-agent => ; start=ok, return code=0 */";}i:17;a:4:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:48:"debug wait test like 'state' option 'timeout'=30";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:4:"name";s:5:"state";s:5:"Value";s:6:"synced";}}}i:18;a:4:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:22:"SELECT * FROM test:pq1";s:10:"total_rows";i:4;s:4:"rows";a:4:{i:0;a:4:{s:2:"id";s:13:"1677721600002";s:5:"query";s:6:"test 2";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:1;a:4:{s:2:"id";s:13:"1677721600003";s:5:"query";s:6:"test 3";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:2;a:4:{s:2:"id";s:13:"1677721600004";s:5:"query";s:7:"test 11";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:3;a:4:{s:2:"id";s:13:"1677721600001";s:5:"query";s:6:"test 1";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}}}i:19;a:3:{s:5:"agent";s:1:"1";s:8:"sphinxql";s:46:"INSERT INTO test:pq1 (query) VALUES('test 12')";s:14:"total_affected";i:1;}i:20;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:57:"debug wait test status 6 like 'state' option 'timeout'=30";s:10:"total_rows";i:1;s:4:"rows";a:1:{i:0;a:2:{s:4:"name";s:5:"state";s:5:"Value";s:1:"6";}}}i:21;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:22:"SELECT * FROM test:pq1";s:10:"total_rows";i:5;s:4:"rows";a:5:{i:0;a:4:{s:2:"id";s:13:"1677721600002";s:5:"query";s:6:"test 2";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:1;a:4:{s:2:"id";s:13:"1677721600003";s:5:"query";s:6:"test 3";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:2;a:4:{s:2:"id";s:13:"1677721600004";s:5:"query";s:7:"test 11";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:3;a:4:{s:2:"id";s:17:"72059271759527937";s:5:"query";s:7:"test 12";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}i:4;a:4:{s:2:"id";s:13:"1677721600001";s:5:"query";s:6:"test 1";s:4:"tags";s:0:"";s:7:"filters";s:0:"";}}}i:22;a:2:{s:8:"sphinxql";s:27:"ALTER CLUSTER test DROP pq1";s:14:"total_affected";i:0;}i:23;a:2:{s:8:"sphinxql";s:14:"drop table pq1";s:14:"total_affected";i:0;}i:24;a:2:{s:8:"sphinxql";s:14:"drop table pq2";s:14:"total_affected";i:0;}i:25;a:2:{s:8:"sphinxql";s:14:"drop table pq3";s:14:"total_affected";i:0;}i:26;a:2:{s:8:"sphinxql";s:19:"DELETE cluster test";s:14:"total_affected";i:0;}i:27;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:42:"join cluster test2 at 'test_tmg_890123765'";s:5:"errno";i:1064;s:5:"error";s:200:"cluster 'test2', no nodes available(test_tmg_890123765), error: test_tmg_890123765 invalid node, error: no AF_INET address found for: test_tmg_890123765, error -3: Temporary failure in name resolution";}i:28;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:47:"join cluster test2 at 'test_tmg_890123765:9312'";s:5:"errno";i:1064;s:5:"error";s:210:"cluster 'test2', no nodes available(test_tmg_890123765:9312), error: test_tmg_890123765:9312 invalid node, error: no AF_INET address found for: test_tmg_890123765, error -3: Temporary failure in name resolution";}i:29;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:52:"join cluster test2 at 'test_tmg_890123765:9312:test'";s:5:"errno";i:1064;s:5:"error";s:159:"cluster 'test2', no nodes available(test_tmg_890123765:9312:test), error: test_tmg_890123765:9312:test invalid node, error: unknown listen protocol type 'test'";}i:30;a:4:{s:5:"agent";s:1:"2";s:8:"sphinxql";s:52:"join cluster test2 at 'test_tmg_890123765:9312:9325'";s:5:"errno";i:1064;s:5:"error";s:159:"cluster 'test2', no nodes available(test_tmg_890123765:9312:9325), error: test_tmg_890123765:9312:9325 invalid node, error: unknown listen protocol type '9325'";}}} \ No newline at end of file