From a327687af626c13b29965019d892ad59b9ac9468 Mon Sep 17 00:00:00 2001
From: yossipapi <yossi.papiashvili@kaltura.com>
Date: Thu, 25 Jul 2019 16:07:29 +0300
Subject: [PATCH 1/2] Change populateFromLog to load shareded index config from
 db config file

---
 .../sphinx_search/scripts/populateFromLog.php  | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/plugins/search/providers/sphinx_search/scripts/populateFromLog.php b/plugins/search/providers/sphinx_search/scripts/populateFromLog.php
index 54d8c44cadf..17dc339238f 100644
--- a/plugins/search/providers/sphinx_search/scripts/populateFromLog.php
+++ b/plugins/search/providers/sphinx_search/scripts/populateFromLog.php
@@ -59,6 +59,12 @@ function filter($i) {
 DbManager::setConfig($dbConf);
 DbManager::initialize();
 
+$splitIndexSettings = null;
+if(isset($dbConf['sphinx_split_index']) && $dbConf['sphinx_split_index']['enabled'] == true)
+{
+	$splitIndexSettings = $dbConf['sphinx_split_index'];
+}
+
 $limit = 1000; 	// The number of sphinxLog records we want to query
 $gap = 500;	// The gap from 'getLastLogId' we want to query
 
@@ -68,7 +74,8 @@ function filter($i) {
 $lastLogs = array();
 $handledRecords = array();
 
-foreach($serverLastLogs as $serverLastLog) {
+foreach($serverLastLogs as $serverLastLog)
+{
 	$lastLogs[$serverLastLog->getDc()] = $serverLastLog;
 	$handledRecords[$serverLastLog->getDc()] = array();
 }
@@ -105,7 +112,11 @@ function filter($i) {
 		$executedServerId = $sphinxLog->getExecutedServerId();
 		$sphinxLogId = $sphinxLog->getId();
 		$sphinxLogIndexName = $sphinxLog->getIndexName();
-		
+		if($isSharded && preg_match('~[0-9]~', $sphinxLogIndexName) == 0 &&  $splitIndexSettings && isset($splitIndexSettings[$sphinxLog->getObjectType()]))
+		{
+			$splitFactor = $splitIndexSettings[$sphinxLog->getObjectType()];
+			$sphinxLogIndexName .= "_" . ($sphinxLog->getPartnerId()/$splitFactor)%$splitFactor;
+		}
 		
 		if($isSharded && $sphinxLogIndexName && !in_array($sphinxLogIndexName, $sphinxRtTables))
 		{
@@ -141,8 +152,7 @@ function filter($i) {
 				$sql = $sphinxLog->getSql();
 				if($isSharded && $sphinxLog->getObjectType() == "entry")
 				{
-					$shardedIndexName = $sphinxLog->getIndexName() ? $sphinxLog->getIndexName() : "kaltura_entry_" . ($sphinxLog->getPartnerId()/10)%10;
-					$sql = str_replace("kaltura_entry", $shardedIndexName, $sql);
+					$sql = str_replace("kaltura_entry", $sphinxLogIndexName, $sql);
 				}
 				
 				// sql update commands are created only via an external script for updating entries plays count

From c0c61f63bbef14f7d5fb9aaea0ca4c996f7ab1b8 Mon Sep 17 00:00:00 2001
From: yossipapi <yossi.papiashvili@kaltura.com>
Date: Sun, 28 Jul 2019 12:19:58 +0300
Subject: [PATCH 2/2] Make index name replcae generic

---
 .../providers/sphinx_search/scripts/populateFromLog.php   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/plugins/search/providers/sphinx_search/scripts/populateFromLog.php b/plugins/search/providers/sphinx_search/scripts/populateFromLog.php
index 17dc339238f..7c95399ce6c 100644
--- a/plugins/search/providers/sphinx_search/scripts/populateFromLog.php
+++ b/plugins/search/providers/sphinx_search/scripts/populateFromLog.php
@@ -73,6 +73,7 @@ function filter($i) {
 $serverLastLogs = SphinxLogServerPeer::retrieveByServer($sphinxServer, $sphinxReadConn);
 $lastLogs = array();
 $handledRecords = array();
+$sphinxRtTables = getSphinxRtTables($sphinxCon);
 
 foreach($serverLastLogs as $serverLastLog)
 {
@@ -95,7 +96,6 @@ function filter($i) {
 	try
 	{
 		$sphinxCon = DbManager::createSphinxConnection($sphinxServer,$sphinxPort);
-		$sphinxRtTables = getSphinxRtTables($sphinxCon);
 		KalturaLog::log("sphinxServer [$sphinxServer], running rt index names [" . implode(",", $sphinxRtTables) . "]");
 	}
 	catch(Exception $e)
@@ -115,7 +115,7 @@ function filter($i) {
 		if($isSharded && preg_match('~[0-9]~', $sphinxLogIndexName) == 0 &&  $splitIndexSettings && isset($splitIndexSettings[$sphinxLog->getObjectType()]))
 		{
 			$splitFactor = $splitIndexSettings[$sphinxLog->getObjectType()];
-			$sphinxLogIndexName .= "_" . ($sphinxLog->getPartnerId()/$splitFactor)%$splitFactor;
+			$sphinxLogIndexName = $sphinxLogIndexName . "_" . ($sphinxLog->getPartnerId()/$splitFactor)%$splitFactor;
 		}
 		
 		if($isSharded && $sphinxLogIndexName && !in_array($sphinxLogIndexName, $sphinxRtTables))
@@ -150,9 +150,9 @@ function filter($i) {
 			else
 			{
 				$sql = $sphinxLog->getSql();
-				if($isSharded && $sphinxLog->getObjectType() == "entry")
+				if($isSharded)
 				{
-					$sql = str_replace("kaltura_entry", $sphinxLogIndexName, $sql);
+					$sql = preg_replace('/replace into (kaltura_.*?) /', "replace into $sphinxLogIndexName ", $sql);
 				}
 				
 				// sql update commands are created only via an external script for updating entries plays count