From d3a0103bacedfecd3adf78f2ef29f6120a696ca2 Mon Sep 17 00:00:00 2001 From: yossipapi Date: Wed, 28 Feb 2024 12:22:35 +0200 Subject: [PATCH 1/4] FOUN-1871: Support mysql connection over tls based on config --- alpha/apps/kaltura/lib/db/KalturaPDO.php | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/alpha/apps/kaltura/lib/db/KalturaPDO.php b/alpha/apps/kaltura/lib/db/KalturaPDO.php index bf8aee7ea8f..b5c8ea239f4 100644 --- a/alpha/apps/kaltura/lib/db/KalturaPDO.php +++ b/alpha/apps/kaltura/lib/db/KalturaPDO.php @@ -16,10 +16,14 @@ class KalturaPDO extends PropelPDO */ const KALTURA_ATTR_NO_TRANSACTION = 'noTransaction'; + const KALTURA_ATTR_SSL_CA = 'sslCa'; + + const KALTURA_ATTR_SSL_VERIFY_SERVER_CERT = 'verifyServerCert'; + /** * Sets the number of retries of doSave() */ - const SAVE_MAX_RETRIES = 4; + const SAVE_MAX_RETRIES = 4; protected static $comment = null; protected $kalturaOptions = array(); @@ -39,6 +43,16 @@ public function __construct($dsn, $username = null, $password = null, $driver_op $this->kalturaOptions = DbManager::getKalturaConfig($this->connectionName); } + if(isset($this->kalturaOptions[KalturaPDO::KALTURA_ATTR_SSL_CA])) + { + $driver_options[PDO::MYSQL_ATTR_SSL_CA] = $this->getKalturaOption(KalturaPDO::KALTURA_ATTR_SSL_CA); + } + + if(isset($this->kalturaOptions[KalturaPDO::KALTURA_ATTR_SSL_VERIFY_SERVER_CERT])) + { + $driver_options[PDO::MYSQL_ATTR_SSL_VERIFY_SERVER_CERT] = $this->getKalturaOption(KalturaPDO::KALTURA_ATTR_SSL_VERIFY_SERVER_CERT); + } + list($mysql, $connection) = explode(':', $dsn); $arguments = explode(';', $connection); foreach($arguments as $argument) From 869349f223ced9964dfad4e7d512302c86366229 Mon Sep 17 00:00:00 2001 From: yossipapi Date: Tue, 16 Apr 2024 11:07:12 +0300 Subject: [PATCH 2/4] FOUN-1871: Support parsing user and password from sphinx conn params if defined --- alpha/apps/kaltura/lib/db/DbManager.php | 11 +++++++---- .../sphinx_search/scripts/populateFromLog.php | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/alpha/apps/kaltura/lib/db/DbManager.php b/alpha/apps/kaltura/lib/db/DbManager.php index c415c87f2d9..0996530324c 100644 --- a/alpha/apps/kaltura/lib/db/DbManager.php +++ b/alpha/apps/kaltura/lib/db/DbManager.php @@ -373,12 +373,15 @@ protected static function getPreferredSphinxIndexByWeight($hostToLag, $hostToInd private static function getSphinxConnectionInternal($key, $connectTimeout, $indexName) { - if(!isset(self::$config['datasources'][$key]['connection']['dsn'])) + $conparams = self::$config['datasources'][$key]['connection']; + if(!isset($conparams['dsn'])) throw new Exception("DB Config [$key] not found"); - - $dataSource = self::$config['datasources'][$key]['connection']['dsn']; + + $dataSource = $conparams['dsn']; + $user = isset($conparams['user']) ? $conparams['user'] : null; + $password = isset($conparams['password']) ? $conparams['password'] : null; self::$sphinxConnection[$indexName] = - new KalturaPDO($dataSource, null, null, array(PDO::ATTR_TIMEOUT => $connectTimeout, KalturaPDO::KALTURA_ATTR_NAME => $key), $key); + new KalturaPDO($dataSource, $user, $password, array(PDO::ATTR_TIMEOUT => $connectTimeout, KalturaPDO::KALTURA_ATTR_NAME => $key), $key); self::$sphinxConnection[$indexName]->setCommentsEnabled(false); return self::$sphinxConnection[$indexName]; diff --git a/plugins/search/providers/sphinx_search/scripts/populateFromLog.php b/plugins/search/providers/sphinx_search/scripts/populateFromLog.php index 19fecb9ee90..2854821c2aa 100644 --- a/plugins/search/providers/sphinx_search/scripts/populateFromLog.php +++ b/plugins/search/providers/sphinx_search/scripts/populateFromLog.php @@ -102,6 +102,7 @@ function filter($i) { $sphinxCon = null; try { + //TODO - need to find a way to pass user and pass when working with tls connection (Do not merge before handling this) $sphinxCon = DbManager::createSphinxConnection($sphinxServer,$sphinxPort); if(!count($sphinxRtTables)) { From 9dbb581515a4b34bec077a86db8f8328bca6b077 Mon Sep 17 00:00:00 2001 From: yossipapi Date: Wed, 17 Apr 2024 16:22:17 +0300 Subject: [PATCH 3/4] Support sphinx tls connection in populate from log --- alpha/apps/kaltura/lib/db/DbManager.php | 4 +- .../sphinx_search/scripts/populateFromLog.php | 92 +++++++++++++++++-- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/alpha/apps/kaltura/lib/db/DbManager.php b/alpha/apps/kaltura/lib/db/DbManager.php index 0996530324c..6edf54b5c5a 100644 --- a/alpha/apps/kaltura/lib/db/DbManager.php +++ b/alpha/apps/kaltura/lib/db/DbManager.php @@ -150,13 +150,13 @@ public static function shutdown() /** * @return KalturaPDO */ - public static function createSphinxConnection($sphinxServer, $port = 9312) + public static function createSphinxConnection($sphinxServer, $port = 9312, $userName = null, $password = null, $driver_options = array(), $config_key = null) { $dsn = "mysql:host=$sphinxServer;port=$port;"; try { - $con = new KalturaPDO($dsn); + $con = new KalturaPDO($dsn, $userName, $password, $driver_options, $config_key); $con->setCommentsEnabled(false); return $con; } diff --git a/plugins/search/providers/sphinx_search/scripts/populateFromLog.php b/plugins/search/providers/sphinx_search/scripts/populateFromLog.php index 2854821c2aa..d3189456979 100644 --- a/plugins/search/providers/sphinx_search/scripts/populateFromLog.php +++ b/plugins/search/providers/sphinx_search/scripts/populateFromLog.php @@ -70,13 +70,14 @@ function filter($i) { $splitIndexSettings = $dbConf['sphinx_split_index']; } -$limit = 1000; // The number of sphinxLog records we want to query $gap = 500; // The gap from 'getLastLogId' we want to query +$limit = 1000; // The number of sphinxLog records we want to query $maxIndexHistory = 2000; //The maximum array size to save unique object ids update and their sphinx log id $sphinxReadConn = myDbHelper::getConnection(myDbHelper::DB_HELPER_CONN_SPHINX_LOG_READ); - $serverLastLogs = SphinxLogServerPeer::retrieveByServer($sphinxServer, $sphinxReadConn); +list($sphinxUser, $sphinxPassword, $dataSourceKey) = getSphinxConnParams($sphinxServer, $dbConf); + $lastLogs = array(); $handledRecords = array(); $sphinxRtTables = array(); @@ -102,8 +103,8 @@ function filter($i) { $sphinxCon = null; try { - //TODO - need to find a way to pass user and pass when working with tls connection (Do not merge before handling this) - $sphinxCon = DbManager::createSphinxConnection($sphinxServer,$sphinxPort); + $sphinxCon = DbManager::createSphinxConnection($sphinxServer, $sphinxPort, $sphinxUser, $sphinxPassword, + array(KalturaPDO::KALTURA_ATTR_NAME => $dataSourceKey), $dataSourceKey); if(!count($sphinxRtTables)) { $sphinxRtTables = getSphinxRtTables($sphinxCon); @@ -116,7 +117,7 @@ function filter($i) { sleep(5); continue; } - + foreach($sphinxLogs as $sphinxLog) { /* @var $sphinxLog SphinxLog */ @@ -152,7 +153,7 @@ function filter($i) { $handledRecords[$dc][] = $sphinxLogId; KalturaLog::log("Sphinx log id $sphinxLogId dc [$dc] executed server id [$executedServerId] Memory: [" . memory_get_usage() . "]"); - + try { $objectId = $sphinxLog->getObjectId(); @@ -221,7 +222,7 @@ function filter($i) { } unset($sphinxCon); - + SphinxLogPeer::clearInstancePool(); } @@ -243,3 +244,80 @@ function getSphinxRtTables($sphinxCon) return $sphinxRtTables; } + +function getSphinxConnParams($sphinxServer, $dbConf) +{ + $sphinxDataSources = isset($dbConf['sphinx_datasources']['datasources']) ? + $dbConf['sphinx_datasources']['datasources'] : + array(DbManager::DB_CONFIG_SPHINX); + + + $sphinxServerIps = array(); + $sphinxServerDnsRecords = dns_get_record($sphinxServer); + foreach ($sphinxServerDnsRecords as $sphinxServerDnsRecord) + { + if(trim($sphinxServerDnsRecord['ip']) == '') + { + continue; + } + + $sphinxServerIps[] = $sphinxServerDnsRecord['ip']; + } + + + foreach ($sphinxDataSources as $sphinxDataSource) + { + if(!isset($dbConf['datasources'][$sphinxDataSource])) + { + KalturaLog::log("Sphinx source [$sphinxDataSource] not found in datasource list"); + continue; + } + + $confParams = $dbConf['datasources'][$sphinxDataSource]['connection']; + $sphinxDataSourceDsn = $confParams['dsn']; + list($mysql, $connectionStr) = explode(':', $sphinxDataSourceDsn); + + $host = null; + $connectionArguments = explode(';', $connectionStr); + foreach($connectionArguments as $connectionArgument) + { + if(trim($connectionArgument) == '') + { + continue; + } + + list($argumentName, $argumentValue) = explode('=', $connectionArgument); + if(strtolower($argumentName) == 'host') + { + $host = $argumentValue; + } + } + + if(!$host) + { + KalturaLog::log("Failed to find host for sphinx source [$sphinxDataSource]"); + continue; + } + + $hostIps = array(); + $hostRecords = dns_get_record($host); + foreach ($hostRecords as $hostRecord) + { + if(trim($hostRecord['ip']) == '') + { + continue; + } + + $hostIps[] = $hostRecord['ip']; + } + + if(count(array_intersect($sphinxServerIps, $hostIps))) + { + return array($confParams['user'], $confParams['password'], $sphinxDataSource); + } + + + } + + return array(null, null, null); +} From 530b64e6598786b4f116ea1842eedd49201ad1d1 Mon Sep 17 00:00:00 2001 From: yossipapi Date: Mon, 6 May 2024 18:39:35 +0300 Subject: [PATCH 4/4] Support tls connection in additional sphinx flows --- alpha/scripts/sphinxCompatCheck.php | 16 +++++++++++----- .../sphinx_search/scripts/populateFromLog.php | 2 -- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/alpha/scripts/sphinxCompatCheck.php b/alpha/scripts/sphinxCompatCheck.php index c94fba038f3..bf545e49de1 100644 --- a/alpha/scripts/sphinxCompatCheck.php +++ b/alpha/scripts/sphinxCompatCheck.php @@ -1,12 +1,18 @@ \n"); + die("Usage:\n\tphp sphinxCompatCheck \n"); -$conn1 = createSphinxConnection($argv[1], $argv[2]); -$conn2 = createSphinxConnection($argv[3], $argv[4]); +$conn1 = createSphinxConnection($argv[1], $argv[2], $argv[5], $argv[6], $argv[7]); +$conn2 = createSphinxConnection($argv[3], $argv[4], $argv[5], $argv[6], $argv[7]); $strictMode = false; $serverTime1 = 0; diff --git a/plugins/search/providers/sphinx_search/scripts/populateFromLog.php b/plugins/search/providers/sphinx_search/scripts/populateFromLog.php index d3189456979..c446491cba8 100644 --- a/plugins/search/providers/sphinx_search/scripts/populateFromLog.php +++ b/plugins/search/providers/sphinx_search/scripts/populateFromLog.php @@ -315,8 +315,6 @@ function getSphinxConnParams($sphinxServer, $dbConf) { return array($confParams['user'], $confParams['password'], $sphinxDataSource); } - - } return array(null, null, null);