Make installer handle comments and utf8mb4 downgrade in sql queries from files

administrator/components/com_admin/sql/updates/mysql/3.4.0-2014-09-16.sql

@@ -3,4 +3,4 @@ ALTER TABLE `#__redirect_links` ADD COLUMN `header` smallint(3) NOT NULL DEFAULT
 -- The following statement has to be disabled because it conflicts with
 -- a later change added with Joomla! 3.5.0 for long URLs in this table
 --
---ALTER TABLE `#__redirect_links` MODIFY `new_url` varchar(255);
+-- ALTER TABLE `#__redirect_links` MODIFY `new_url` varchar(255);

administrator/components/com_admin/sql/updates/mysql/3.5.0-2016-02-26.sql

@@ -9,6 +9,6 @@
 
 CREATE TABLE IF NOT EXISTS `#__utf8_conversion` (
   `converted` tinyint(4) NOT NULL DEFAULT 0
-) ENGINE=InnoDB DEFAULT CHARSET=utf8 DEFAULT COLLATE=utf8_unicode_ci;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_unicode_ci;
 
 INSERT INTO `#__utf8_conversion` (`converted`) VALUES (0);

administrator/components/com_admin/sql/updates/postgresql/3.4.0-2014-09-16.sql

@@ -3,4 +3,4 @@ ALTER TABLE "#__redirect_links" ADD COLUMN "header" INTEGER DEFAULT 301 NOT NULL
 -- The following statement has to be disabled because it conflicts with
 -- a later change added with Joomla! 3.5.0 for long URLs in this table
 --
---ALTER TABLE "#__redirect_links" ALTER COLUMN "new_url" DROP NOT NULL;
+-- ALTER TABLE "#__redirect_links" ALTER COLUMN "new_url" DROP NOT NULL;

administrator/components/com_admin/sql/updates/sqlazure/3.4.0-2014-09-16.sql

@@ -3,4 +3,4 @@ ALTER TABLE [#__redirect_links] ADD [header] [smallint] NOT NULL DEFAULT 301;
 -- The following statement has to be disabled because it conflicts with
 -- a later change added with Joomla! 3.5.0 for long URLs in this table
 --
---ALTER TABLE [#__redirect_links] ALTER COLUMN [new_url] [nvarchar](255) NULL;
+-- ALTER TABLE [#__redirect_links] ALTER COLUMN [new_url] [nvarchar](255) NULL;

libraries/cms/installer/installer.php

@@ -854,20 +854,44 @@ public function parseQueries(SimpleXMLElement $element)
 			return 0;
 		}
 
+		$dbDriver = strtolower($db->name);
+
+		if ($dbDriver == 'mysql' || $dbDriver == 'mysqli' || $dbDriver == 'pdomysql')
+		{
+			$doUtf8mb4ToUtf8 = !$this->serverClaimsUtf8mb4Support($dbDriver);
+		}
+		else
+		{
+			$doUtf8mb4ToUtf8 = false;
+		}
+
+		$update_count = 0;
+
 		// Process each query in the $queries array (children of $tagName).
 		foreach ($queries as $query)
 		{
-			$db->setQuery($query->data());
-
-			if (!$db->execute())
+			if ($trimmedQuery = $this->trimQuery($query->data()))
 			{
-				JLog::add(JText::sprintf('JLIB_INSTALLER_ERROR_SQL_ERROR', $db->stderr(true)), JLog::WARNING, 'jerror');
+				// If we don't have UTF-8 Multibyte support we'll have to convert queries to plain UTF-8
+				if ($doUtf8mb4ToUtf8)
+				{
+					$trimmedQuery = $this->convertUtf8mb4QueryToUtf8($trimmedQuery);
+				}
 
-				return false;
+				$db->setQuery($trimmedQuery);
+
+				if (!$db->execute())
+				{
+					JLog::add(JText::sprintf('JLIB_INSTALLER_ERROR_SQL_ERROR', $db->stderr(true)), JLog::WARNING, 'jerror');
+
+					return false;
+				}
+
+				$update_count++;
 			}
 		}
 
-		return (int) count($queries);
+		return $update_count;
 	}
 
 	/**
@@ -891,11 +915,20 @@ public function parseSQLFiles($element)
 		$db = & $this->_db;
 		$dbDriver = strtolower($db->name);
 
+		$doUtf8mb4ToUtf8 = !$this->serverClaimsUtf8mb4Support($dbDriver);
+
 		if ($dbDriver == 'mysqli' || $dbDriver == 'pdomysql')
 		{
 			$dbDriver = 'mysql';
 		}
 
+		if ($dbDriver != 'mysql')
+		{
+			$doUtf8mb4ToUtf8 = false;
+		}
+
+		$update_count = 0;
+
 		// Get the name of the sql file to process
 		foreach ($element->children() as $file)
 		{
@@ -941,41 +974,30 @@ public function parseSQLFiles($element)
 				// Process each query in the $queries array (split out of sql file).
 				foreach ($queries as $query)
 				{
-					$query = trim($query);
-
-					if ($query != '' && $query{0} != '#')
+					if ($trimmedQuery = $this->trimQuery($query))
 					{
-						/**
-						 * If we don't have UTF-8 Multibyte support we'll have to convert queries to plain UTF-8
-						 *
-						 * Note: the JDatabaseDriver::convertUtf8mb4QueryToUtf8 performs the conversion ONLY when
-						 * necessary, so there's no need to check the conditions in JInstaller.
-						 */
-						$query = $db->convertUtf8mb4QueryToUtf8($query);
-
-						/**
-						 * This is a query which was supposed to convert tables to utf8mb4 charset but the server doesn't
-						 * support utf8mb4. Therefore we don't have to run it, it has no effect and it's a mere waste of time.
-						 */
-						if (!$db->hasUTF8mb4Support() && stristr($query, 'CONVERT TO CHARACTER SET utf8 '))
+						// If we don't have UTF-8 Multibyte support we'll have to convert queries to plain UTF-8
+						if ($doUtf8mb4ToUtf8)
 						{
-							continue;
+							$trimmedQuery = $this->convertUtf8mb4QueryToUtf8($trimmedQuery);
 						}
 
-						$db->setQuery($query);
+						$db->setQuery($trimmedQuery);
 
 						if (!$db->execute())
 						{
 							JLog::add(JText::sprintf('JLIB_INSTALLER_ERROR_SQL_ERROR', $db->stderr(true)), JLog::WARNING, 'jerror');
 
 							return false;
 						}
+
+						$update_count++;
 					}
 				}
 			}
 		}
 
-		return (int) count($queries);
+		return $update_count;
 	}
 
 	/**
@@ -1071,11 +1093,18 @@ public function parseSchemaUpdates(SimpleXMLElement $schema, $eid)
 			{
 				$dbDriver = strtolower($db->name);
 
+				$doUtf8mb4ToUtf8 = !$this->serverClaimsUtf8mb4Support($dbDriver);
+
 				if ($dbDriver == 'mysqli' || $dbDriver == 'pdomysql')
 				{
 					$dbDriver = 'mysql';
 				}
 
+				if ($dbDriver != 'mysql')
+				{
+					$doUtf8mb4ToUtf8 = false;
+				}
+
 				$schemapath = '';
 
 				foreach ($schemapaths as $entry)
@@ -1146,11 +1175,15 @@ public function parseSchemaUpdates(SimpleXMLElement $schema, $eid)
 							// Process each query in the $queries array (split out of sql file).
 							foreach ($queries as $query)
 							{
-								$query = trim($query);
-
-								if ($query != '' && $query{0} != '#')
+								if ($trimmedQuery = $this->trimQuery($query))
 								{
-									$db->setQuery($query);
+									// If we don't have UTF-8 Multibyte support we'll have to convert queries to plain UTF-8
+									if ($doUtf8mb4ToUtf8)
+									{
+										$trimmedQuery = $this->convertUtf8mb4QueryToUtf8($trimmedQuery);
+									}
+
+									$db->setQuery($trimmedQuery);
 
 									if (!$db->execute())
 									{
@@ -1160,7 +1193,7 @@ public function parseSchemaUpdates(SimpleXMLElement $schema, $eid)
 									}
 									else
 									{
-										$queryString = (string) $query;
+										$queryString = (string) $trimmedQuery;
 										$queryString = str_replace(array("\r", "\n"), array('', ' '), substr($queryString, 0, 80));
 										JLog::add(JText::sprintf('JLIB_INSTALLER_UPDATE_LOG_QUERY', $file, $queryString), JLog::INFO, 'Update');
 									}
@@ -2371,4 +2404,116 @@ public function loadAllAdapters($options = array())
 	{
 		$this->getAdapters($options);
 	}
+
+	/**
+	 * Does the database server claim to have support for UTF-8 Multibyte (utf8mb4) collation?
+	 * 
+	 * This is a modified version of the function in JDatabase::serverClaimsUtf8mb4Support() - it is
+	 * duplicated here for people upgrading from a version lower than 3.5.0 through extension manager
+	 * which will still have the old database driver loaded at this point.
+	 *
+	 * @param   string  $format  The type of database connection.
+	 *
+	 * @return  boolean
+	 *
+	 * @since   3.5.0
+	 */
+	private function serverClaimsUtf8mb4Support($format)
+	{
+		$db = JFactory::getDbo();
+
+		switch ($format)
+		{
+			case 'mysql':
+				$client_version = mysql_get_client_info();
+				$server_version = $db->getVersion();
+				break;
+			case 'mysqli':
+				$client_version = mysqli_get_client_info();
+				$server_version = $db->getVersion();
+				break;
+			case 'pdomysql':
+				$client_version = $db->getOption(PDO::ATTR_CLIENT_VERSION);
+				$server_version = $db->getOption(PDO::ATTR_SERVER_VERSION);
+				break;
+			default:
+				$client_version = false;
+				$server_version = false;
+		}
+
+		if ($client_version && version_compare($server_version, '5.5.3', '>='))
+		{
+			if (strpos($client_version, 'mysqlnd') !== false)
+			{
+				$client_version = preg_replace('/^\D+([\d.]+).*/', '$1', $client_version);
+
+				return version_compare($client_version, '5.0.9', '>=');
+			}
+			else
+			{
+				return version_compare($client_version, '5.5.3', '>=');
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	 * Downgrade a CREATE TABLE or ALTER TABLE query from utf8mb4 (UTF-8 Multibyte) to plain utf8. Used when the server
+	 * doesn't support UTF-8 Multibyte.
+	 *
+	 * This is a modified version of the function in JDatabase::convertUtf8mb4QueryToUtf8() - it is duplicated here for
+	 * people upgrading from a version lower than 3.5.0 through installer which will still have the old database
+	 * driver loaded at this point. This is missing the check for utf8mb4 in JDatabaseDriver we make this check in the
+	 * updater elsewhere.
+	 *
+	 * @param   string  $query  The query to convert
+	 *
+	 * @return  string  The converted query
+	 */
+	private function convertUtf8mb4QueryToUtf8($query)
+	{
+		// If it's not an ALTER TABLE or CREATE TABLE command there's nothing to convert
+		$beginningOfQuery = substr($query, 0, 12);
+		$beginningOfQuery = strtoupper($beginningOfQuery);
+
+		if (!in_array($beginningOfQuery, array('ALTER TABLE ', 'CREATE TABLE')))
+		{
+			return $query;
+		}
+
+		// Replace utf8mb4 with utf8
+		return str_replace('utf8mb4', 'utf8', $query);
+	}
+
+	/**
+	 * Trim comment and blank lines out of a query string
+	 *
+	 * @param   string  $query  query string to be trimmed
+	 *
+	 * @return  string  String with leading comment lines removed
+	 *
+	 * @since   3.5
+	 */
+	private function trimQuery($query)
+	{
+		$query = trim($query);
+
+		while (substr($query, 0, 1) == '#' || substr($query, 0, 2) == '--' || substr($query, 0, 2) == '/*')
+		{
+			$endChars = (substr($query, 0, 1) == '#' || substr($query, 0, 2) == '--') ? "\n" : "*/";
+
+			if ($position = strpos($query, $endChars))
+			{
+				$query = trim(substr($query, $position + strlen($endChars)));
+			}
+			else
+			{
+				// If no newline, the rest of the file is a comment, so return an empty string.
+				return '';
+			}
+		}
+
+		return trim($query);
+	}
 }