Skip to content

Commit

Permalink
Adding Unicode Supplementary Characters support in Joomla!
Browse files Browse the repository at this point in the history 
Fixes #7173
  • Loading branch information
@wilsonge
Nicholas K. Dionysopoulos authored and wilsonge committed Jul 10, 2015
1 parent d93dce0 commit 04f09d5
Show file tree
Hide file tree
Showing 20 changed files with 968 additions and 92 deletions.
1 change: 1 addition & 0 deletions administrator/components/com_admin/models/sysinfo.php
View file
Expand Up @@ -141,6 +141,7 @@ public function &getInfo()
$this->info['php'] = php_uname();
$this->info['dbversion'] = $db->getVersion();
$this->info['dbcollation'] = $db->getCollation();
$this->info['dbconnectioncollation'] = $db->getConnectionCollation();
$this->info['phpversion'] = phpversion();
$this->info['server'] = isset($_SERVER['SERVER_SOFTWARE']) ? $_SERVER['SERVER_SOFTWARE'] : getenv('SERVER_SOFTWARE');
$this->info['sapi_name'] = php_sapi_name();
Expand Down
89 changes: 89 additions & 0 deletions administrator/components/com_admin/sql/updates/mysql/3.5.0-2015-07-01.sql
View file
@@ -0,0 +1,89 @@
-- WARNING: Do not rename this file with a different date. It MUST run before any other table updates when upgrading to Joomla! 3.5.0

-- Index and field changes to cater for UTF-8 Multibyte (utf8mb4)
ALTER TABLE `#__menu` DROP KEY `idx_client_id_parent_id_alias_language`, ADD UNIQUE KEY `idx_client_id_parent_id_alias_language` (`client_id`,`parent_id`,`alias`(191),`language`);

ALTER TABLE `#__redirect_links` DROP KEY `idx_link_old`, ADD UNIQUE KEY `idx_link_old` (`old_url`(191));

ALTER TABLE `#__session` MODIFY `session_id` varchar(191) NOT NULL DEFAULT '';

ALTER TABLE `#__user_keys` MODIFY `series` varchar(191) NOT NULL;

-- Convert utf8_bin collated columns to utf8mb4_bin collation
ALTER TABLE `#__banners` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '';
ALTER TABLE `#__categories` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '';
ALTER TABLE `#__contact_details` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '';
ALTER TABLE `#__content` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '';
ALTER TABLE `#__menu` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'The SEF alias of the menu item.';
ALTER TABLE `#__newsfeeds` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '';
ALTER TABLE `#__tags` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '';
ALTER TABLE `#__ucm_content` MODIFY `core_alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '';

-- Convert all tables to UTF-8 Multibyte (utf8mb4)
ALTER TABLE `#__assets` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__associations` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__banners` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__banner_clients` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__banner_tracks` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__categories` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__contact_details` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__content` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__content_frontpage` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__content_rating` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__content_types` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__contentitem_tag_map` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__core_log_searches` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__extensions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_filters` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_terms0` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_terms1` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_terms2` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_terms3` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_terms4` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_terms5` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_terms6` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_terms7` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_terms8` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_terms9` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_termsa` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_termsb` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_termsc` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_termsd` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_termse` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_links_termsf` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_taxonomy` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_taxonomy_map` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_terms` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_terms_common` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_tokens` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_tokens_aggregate` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__finder_types` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__languages` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__menu` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__menu_types` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__messages` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__messages_cfg` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__modules` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__modules_menu` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__newsfeeds` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__overrider` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__postinstall_messages` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__redirect_links` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__schemas` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__session` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__tags` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__template_styles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__ucm_base` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__ucm_content` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__ucm_history` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__updates` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__update_sites` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__update_sites_extensions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__usergroups` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__users` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__user_keys` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__user_notes` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__user_profiles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__user_usergroup_map` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
ALTER TABLE `#__viewlevels` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
8 changes: 8 additions & 0 deletions administrator/components/com_admin/views/sysinfo/tmpl/default_system.php
View file
Expand Up @@ -52,6 +52,14 @@
<?php echo $this->info['dbcollation']; ?>
</td>
</tr>
<tr>
<td>
<strong><?php echo JText::_('COM_ADMIN_DATABASE_CONNECTION_COLLATION'); ?></strong>
</td>
<td>
<?php echo $this->info['dbconnectioncollation']; ?>
</td>
</tr>
<tr>
<td>
<strong><?php echo JText::_('COM_ADMIN_PHP_VERSION'); ?></strong>
Expand Down
44 changes: 44 additions & 0 deletions administrator/components/com_installer/models/database.php
View file
Expand Up @@ -63,6 +63,9 @@ public function fix()
$installer = new JoomlaInstallerScript;
$installer->deleteUnexistingFiles();
$this->fixDefaultTextFilters();

// Finally, make sure the database is converted to utf8mb4 if supported by the server
$this->convertTablesToUtf8mb4();
}

/**
Expand Down Expand Up @@ -255,4 +258,45 @@ public function fixDefaultTextFilters()
}
}
}

public function convertTablesToUtf8mb4()
{
$db = JFactory::getDbo();

// If the database does not have UTF-8 Multibyte (utf8mb4) support we can't do much about it.
if (!$db->hasUTF8mb4Support())
{
return;
}

// Get the SQL file to convert the core tables. Yes, this is hardcoded because we have all sorts of index
// conversions and funky things we can't automate in core tables without an actual SQL file.
$serverType = $db->getServerType();
$fileName = JPATH_ADMINISTRATOR . "/components/com_admin/sql/updates/$serverType/3.5.0-2015-01-01.sql";

if (!is_file($fileName))
{
return;
}

$fileContents = @file_get_contents($fileName);
$queries = $db->splitSql($fileContents);

if (empty($queries))
{
return;
}

foreach ($queries as $query)
{
try
{
$db->setQuery($query)->execute();
}
catch (Exception $e)
{
// If the query fails we will go on. It probably means we've already done this conversion.
}
}
}
}
1 change: 1 addition & 0 deletions administrator/language/en-GB/en-GB.com_admin.ini
View file
Expand Up @@ -9,6 +9,7 @@ COM_ADMIN_CACHE_DIRECTORY="(Cache Directory)"
COM_ADMIN_CLEAR_RESULTS="Clear results"
COM_ADMIN_CONFIGURATION_FILE="Configuration File"
COM_ADMIN_DATABASE_COLLATION="Database Collation"
COM_ADMIN_DATABASE_CONNECTION_COLLATION="Database Connection Collation"
COM_ADMIN_DATABASE_VERSION="Database Version"
COM_ADMIN_DIRECTORY="Directory"
COM_ADMIN_DIRECTORY_PERMISSIONS="Directory Permissions"
Expand Down
17 changes: 17 additions & 0 deletions installation/model/database.php
View file
Expand Up @@ -896,6 +896,23 @@ public function populateDatabase($db, $schema)
// If the query isn't empty and is not a MySQL or PostgreSQL comment, execute it.
if (!empty($query) && ($query{0} != '#') && ($query{0} != '-'))
{
/**
* If we don't have UTF-8 Multibyte support we'll have to convert queries to plain UTF-8
*
* Note: the JDatabaseDriver::convertUtf8mb4QueryToUtf8 performs the conversion ONLY when
* necessary, so there's no need to check the conditions in JInstaller.
*/
$query = $db->convertUtf8mb4QueryToUtf8($query);

/**
* This is a query which was supposed to convert tables to utf8mb4 charset but the server doesn't
* support utf8mb4. Therefore we don't have to run it, it has no effect and it's a mere waste of time.
*/
if (!$db->hasUTF8mb4Support() && stristr($query, 'CONVERT TO CHARACTER SET utf8 '))
{
continue;
}

// Execute the query.
$db->setQuery($query);

Expand Down

0 comments on commit 04f09d5

Please sign in to comment.