MDL-48228 database: Make utf8mb4 the default character set for mysql.

Thanks to Jetha Chan for providing the initial patch that this is based on.
moodle · Mar 1, 2017 · 0bbefd8 · 0bbefd8
1 parent 0f59b6d
commit 0bbefd8
Show file tree

Hide file tree

Showing 8 changed files with 354 additions and 33 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -134,11 +134,13 @@ before_script:
           sed -i \
             -e "s%= 'pgsql'%= 'mysqli'%" \
             -e "s%= 'username'%= 'travis'%" \
+            -e "s%=> 'utf8mb4_unicode_ci'%=> 'utf8mb4_bin'%" \
             config.php;
 
           mysql -u root -e 'SET GLOBAL innodb_file_format=barracuda;' ;
           mysql -u root -e 'SET GLOBAL innodb_file_per_table=ON;' ;
-          mysql -e 'CREATE DATABASE travis_ci_test DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_bin;' ;
+          mysql -u root -e 'SET GLOBAL innodb_large_prefix=ON;' ;
+          mysql -e 'CREATE DATABASE travis_ci_test DEFAULT CHARACTER SET utf8mb4 DEFAULT COLLATE utf8mb4_bin;' ;
         fi
       fi
 

diff --git a/admin/cli/mysql_collation.php b/admin/cli/mysql_collation.php
@@ -64,6 +64,71 @@
         cli_error("Error: collation '$collation' is not available on this server!");
     }
 
+    $collationinfo = explode('_', $collation);
+    $charset = reset($collationinfo);
+
+    $engine = strtolower($DB->get_dbengine());
+
+    // Do checks for utf8mb4.
+    if (strpos($collation, 'utf8mb4') === 0) {
+        // Do we have the right engine?
+        if ($engine !== 'innodb' && $engine !== 'xtradb') {
+            cli_error("Error: '$collation' requires InnoDB or XtraDB set as the engine.");
+        }
+        // Are we using Barracuda?
+        if ($DB->get_row_format() != 'Barracuda') {
+            // Try setting it here.
+            try {
+                $DB->execute("SET GLOBAL innodb_file_format=Barracuda");
+            } catch (dml_exception $e) {
+                cli_error("Error: '$collation' requires the file format to be set to Barracuda.
+                        An attempt was made to change the format, but it failed. Please try doing this manually.");
+            }
+            echo "GLOBAL SETTING: innodb_file_format changed to Barracuda\n";
+        }
+        // Is one file per table being used?
+        if (!$DB->is_file_per_table_enabled()) {
+            try {
+                $DB->execute("SET GLOBAL innodb_file_per_table=1");
+            } catch (dml_exception $e) {
+                cli_error("Error: '$collation' requires the setting 'innodb_file_per_table' be set to 'ON'.
+                        An attempt was made to change the format, but it failed. Please try doing this manually.");
+            }
+            echo "GLOBAL SETTING: innodb_file_per_table changed to 1\n";
+        }
+        // Is large prefix set?
+        if (!$DB->is_large_prefix_enabled()) {
+            try {
+                $DB->execute("SET GLOBAL innodb_large_prefix=1");
+            } catch (dml_exception $e) {
+                cli_error("Error: '$collation' requires the setting 'innodb_large_prefix' be set to 'ON'.
+                        An attempt was made to change the format, but it failed. Please try doing this manually.");
+            }
+            echo "GLOBAL SETTING: innodb_large_prefix changed to 1\n";
+        }
+    }
+
+    $sql = "SHOW VARIABLES LIKE 'collation_database'";
+    if (!$dbcollation = $DB->get_record_sql($sql)) {
+        cli_error("Error: Could not access collation information on the database.");
+    }
+    $sql = "SHOW VARIABLES LIKE 'character_set_database'";
+    if (!$dbcharset = $DB->get_record_sql($sql)) {
+        cli_error("Error: Could not access character set information on the database.");
+    }
+    if ($dbcollation->value !== $collation || $dbcharset->value !== $charset) {
+        // Try to convert the DB.
+        echo "Converting database to '$collation' for $CFG->wwwroot:\n";
+        $sql = "ALTER DATABASE $CFG->dbname DEFAULT CHARACTER SET $charset DEFAULT COLLATE = $collation";
+        try {
+            $DB->change_database_structure($sql);
+        } catch (exception $e) {
+            cli_error("Error: Tried to alter the database with no success. Please try manually changing the database
+                    to the new collation and character set and then run this script again.");
+        }
+        echo "DATABASE CONVERTED\n";
+    }
+
     echo "Converting tables and columns to '$collation' for $CFG->wwwroot:\n";
     $prefix = $DB->get_prefix();
     $prefix = str_replace('_', '\\_', $prefix);
@@ -80,7 +145,7 @@
             $skipped++;
 
         } else {
-            $DB->change_database_structure("ALTER TABLE $table->name DEFAULT COLLATE = $collation");
+            $DB->change_database_structure("ALTER TABLE $table->name DEFAULT CHARACTER SET $charset DEFAULT COLLATE = $collation");
             echo "CONVERTED\n";
             $converted++;
         }
@@ -96,18 +161,32 @@
                 continue;
             }
 
+            // Check for utf8mb4 collation.
+            $rowformat = $DB->get_row_format_sql($engine, $collation);
+
             if ($column->type === 'tinytext' or $column->type === 'mediumtext' or $column->type === 'text' or $column->type === 'longtext') {
                 $notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
                 $default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : '';
                 // primary, unique and inc are not supported for texts
-                $sql = "ALTER TABLE $table->name MODIFY COLUMN $column->field $column->type COLLATE $collation $notnull $default";
+                $sql = "ALTER TABLE $table->name
+                        MODIFY COLUMN $column->field $column->type
+                        CHARACTER SET $charset
+                        COLLATE $collation $notnull $default";
                 $DB->change_database_structure($sql);
 
             } else if (strpos($column->type, 'varchar') === 0) {
                 $notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL';
                 $default = !is_null($column->default) ? "DEFAULT '$column->default'" : '';
-                // primary, unique and inc are not supported for texts
-                $sql = "ALTER TABLE $table->name MODIFY COLUMN $column->field $column->type COLLATE $collation $notnull $default";
+
+                if ($rowformat != '') {
+                    $sql = "ALTER TABLE $table->name $rowformat";
+                    $DB->change_database_structure($sql);
+                }
+
+                $sql = "ALTER TABLE $table->name
+                        MODIFY COLUMN $column->field $column->type
+                        CHARACTER SET $charset
+                        COLLATE $collation $notnull $default";
                 $DB->change_database_structure($sql);
             } else {
                 echo "ERROR (unknown column type: $column->type)\n";
@@ -180,7 +259,9 @@ function mysql_get_collations() {
     global $DB;
 
     $collations = array();
-    $sql = "SHOW COLLATION WHERE Collation LIKE 'utf8\_%' AND Charset = 'utf8'";
+    $sql = "SHOW COLLATION
+            WHERE Collation LIKE 'utf8\_%' AND Charset = 'utf8'
+               OR Collation LIKE 'utf8mb4\_%' AND Charset = 'utf8mb4'";
     $rs = $DB->get_recordset_sql($sql);
     foreach ($rs as $collation) {
         $collations[$collation->collation] = $collation->collation;

diff --git a/admin/environment.xml b/admin/environment.xml
@@ -1723,6 +1723,21 @@
           <ON_CHECK message="libcurlwarning" />
         </FEEDBACK>
       </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_file_format" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddbfileformat" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_file_per_table" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddbfilepertable" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_large_prefix" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddblargeprefix" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
     </CUSTOM_CHECKS>
   </MOODLE>
   <MOODLE version="3.2" requires="2.7">
@@ -1873,6 +1888,21 @@
           <ON_CHECK message="libcurlwarning" />
         </FEEDBACK>
       </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_file_format" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddbfileformat" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_file_per_table" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddbfilepertable" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
+      <CUSTOM_CHECK file="lib/upgradelib.php" function="check_mysql_large_prefix" level="required">
+        <FEEDBACK>
+          <ON_ERROR message="unsupporteddblargeprefix" />
+        </FEEDBACK>
+      </CUSTOM_CHECK>
     </CUSTOM_CHECKS>
   </MOODLE>
 </COMPATIBILITY_MATRIX>
diff --git a/config-dist.php b/config-dist.php
@@ -63,6 +63,13 @@
                                 // support advanced options on connection.
                                 // If you set those in the database then
                                 // the advanced settings will not be sent.
+    'dbcollation' => 'utf8mb4_unicode_ci', // MySQL has partial and full UTF-8
+                                // support. If you wish to use partial UTF-8
+                                // (three bytes) then set this option to
+                                // 'utf8_unicode_ci', otherwise this option
+                                // can be removed for MySQL (by default it will
+                                // use 'utf8mb4_unicode_ci'. This option should
+                                // be removed for all other databases.
 );
 
 

diff --git a/lang/en/admin.php b/lang/en/admin.php
@@ -596,6 +596,7 @@
 $string['http'] = 'HTTP';
 $string['httpsecurity'] = 'HTTP security';
 $string['hubs'] = 'Hubs';
+$string['incompleteunicodesupport'] = 'The current setup of MySQL or MariaDB is using \'utf8\'. This character set does not support four byte characters which include some emoji. Trying to use these characters will result in an error when updating a record, and any information being sent to the database will be lost. Please consider changing your settings to \'utf8mb4\'. See the documentation for full details.';
 $string['change'] = 'change';
 $string['checkboxno'] = 'No';
 $string['checkboxyes'] = 'Yes';
@@ -1133,6 +1134,9 @@
 $string['unoconvwarning'] = 'The version of unoconv you have installed is not supported. Moodle\'s assignment grading feature requires version 0.7 or higher.';
 $string['unsettheme'] = 'Unset theme';
 $string['unsupported'] = 'Unsupported';
+$string['unsupporteddbfileformat'] = 'Your database has tables using Antelope as the file format. Full UTF-8 support in MySQL and MariaDB requires the Barracuda file format. Please convert the tables to the Barracuda file format. See the documentation <a href="https://docs.moodle.org/en/cli">Administration via command line</a> for details of a tool for converting InnoDB tables to Barracuda.';
+$string['unsupporteddbfilepertable'] = 'For full support of UTF-8 both MySQL and MariaDB require you to change your MySQL setting \'innodb_file_per_table\' to \'ON\'. See the documentation for further details.';
+$string['unsupporteddblargeprefix'] = 'For full support of UTF-8 both MySQL and MariaDB require you to change your MySQL setting \'innodb_large_prefix\' to \'ON\'. See the documentation for further details.';
 $string['unsupporteddbstorageengine'] = 'The database storage engine being used is no longer supported.';
 $string['unsupporteddbtablerowformat'] = 'Your database has tables using Antelope as the file format. You are recommended to convert the tables to the Barracuda file format. See the documentation <a href="https://docs.moodle.org/en/cli">Administration via command line</a> for details of a tool for converting InnoDB tables to Barracuda.';
 $string['unsupportedphpversion7'] = 'PHP version 7 is not supported.';

diff --git a/lib/ddl/mysql_sql_generator.php b/lib/ddl/mysql_sql_generator.php
@@ -218,6 +218,9 @@ public function getCreateTableSQL($xmldb_table) {
             }
         }
 
+        $utf8mb4rowformat = $this->mdb->get_row_format_sql($engine, $collation);
+        $rowformat = ($utf8mb4rowformat == '') ? $rowformat : $utf8mb4rowformat;
+
         $sqlarr = parent::getCreateTableSQL($xmldb_table);
 
         // This is a very nasty hack that tries to use just one query per created table
@@ -238,7 +241,7 @@ public function getCreateTableSQL($xmldb_table) {
                     if (strpos($collation, 'utf8_') === 0) {
                         $sql .= "\n DEFAULT CHARACTER SET utf8";
                     }
-                    $sql .= "\n DEFAULT COLLATE = $collation";
+                    $sql .= "\n DEFAULT COLLATE = $collation ";
                 }
                 if ($rowformat) {
                     $sql .= $rowformat;
@@ -326,10 +329,13 @@ public function getAddFieldSQL($xmldb_table, $xmldb_field, $skip_type_clause = N
      * @return array of sql statements
      */
     public function getCreateTempTableSQL($xmldb_table) {
+        $engine = $this->mdb->get_dbengine();
         // Do we know collation?
         $collation = $this->mdb->get_dbcollation();
         $this->temptables->add_temptable($xmldb_table->getName());
 
+        $rowformat = $this->mdb->get_row_format_sql($engine, $collation);
+
         $sqlarr = parent::getCreateTableSQL($xmldb_table);
 
         // Let's inject the extra MySQL tweaks.
@@ -341,7 +347,7 @@ public function getCreateTempTableSQL($xmldb_table) {
                     if (strpos($collation, 'utf8_') === 0) {
                         $sqlarr[$i] .= " DEFAULT CHARACTER SET utf8";
                     }
-                    $sqlarr[$i] .= " DEFAULT COLLATE $collation";
+                    $sqlarr[$i] .= " DEFAULT COLLATE $collation $rowformat";
                 }
             }
         }