Skip to content

Commit

Permalink
Merge branch 'wip_MDL-46235_m27_innodbtext' of github.com:skodak/mood…
Browse files Browse the repository at this point in the history 
…le into MOODLE_27_STABLE
  • Loading branch information
@danpoltawski
danpoltawski committed Oct 14, 2014
2 parents 0cda7b5 + 0914be8 commit 7252078
Show file tree
Hide file tree
Showing 4 changed files with 478 additions and 0 deletions.
202 changes: 202 additions & 0 deletions admin/cli/mysql_compressed_rows.php
View file
@@ -0,0 +1,202 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.

/**
* MySQL table row compression tool tool.
*
* @package core
* @copyright 2014 Totara Learning Solutions Ltd {@link http://www.totaralms.com/}
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
* @author Petr Skoda <petr.skoda@totaralms.com>
*/

define('CLI_SCRIPT', true);

require(dirname(__FILE__).'/../../config.php');
require_once($CFG->libdir . '/clilib.php');

if ($DB->get_dbfamily() !== 'mysql') {
cli_error('This script is used for MySQL databases only.');
}

$engine = strtolower($DB->get_dbengine());
if ($engine !== 'innodb' and $engine !== 'xtradb') {
cli_error('This script is for MySQL servers using InnoDB or XtraDB engines only.');
}

list($options, $unrecognized) = cli_get_params(
array('help' => false, 'info' => false, 'list' => false, 'fix' => false, 'showsql' => false),
array('h' => 'help', 'i' => 'info', 'l' => 'list', 'f' => 'fix', 's' => 'showsql')
);

if ($unrecognized) {
$unrecognized = implode("\n ", $unrecognized);
cli_error(get_string('cliunknowoption', 'admin', $unrecognized));
}

$help =
"Script for detection of row size problems in MySQL InnoDB tables.
By default InnoDB storage table is using legacy Antelope file format
which has major restriction on database row size.
Use this script to detect and fix database tables with potentail data
overflow problems.
Options:
-i, --info Show database information
-l, --list List problematic tables
-f, --fix Attempt to fix all tables (requires SUPER privilege)
-s, --showsql Print SQL statements for fixing of tables
-h, --help Print out this help
Example:
\$ sudo -u www-data /usr/bin/php admin/cli/mysql_compressed_rows.php -l
";

/** @var mysql_sql_generator $generator */
$generator = $DB->get_manager()->generator;

$info = $DB->get_server_info();
$filepertable = $DB->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_per_table'");
$filepertable = $filepertable ? $filepertable->value : '';
$fileformat = $DB->get_record_sql("SHOW VARIABLES LIKE 'innodb_file_format'");
$fileformat = $fileformat ? $fileformat->value : '';
$prefix = $DB->get_prefix();
$database = $CFG->dbname;

if (!empty($options['info'])) {
echo "Database version: " . $info['description'] . "\n";
echo "Database name: $database\n";
echo "Database engine: " . $DB->get_dbengine() . "\n";
echo "innodb_file_per_table: $filepertable\n";
echo "innodb_file_format: $fileformat\n";

exit(0);

} else if (!empty($options['list'])) {
$problem = false;
foreach ($DB->get_tables(false) as $table) {
$columns = $DB->get_columns($table, false);
$size = $generator->guess_antolope_row_size($columns);
$format = $DB->get_row_format($table);
if ($size <= $generator::ANTELOPE_MAX_ROW_SIZE) {
continue;
}

echo str_pad($prefix . $table, 32, ' ', STR_PAD_RIGHT);
echo str_pad($format, 11, ' ', STR_PAD_RIGHT);

if ($format === 'Compact' or $format === 'Redundant') {
$problem = true;
echo " (needs fixing)\n";

} else if ($format !== 'Compressed' and $format !== 'Dynamic') {
echo " (unknown)\n";

} else {
echo "\n";
}
}

if ($problem) {
exit(1);
}
exit(0);

} else if (!empty($options['fix'])) {
$fixtables = array();
foreach ($DB->get_tables(false) as $table) {
$columns = $DB->get_columns($table, false);
$size = $generator->guess_antolope_row_size($columns);
$format = $DB->get_row_format($table);
if ($size <= $generator::ANTELOPE_MAX_ROW_SIZE) {
continue;
}
if ($format === 'Compact' or $format === 'Redundant') {
$fixtables[$table] = $table;
}
}

if (!$fixtables) {
echo "No changes necessary\n";
exit(0);
}

if ($filepertable !== 'ON') {
try {
$DB->execute("SET GLOBAL innodb_file_per_table=1");
} catch (dml_exception $e) {
echo "Cannot enable GLOBAL innodb_file_per_table setting, use --showsql option and execute the statements manually.";
throw $e;
}
}
if ($fileformat !== 'Barracuda') {
try {
$DB->execute("SET GLOBAL innodb_file_format=Barracuda");
} catch (dml_exception $e) {
echo "Cannot change GLOBAL innodb_file_format setting, use --showsql option and execute the statements manually.";
throw $e;
}
}

if (!$DB->is_compressed_row_format_supported(false)) {
echo "MySQL server is not compatible with compressed row format.";
exit(1);
}

foreach ($fixtables as $table) {
$DB->change_database_structure("ALTER TABLE {$prefix}$table ROW_FORMAT=Compressed");
echo str_pad($prefix . $table, 32, ' ', STR_PAD_RIGHT) . " ... Compressed\n";
}

exit(0);

} else if (!empty($options['showsql'])) {
$fixtables = array();

foreach ($DB->get_tables(false) as $table) {
$columns = $DB->get_columns($table, false);
$size = $generator->guess_antolope_row_size($columns);
$format = $DB->get_row_format($table);
if ($size <= $generator::ANTELOPE_MAX_ROW_SIZE) {
continue;
}
if ($format === 'Compact' or $format === 'Redundant') {
$fixtables[$table] = $table;
}
}
if (!$fixtables) {
echo "No changes necessary\n";
exit(0);
}

echo "Copy the following SQL statements and execute them using account with SUPER privilege:\n\n";
echo "USE $database;\n";
echo "SET SESSION sql_mode=STRICT_ALL_TABLES;\n";
echo "SET GLOBAL innodb_file_per_table=1;\n";
echo "SET GLOBAL innodb_file_format=Barracuda;\n";
foreach ($fixtables as $table) {
echo "ALTER TABLE {$prefix}$table ROW_FORMAT=Compressed;\n";
}
echo "\n";
exit(0);

} else {
echo $help;
die;
}


120 changes: 120 additions & 0 deletions lib/ddl/mysql_sql_generator.php
View file
Expand Up @@ -88,6 +88,9 @@ class mysql_sql_generator extends sql_generator {
/** @var string SQL sentence to rename one key 'TABLENAME', 'OLDKEYNAME' and 'NEWKEYNAME' are dynamically replaced.*/
public $rename_key_sql = null;

/** Maximum size of InnoDB row in Antelope file format */
const ANTELOPE_MAX_ROW_SIZE = 8126;

/**
* Reset a sequence to the id field of a table.
*
Expand All @@ -108,6 +111,78 @@ public function getResetSequenceSQL($table) {
return array("ALTER TABLE $this->prefix$tablename AUTO_INCREMENT = $value");
}

/**
* Calculate proximate row size when using InnoDB
* tables in Antelope row format.
*
* Note: the returned value is a bit higher to compensate for
* errors and changes of column data types.
*
* @param xmldb_field[]|database_column_info[] $columns
* @return int approximate row size in bytes
*/
public function guess_antolope_row_size(array $columns) {
if (empty($columns)) {
return 0;
}

$size = 0;
$first = reset($columns);

if (count($columns) > 1) {
// Do not start with zero because we need to cover changes of field types and
// this calculation is most probably not be accurate.
$size += 1000;
}

if ($first instanceof xmldb_field) {
foreach ($columns as $field) {
switch ($field->getType()) {
case XMLDB_TYPE_TEXT:
$size += 768;
break;
case XMLDB_TYPE_BINARY:
$size += 768;
break;
case XMLDB_TYPE_CHAR:
$bytes = $field->getLength() * 3;
if ($bytes > 768) {
$bytes = 768;
}
$size += $bytes;
break;
default:
// Anything else is usually maximum 8 bytes.
$size += 8;
}
}

} else if ($first instanceof database_column_info) {
foreach ($columns as $column) {
switch ($column->meta_type) {
case 'X':
$size += 768;
break;
case 'B':
$size += 768;
break;
case 'C':
$bytes = $column->max_length * 3;
if ($bytes > 768) {
$bytes = 768;
}
$size += $bytes;
break;
default:
// Anything else is usually maximum 8 bytes.
$size += 8;
}
}
}

return $size;
}

/**
* Given one correct xmldb_table, returns the SQL statements
* to create it (inside one array).
Expand All @@ -122,6 +197,15 @@ public function getCreateTableSQL($xmldb_table) {
// Do we know collation?
$collation = $this->mdb->get_dbcollation();

// Do we need to use compressed format for rows?
$rowformat = "";
$size = $this->guess_antolope_row_size($xmldb_table->getFields());
if ($size > self::ANTELOPE_MAX_ROW_SIZE) {
if ($this->mdb->is_compressed_row_format_supported()) {
$rowformat = "\n ROW_FORMAT=Compressed";
}
}

$sqlarr = parent::getCreateTableSQL($xmldb_table);

// This is a very nasty hack that tries to use just one query per created table
Expand All @@ -144,6 +228,9 @@ public function getCreateTableSQL($xmldb_table) {
}
$sql .= "\n DEFAULT COLLATE = $collation";
}
if ($rowformat) {
$sql .= $rowformat;
}
$sqls[] = $sql;
continue;
}
Expand Down Expand Up @@ -186,6 +273,39 @@ public function getCreateTableSQL($xmldb_table) {
return $sqls;
}

/**
* Given one xmldb_table and one xmldb_field, return the SQL statements needed to add the field to the table.
*
* @param xmldb_table $xmldb_table The table related to $xmldb_field.
* @param xmldb_field $xmldb_field The instance of xmldb_field to create the SQL from.
* @param string $skip_type_clause The type clause on alter columns, NULL by default.
* @param string $skip_default_clause The default clause on alter columns, NULL by default.
* @param string $skip_notnull_clause The null/notnull clause on alter columns, NULL by default.
* @return array The SQL statement for adding a field to the table.
*/
public function getAddFieldSQL($xmldb_table, $xmldb_field, $skip_type_clause = NULL, $skip_default_clause = NULL, $skip_notnull_clause = NULL) {
$sqls = parent::getAddFieldSQL($xmldb_table, $xmldb_field, $skip_type_clause, $skip_default_clause, $skip_notnull_clause);

if ($this->table_exists($xmldb_table)) {
$tablename = $xmldb_table->getName();

$size = $this->guess_antolope_row_size($this->mdb->get_columns($tablename));
$size += $this->guess_antolope_row_size(array($xmldb_field));

if ($size > self::ANTELOPE_MAX_ROW_SIZE) {
if ($this->mdb->is_compressed_row_format_supported()) {
$format = strtolower($this->mdb->get_row_format($tablename));
if ($format === 'compact' or $format === 'redundant') {
// Change the format before conversion so that we do not run out of space.
array_unshift($sqls, "ALTER TABLE {$this->prefix}$tablename ROW_FORMAT=Compressed");
}
}
}
}

return $sqls;
}

/**
* Given one correct xmldb_table, returns the SQL statements
* to create temporary table (inside one array).
Expand Down

0 comments on commit 7252078

Please sign in to comment.