From 1264dd11a048130b54a2c60445d864c2cf3e5699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20S=CC=8Ckoda?= Date: Sat, 10 Nov 2012 13:23:47 +0100 Subject: [PATCH 1/2] MDL-36493 improve performance of mysql unsigned/lob upgrade --- lib/db/upgrade.php | 13 ++----- lib/db/upgradelib.php | 89 ++++++++++++++----------------------------- 2 files changed, 31 insertions(+), 71 deletions(-) diff --git a/lib/db/upgrade.php b/lib/db/upgrade.php index 7dee7a7734753..0dec8901cd2c4 100644 --- a/lib/db/upgrade.php +++ b/lib/db/upgrade.php @@ -227,17 +227,10 @@ function xmldb_main_upgrade($oldversion) { upgrade_main_savepoint(true, 2012030100.01); } - if ($oldversion < 2012030100.02) { - // migrate all numbers to signed - it should be safe to interrupt this and continue later - upgrade_mysql_fix_unsigned_columns(); - - // Main savepoint reached - upgrade_main_savepoint(true, 2012030100.02); - } - if ($oldversion < 2012030900.01) { - // migrate all texts and binaries to big size - it should be safe to interrupt this and continue later - upgrade_mysql_fix_lob_columns(); + // Migrate all numbers to signed & all texts and binaries to big size. + // It should be safe to interrupt this and continue later. + upgrade_mysql_fix_unsigned_and_lob_columns(); // Main savepoint reached upgrade_main_savepoint(true, 2012030900.01); diff --git a/lib/db/upgradelib.php b/lib/db/upgradelib.php index 5247e49429e2d..f0ee0fd78456e 100644 --- a/lib/db/upgradelib.php +++ b/lib/db/upgradelib.php @@ -77,14 +77,15 @@ function upgrade_mysql_get_supported_tables() { } /** - * Remove all signed numbers from current database - mysql only. + * Remove all signed numbers from current database and change + * text fields to long texts - mysql only. */ -function upgrade_mysql_fix_unsigned_columns() { - // we are not using standard API for changes of column - // because everything 'signed'-related will be removed soon +function upgrade_mysql_fix_unsigned_and_lob_columns() { + // We are not using standard API for changes of column + // because everything 'signed'-related will be removed soon. - // if anybody already has numbers higher than signed limit the execution stops - // and tables must be fixed manually before continuing upgrade + // If anybody already has numbers higher than signed limit the execution stops + // and tables must be fixed manually before continuing upgrade. global $DB; @@ -92,7 +93,7 @@ function upgrade_mysql_fix_unsigned_columns() { return; } - $pbar = new progress_bar('mysqlconvertunsigned', 500, true); + $pbar = new progress_bar('mysqlconvertunsignedlobs', 500, true); $prefix = $DB->get_prefix(); $tables = upgrade_mysql_get_supported_tables(); @@ -101,11 +102,13 @@ function upgrade_mysql_fix_unsigned_columns() { $i = 0; foreach ($tables as $table) { $i++; - // set appropriate timeout - 5 minutes per milion of records should be enough, min 60 minutes just in case + // Set appropriate timeout - 1 minute per thousand of records should be enough, min 60 minutes just in case. $count = $DB->count_records($table, array()); - $timeout = ($count/1000000)*5*60; + $timeout = ($count/1000)*60; $timeout = ($timeout < 60*60) ? 60*60 : (int)$timeout; + $changes = array(); + $sql = "SHOW COLUMNS FROM `{{$table}}`"; $rs = $DB->get_recordset_sql($sql); foreach ($rs as $column) { @@ -134,67 +137,31 @@ function upgrade_mysql_fix_unsigned_columns() { $notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL'; $default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : ''; $autoinc = (stripos($column->extra, 'auto_increment') !== false) ? 'AUTO_INCREMENT' : ''; - // primary and unique not necessary here, change_database_structure does not add prefix - $sql = "ALTER TABLE `{$prefix}$table` MODIFY COLUMN `$column->field` $type $notnull $default $autoinc"; - $DB->change_database_structure($sql); - } - } - $rs->close(); - - $pbar->update($i, $tablecount, "Converted unsigned columns in MySQL database - $i/$tablecount."); - } -} - -/** - * Migrate all text and binary columns to big size - mysql only. - */ -function upgrade_mysql_fix_lob_columns() { - // we are not using standard API for changes of column intentionally - - global $DB; - - if ($DB->get_dbfamily() !== 'mysql') { - return; - } - - $pbar = new progress_bar('mysqlconvertlobs', 500, true); + // Primary and unique not necessary here, change_database_structure does not add prefix. - $prefix = $DB->get_prefix(); - $tables = upgrade_mysql_get_supported_tables(); - asort($tables); + $changes[] = "MODIFY COLUMN `$column->field` $type $notnull $default $autoinc"; - $tablecount = count($tables); - $i = 0; - foreach ($tables as $table) { - $i++; - // set appropriate timeout - 1 minute per thousand of records should be enough, min 60 minutes just in case - $count = $DB->count_records($table, array()); - $timeout = ($count/1000)*60; - $timeout = ($timeout < 60*60) ? 60*60 : (int)$timeout; - - $sql = "SHOW COLUMNS FROM `{{$table}}`"; - $rs = $DB->get_recordset_sql($sql); - foreach ($rs as $column) { - upgrade_set_timeout($timeout); - - $column = (object)array_change_key_case((array)$column, CASE_LOWER); - if ($column->type === 'tinytext' or $column->type === 'mediumtext' or $column->type === 'text') { + } else if ($column->type === 'tinytext' or $column->type === 'mediumtext' or $column->type === 'text') { $notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL'; $default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : ''; - // primary, unique and inc are not supported for texts - $sql = "ALTER TABLE `{$prefix}$table` MODIFY COLUMN `$column->field` LONGTEXT $notnull $default"; - $DB->change_database_structure($sql); - } - if ($column->type === 'tinyblob' or $column->type === 'mediumblob' or $column->type === 'blob') { + // Primary, unique and inc are not supported for texts. + $changes[] = "MODIFY COLUMN `$column->field` LONGTEXT $notnull $default"; + + } else if ($column->type === 'tinyblob' or $column->type === 'mediumblob' or $column->type === 'blob') { $notnull = ($column->null === 'NO') ? 'NOT NULL' : 'NULL'; $default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : ''; - // primary, unique and inc are not supported for blobs - $sql = "ALTER TABLE `{$prefix}$table` MODIFY COLUMN `$column->field` LONGBLOB $notnull $default"; - $DB->change_database_structure($sql); + // Primary, unique and inc are not supported for blobs. + $changes[] = "MODIFY COLUMN `$column->field` LONGBLOB $notnull $default"; } + } $rs->close(); - $pbar->update($i, $tablecount, "Converted LOB columns in MySQL database - $i/$tablecount."); + if ($changes) { + $sql = "ALTER TABLE `{$prefix}$table` ".implode(', ', $changes); + $DB->change_database_structure($sql); + } + + $pbar->update($i, $tablecount, "Converted unsigned/lob columns in MySQL database - $i/$tablecount."); } } From 3d4de0bc4eabcc22b93049df5d8bfbf2f254607b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20S=CC=8Ckoda?= Date: Mon, 12 Nov 2012 19:48:18 +0100 Subject: [PATCH 2/2] MDL-36493 improve timeout handling in mysql upgrade Credit goes to Mark Nielsen, thanks. --- lib/db/upgradelib.php | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/db/upgradelib.php b/lib/db/upgradelib.php index f0ee0fd78456e..6edc5d8c827e9 100644 --- a/lib/db/upgradelib.php +++ b/lib/db/upgradelib.php @@ -102,18 +102,12 @@ function upgrade_mysql_fix_unsigned_and_lob_columns() { $i = 0; foreach ($tables as $table) { $i++; - // Set appropriate timeout - 1 minute per thousand of records should be enough, min 60 minutes just in case. - $count = $DB->count_records($table, array()); - $timeout = ($count/1000)*60; - $timeout = ($timeout < 60*60) ? 60*60 : (int)$timeout; $changes = array(); $sql = "SHOW COLUMNS FROM `{{$table}}`"; $rs = $DB->get_recordset_sql($sql); foreach ($rs as $column) { - upgrade_set_timeout($timeout); - $column = (object)array_change_key_case((array)$column, CASE_LOWER); if (stripos($column->type, 'unsigned') !== false) { $maxvalue = 0; @@ -138,7 +132,6 @@ function upgrade_mysql_fix_unsigned_and_lob_columns() { $default = (!is_null($column->default) and $column->default !== '') ? "DEFAULT '$column->default'" : ''; $autoinc = (stripos($column->extra, 'auto_increment') !== false) ? 'AUTO_INCREMENT' : ''; // Primary and unique not necessary here, change_database_structure does not add prefix. - $changes[] = "MODIFY COLUMN `$column->field` $type $notnull $default $autoinc"; } else if ($column->type === 'tinytext' or $column->type === 'mediumtext' or $column->type === 'text') { @@ -158,6 +151,12 @@ function upgrade_mysql_fix_unsigned_and_lob_columns() { $rs->close(); if ($changes) { + // Set appropriate timeout - 1 minute per thousand of records should be enough, min 60 minutes just in case. + $count = $DB->count_records($table, array()); + $timeout = ($count/1000)*60; + $timeout = ($timeout < 60*60) ? 60*60 : (int)$timeout; + upgrade_set_timeout($timeout); + $sql = "ALTER TABLE `{$prefix}$table` ".implode(', ', $changes); $DB->change_database_structure($sql); }