Skip to content

Commit

Permalink
Merge branch 'MDL-40103_temptable' of https://github.com/mr-russ/moodle
Browse files Browse the repository at this point in the history 
Conflicts:
	lib/db/upgrade.php
  • Loading branch information
@stronk7
stronk7 committed Jun 17, 2013
2 parents 407020e + 62d6f18 commit 96c90fa
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 160 deletions.
78 changes: 32 additions & 46 deletions backup/util/dbops/backup_controller_dbops.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,58 +103,44 @@ public static function load_controller($backupid) {
}

public static function create_backup_ids_temp_table($backupid) {
self::create_temptable_from_real_table($backupid, 'backup_ids_template', 'backup_ids_temp');
global $CFG, $DB;
$dbman = $DB->get_manager(); // We are going to use database_manager services

$xmldb_table = new xmldb_table('backup_ids_temp');
$xmldb_table->add_field('id', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
// Set default backupid (not needed but this enforce any missing backupid). That's hackery in action!
$xmldb_table->add_field('backupid', XMLDB_TYPE_CHAR, 32, null, XMLDB_NOTNULL, null, $backupid);
$xmldb_table->add_field('itemname', XMLDB_TYPE_CHAR, 160, null, XMLDB_NOTNULL, null, null);
$xmldb_table->add_field('itemid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, null);
$xmldb_table->add_field('newitemid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$xmldb_table->add_field('parentitemid', XMLDB_TYPE_INTEGER, 10, null, null, null, '0');
$xmldb_table->add_field('info', XMLDB_TYPE_TEXT, 1333, null, null, null, null);
$xmldb_table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
$xmldb_table->add_key('backupid_itemname_itemid_uk', XMLDB_KEY_UNIQUE, array('backupid','itemname','itemid'));
$xmldb_table->add_index('backupid_parentitemid_ix', null, array('backupid','itemname','parentitemid'));
$xmldb_table->add_index('backupid_itemname_newitemid_ix', null, array('backupid','itemname','newitemid'));

$dbman->create_temp_table($xmldb_table); // And create it

}

/**
* Given one "real" tablename, create one temp table suitable for be used in backup/restore operations
*/
public static function create_temptable_from_real_table($backupid, $realtablename, $temptablename) {
public static function create_backup_files_temp_table($backupid) {
global $CFG, $DB;
$dbman = $DB->get_manager(); // We are going to use database_manager services

// As far as xmldb objects use a lot of circular references (prev and next) and we aren't destroying
// them at all, that causes one memory leak of about 3M per backup execution, not problematic for
// individual backups but critical for automated (multiple) ones.
// So we are statically caching the xmldb_table definition here to produce the leak "only" once
static $xmldb_tables = array();

// Not cached, get it
if (!isset($xmldb_tables[$realtablename])) {
// Note: For now we are going to load the realtablename from core lib/db/install.xml
// that way, any change in the "template" will be applied here automatically. If this causes
// too much slow, we can always forget about the template and keep maintained the xmldb_table
// structure inline - manually - here.
// TODO: Right now, loading the whole lib/db/install.xml is "eating" 10M, we should
// change our way here in order to decrease that memory usage
$templatetablename = $realtablename;
$targettablename = $temptablename;
$xmlfile = $CFG->dirroot . '/lib/db/install.xml';
$xmldb_file = new xmldb_file($xmlfile);
if (!$xmldb_file->fileExists()) {
throw new ddl_exception('ddlxmlfileerror', null, 'File does not exist');
}
$loaded = $xmldb_file->loadXMLStructure();
if (!$loaded || !$xmldb_file->isLoaded()) {
throw new ddl_exception('ddlxmlfileerror', null, 'not loaded??');
}
$xmldb_structure = $xmldb_file->getStructure();
$xmldb_table = $xmldb_structure->getTable($templatetablename);
if (is_null($xmldb_table)) {
throw new ddl_exception('ddlunknowntable', null, 'The table ' . $templatetablename . ' is not defined in file ' . $xmlfile);
}
// Clean prev & next, we are alone
$xmldb_table->setNext(null);
$xmldb_table->setPrevious(null);
// Rename
$xmldb_table->setName($targettablename);
// Cache it
$xmldb_tables[$realtablename] = $xmldb_table;
}
// Arrived here, we have the table always in static cache, get it
$xmldb_table = $xmldb_tables[$realtablename];
$xmldb_table = new xmldb_table('backup_files_temp');
$xmldb_table->add_field('id', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
// Set default backupid (not needed but this enforce any missing backupid). That's hackery in action!
$xmldb_table->getField('backupid')->setDefault($backupid);
$xmldb_table->add_field('backupid', XMLDB_TYPE_CHAR, 32, null, XMLDB_NOTNULL, null, $backupid);
$xmldb_table->add_field('contextid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, null);
$xmldb_table->add_field('component', XMLDB_TYPE_CHAR, 100, null, XMLDB_NOTNULL, null, null);
$xmldb_table->add_field('filearea', XMLDB_TYPE_CHAR, 50, null, XMLDB_NOTNULL, null, null);
$xmldb_table->add_field('itemid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, null);
$xmldb_table->add_field('info', XMLDB_TYPE_TEXT, 1333, null, null, null, null);
$xmldb_table->add_field('newcontextid', XMLDB_TYPE_INTEGER, 10, null, null, null, '0');
$xmldb_table->add_field('newitemid', XMLDB_TYPE_INTEGER, 10, null, null, null, '0');
$xmldb_table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
$xmldb_table->add_index('backupid_contextid_component_filearea_itemid_ix', null, array('backupid','contextid','component','filearea','itemid'));

$dbman->create_temp_table($xmldb_table); // And create it
}
Expand Down
4 changes: 2 additions & 2 deletions backup/util/dbops/restore_controller_dbops.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,8 @@ public static function create_restore_temp_tables($restoreid) {
// TODO: If not match, exception, table corresponds to another backup/restore operation
return true;
}
backup_controller_dbops::create_temptable_from_real_table($restoreid, 'backup_ids_template', 'backup_ids_temp');
backup_controller_dbops::create_temptable_from_real_table($restoreid, 'backup_files_template', 'backup_files_temp');
create_backup_ids_temp_table($restoreid);
create_backup_files_temp_table($restoreid);
return false;
}

Expand Down
71 changes: 0 additions & 71 deletions lib/db/install.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2736,44 +2736,6 @@
<INDEX NAME="typeitem_ix" UNIQUE="false" FIELDS="type, itemid"/>
</INDEXES>
</TABLE>
<TABLE NAME="backup_ids_template" COMMENT="To store all sort of ids along the backup process. Note this table isn't really used but its temporary counterpart.">
<FIELDS>
<FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
<FIELD NAME="backupid" TYPE="char" LENGTH="32" NOTNULL="true" SEQUENCE="false" COMMENT="unique id of the backup"/>
<FIELD NAME="itemname" TYPE="char" LENGTH="160" NOTNULL="true" SEQUENCE="false" COMMENT="name of the component this id belongs to (usually table names)"/>
<FIELD NAME="itemid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="id of the component (usually table-&amp;gt;id values)"/>
<FIELD NAME="newitemid" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false" COMMENT="new id of the component after restore"/>
<FIELD NAME="parentitemid" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false" COMMENT="other id that can be useful to represent parent-child relations"/>
<FIELD NAME="info" TYPE="text" NOTNULL="false" SEQUENCE="false" COMMENT="to store information related with the item, base64(serialize())"/>
</FIELDS>
<KEYS>
<KEY NAME="primary" TYPE="primary" FIELDS="id"/>
<KEY NAME="backupid_itemname_itemid_uk" TYPE="unique" FIELDS="backupid, itemname, itemid"/>
</KEYS>
<INDEXES>
<INDEX NAME="backupid_parentitemid_ix" UNIQUE="false" FIELDS="backupid, itemname, parentitemid"/>
<INDEX NAME="backupid_itemname_newitemid_ix" UNIQUE="false" FIELDS="backupid, itemname, newitemid"/>
</INDEXES>
</TABLE>
<TABLE NAME="backup_files_template" COMMENT="To store files along the backup process. Note this table isn't really used but its temporary counterpart.">
<FIELDS>
<FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
<FIELD NAME="backupid" TYPE="char" LENGTH="32" NOTNULL="true" SEQUENCE="false" COMMENT="unique backup id this record corresponds to"/>
<FIELD NAME="contextid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false" COMMENT="source contextid of the file"/>
<FIELD NAME="component" TYPE="char" LENGTH="100" NOTNULL="true" SEQUENCE="false"/>
<FIELD NAME="filearea" TYPE="char" LENGTH="50" NOTNULL="true" SEQUENCE="false"/>
<FIELD NAME="itemid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
<FIELD NAME="info" TYPE="text" NOTNULL="false" SEQUENCE="false" COMMENT="to store the complete file record (serialized)"/>
<FIELD NAME="newcontextid" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false" COMMENT="The new contextid that the file has been restored to."/>
<FIELD NAME="newitemid" TYPE="int" LENGTH="10" NOTNULL="false" SEQUENCE="false" COMMENT="The new itemid the file has been restored to."/>
</FIELDS>
<KEYS>
<KEY NAME="primary" TYPE="primary" FIELDS="id"/>
</KEYS>
<INDEXES>
<INDEX NAME="backupid_contextid_component_filearea_itemid_ix" UNIQUE="false" FIELDS="backupid, contextid, component, filearea, itemid"/>
</INDEXES>
</TABLE>
<TABLE NAME="backup_logs" COMMENT="To store all the logs from backup and restore operations (by db logger)">
<FIELDS>
<FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
Expand Down Expand Up @@ -2901,39 +2863,6 @@
<KEY NAME="primary" TYPE="primary" FIELDS="id"/>
</KEYS>
</TABLE>
<TABLE NAME="temp_enroled_template" COMMENT="Temporary storage for course enrolments">
<FIELDS>
<FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
<FIELD NAME="userid" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false"/>
<FIELD NAME="courseid" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false"/>
<FIELD NAME="roleid" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="false"/>
</FIELDS>
<KEYS>
<KEY NAME="primary" TYPE="primary" FIELDS="id"/>
</KEYS>
<INDEXES>
<INDEX NAME="userid" UNIQUE="false" FIELDS="userid"/>
<INDEX NAME="courseid" UNIQUE="false" FIELDS="courseid"/>
<INDEX NAME="roleid" UNIQUE="false" FIELDS="roleid"/>
</INDEXES>
</TABLE>
<TABLE NAME="temp_log_template" COMMENT="Temporary storage for daily logs">
<FIELDS>
<FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
<FIELD NAME="userid" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false"/>
<FIELD NAME="course" TYPE="int" LENGTH="10" NOTNULL="true" DEFAULT="0" SEQUENCE="false"/>
<FIELD NAME="action" TYPE="char" LENGTH="40" NOTNULL="true" SEQUENCE="false"/>
</FIELDS>
<KEYS>
<KEY NAME="primary" TYPE="primary" FIELDS="id"/>
</KEYS>
<INDEXES>
<INDEX NAME="action" UNIQUE="false" FIELDS="action"/>
<INDEX NAME="course" UNIQUE="false" FIELDS="course"/>
<INDEX NAME="user" UNIQUE="false" FIELDS="userid"/>
<INDEX NAME="usercourseaction" UNIQUE="false" FIELDS="userid, course, action"/>
</INDEXES>
</TABLE>
<TABLE NAME="badge" COMMENT="Defines badge">
<FIELDS>
<FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
Expand Down
18 changes: 18 additions & 0 deletions lib/db/upgrade.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2172,5 +2172,23 @@ function xmldb_main_upgrade($oldversion) {
upgrade_main_savepoint(true, 2013061400.01);
}

if ($oldversion < 2013061700.00) {
// MDL-40103: Remove unused template tables from the database.
// These are now created inline with xmldb_table

$tablestocleanup = array('temp_enroled_template','temp_log_template','backup_files_template','backup_ids_template');
$dbman = $DB->get_manager();

foreach ($tablestocleanup as $table) {
$xmltable = new xmldb_table($table);
if ($dbman->table_exists($xmltable)) {
$dbman->drop_table($xmltable);
}
}

// Main savepoint reached.
upgrade_main_savepoint(true, 2013061700.00);
}

return true;
}
101 changes: 60 additions & 41 deletions lib/statslib.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1589,47 +1589,66 @@ function stats_temp_table_create() {

stats_temp_table_drop();

$xmlfile = $CFG->dirroot . '/lib/db/install.xml';
$tables = array();

// Allows for the additional xml files to be used (if necessary)
$files = array(
$xmlfile => array(
'stats_daily' => array('temp_stats_daily'),
'stats_user_daily' => array('temp_stats_user_daily'),
'temp_enroled_template' => array('temp_enroled'),
'temp_log_template' => array('temp_log1', 'temp_log2'),
),
);

foreach ($files as $file => $contents) {

$xmldb_file = new xmldb_file($file);
if (!$xmldb_file->fileExists()) {
throw new ddl_exception('ddlxmlfileerror', null, 'File does not exist');
}
$loaded = $xmldb_file->loadXMLStructure();
if (!$loaded || !$xmldb_file->isLoaded()) {
throw new ddl_exception('ddlxmlfileerror', null, 'not loaded??');
}
$xmldb_structure = $xmldb_file->getStructure();

foreach ($contents as $template => $names) {
$table = $xmldb_structure->getTable($template);

if (is_null($table)) {
throw new ddl_exception('ddlunknowntable', null, 'The table '. $name .' is not defined in the file '. $xmlfile);
}
$table->setNext(null);
$table->setPrevious(null);

foreach ($names as $name) {
$named = clone $table;
$named->setName($name);
$tables[$name] = $named;
}
}
}
$tables = array();

/// Define tables user to be created
$table = new xmldb_table('temp_stats_daily');
$table->add_field('id', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
$table->add_field('courseid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, null);
$table->add_field('timeend', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, null);
$table->add_field('roleid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, null);
$table->add_field('stattype', XMLDB_TYPE_CHAR, 20, null, XMLDB_NOTNULL, null, 'activity');
$table->add_field('stat1', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, null);
$table->add_field('stat2', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, null);
$table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
$table->add_index('courseid', null, array('courseid'));
$table->add_index('timeend', null, array('timeend'));
$table->add_index('roleid', null, array('roleid'));
$tables['temp_stats_daily'] = $table;

$table = new xmldb_table('temp_stats_user_daily');
$table->add_field('id', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
$table->add_field('courseid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_field('userid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_field('roleid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_field('timeend', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_field('statsreads', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_field('statswrites', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_field('stattyspe', XMLDB_TYPE_CHAR, 30, null, XMLDB_NOTNULL, null, null);
$table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
$table->add_index('courseid', null, array('courseid'));
$table->add_index('userid', null, array('userid'));
$table->add_index('timeend', null, array('timeend'));
$table->add_index('roleid', null, array('roleid'));
$tables['temp_stats_user_daily'] = $table;

$table = new xmldb_table('temp_enroled');
$table->add_field('id', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
$table->add_field('userid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_field('courseid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_field('roleid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
$table->add_index('userid', null, array('userid'));
$table->add_index('courseid', null, array('courseid'));
$table->add_index('roleid', null, array('roleid'));
$tables['temp_enroled'] = $table;


$table = new xmldb_table('temp_log1');
$table->add_field('id', XMLDB_TYPE_INTEGER, '10', XMLDB_UNSIGNED, XMLDB_NOTNULL, XMLDB_SEQUENCE, null);
$table->add_field('userid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_field('courseid', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, null, '0');
$table->add_field('action', XMLDB_TYPE_CHAR, 40, null, XMLDB_NOTNULL, null, null);
$table->add_key('primary', XMLDB_KEY_PRIMARY, array('id'));
$table->add_index('action', null, array('action'));
$table->add_index('course', null, array('courseid'));
$table->add_index('user', null, array('userid'));
$table->add_index('usercourseaction', null, array('userid','course','action'));
$tables['temp_log1'] = $table;

/// temp_log2 is exactly the same as temp_log1.
$tables['temp_log2'] = clone $tables['temp_log1'];
$tables['temp_log2']->setName('temp_log2');

try {

Expand Down

0 comments on commit 96c90fa

Please sign in to comment.