Permalink
Browse files

Add skipping to nth page option/ability for dump importing process

Usage: php importDump.php --skip-to 271500 /path_to/dumpfile.xml.gz

When importing a database dump and the import process crashes
(for random reasons) after a certain number of pages, the
"--skip-to" parameter allows restarting the import process at
a certain page instead of starting the import from scratch.

Change-Id: Ib36063b69d6846fc197800bba44287493b0632c0
  • Loading branch information...
mertyildiran committed May 26, 2017
1 parent f0e12ae commit 4951c2b54f429037038cca617eb3e5860c1c271f
Showing with 27 additions and 0 deletions.
  1. +21 −0 includes/import/WikiImporter.php
  2. +6 −0 maintenance/importDump.php
@@ -39,6 +39,7 @@ class WikiImporter {
private $mNoticeCallback, $mDebug; private $mNoticeCallback, $mDebug;
private $mImportUploads, $mImageBasePath; private $mImportUploads, $mImageBasePath;
private $mNoUpdates = false; private $mNoUpdates = false;
private $pageOffset = 0;
/** @var Config */ /** @var Config */
private $config; private $config;
/** @var ImportTitleFactory */ /** @var ImportTitleFactory */
@@ -146,6 +147,16 @@ function setNoUpdates( $noupdates ) {
$this->mNoUpdates = $noupdates; $this->mNoUpdates = $noupdates;
} }
/**
* Sets 'pageOffset' value. So it will skip the first n-1 pages
* and start from the nth page. It's 1-based indexing.
* @param int $nthPage
* @since 1.29
*/
function setPageOffset( $nthPage ) {
$this->pageOffset = $nthPage;
}
/** /**
* Set a callback that displays notice messages * Set a callback that displays notice messages
* *
@@ -562,9 +573,19 @@ public function doImport() {
$keepReading = $this->reader->read(); $keepReading = $this->reader->read();
$skip = false; $skip = false;
$rethrow = null; $rethrow = null;
$pageCount = 0;
try { try {
while ( $keepReading ) { while ( $keepReading ) {
$tag = $this->reader->localName; $tag = $this->reader->localName;
if ( $this->pageOffset ) {
if ( $tag === 'page' ) {
$pageCount++;
}
if ( $pageCount < $this->pageOffset ) {
$keepReading = $this->reader->next();
continue;
}
}
$type = $this->reader->nodeType; $type = $this->reader->nodeType;
if ( !Hooks::run( 'ImportHandleToplevelXMLTag', [ $this ] ) ) { if ( !Hooks::run( 'ImportHandleToplevelXMLTag', [ $this ] ) ) {
@@ -80,6 +80,7 @@ function __construct() {
'Disable link table updates. Is faster but leaves the wiki in an inconsistent state' 'Disable link table updates. Is faster but leaves the wiki in an inconsistent state'
); );
$this->addOption( 'image-base-path', 'Import files from a specified path', false, true ); $this->addOption( 'image-base-path', 'Import files from a specified path', false, true );
$this->addOption( 'skip-to', 'Start from nth page by skipping first n-1 pages', false, true );
$this->addArg( 'file', 'Dump file to import [else use stdin]', false ); $this->addArg( 'file', 'Dump file to import [else use stdin]', false );
} }
@@ -301,6 +302,11 @@ function importFromHandle( $handle ) {
return false; return false;
} }
} }
if ( $this->hasOption( 'skip-to' ) ) {
$nthPage = (int)$this->getOption( 'skip-to' );
$importer->setPageOffset( $nthPage );
$this->pageCount = $nthPage - 1;
}
$importer->setPageCallback( [ $this, 'reportPage' ] ); $importer->setPageCallback( [ $this, 'reportPage' ] );
$this->importCallback = $importer->setRevisionCallback( $this->importCallback = $importer->setRevisionCallback(
[ $this, 'handleRevision' ] ); [ $this, 'handleRevision' ] );

0 comments on commit 4951c2b

Please sign in to comment.