Skip to content

Commit

Permalink
Add skipping to nth page option/ability for dump importing process
Browse files Browse the repository at this point in the history
Usage: php importDump.php --skip-to 271500 /path_to/dumpfile.xml.gz

When importing a database dump and the import process crashes
(for random reasons) after a certain number of pages, the
"--skip-to" parameter allows restarting the import process at
a certain page instead of starting the import from scratch.

Change-Id: Ib36063b69d6846fc197800bba44287493b0632c0
  • Loading branch information
mertyildiran committed Jun 8, 2017
1 parent f0e12ae commit 4951c2b
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 0 deletions.
21 changes: 21 additions & 0 deletions includes/import/WikiImporter.php
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class WikiImporter {
private $mNoticeCallback, $mDebug;
private $mImportUploads, $mImageBasePath;
private $mNoUpdates = false;
private $pageOffset = 0;
/** @var Config */
private $config;
/** @var ImportTitleFactory */
Expand Down Expand Up @@ -146,6 +147,16 @@ function setNoUpdates( $noupdates ) {
$this->mNoUpdates = $noupdates;
}

/**
* Sets 'pageOffset' value. So it will skip the first n-1 pages
* and start from the nth page. It's 1-based indexing.
* @param int $nthPage
* @since 1.29
*/
function setPageOffset( $nthPage ) {
$this->pageOffset = $nthPage;
}

/**
* Set a callback that displays notice messages
*
Expand Down Expand Up @@ -562,9 +573,19 @@ public function doImport() {
$keepReading = $this->reader->read();
$skip = false;
$rethrow = null;
$pageCount = 0;
try {
while ( $keepReading ) {
$tag = $this->reader->localName;
if ( $this->pageOffset ) {
if ( $tag === 'page' ) {
$pageCount++;
}
if ( $pageCount < $this->pageOffset ) {
$keepReading = $this->reader->next();
continue;
}
}
$type = $this->reader->nodeType;

if ( !Hooks::run( 'ImportHandleToplevelXMLTag', [ $this ] ) ) {
Expand Down
6 changes: 6 additions & 0 deletions maintenance/importDump.php
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ function __construct() {
'Disable link table updates. Is faster but leaves the wiki in an inconsistent state'
);
$this->addOption( 'image-base-path', 'Import files from a specified path', false, true );
$this->addOption( 'skip-to', 'Start from nth page by skipping first n-1 pages', false, true );
$this->addArg( 'file', 'Dump file to import [else use stdin]', false );
}

Expand Down Expand Up @@ -301,6 +302,11 @@ function importFromHandle( $handle ) {
return false;
}
}
if ( $this->hasOption( 'skip-to' ) ) {
$nthPage = (int)$this->getOption( 'skip-to' );
$importer->setPageOffset( $nthPage );
$this->pageCount = $nthPage - 1;
}
$importer->setPageCallback( [ $this, 'reportPage' ] );
$this->importCallback = $importer->setRevisionCallback(
[ $this, 'handleRevision' ] );
Expand Down

0 comments on commit 4951c2b

Please sign in to comment.