Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use scroll API for CSV export job, refs #11534 #625

Closed
wants to merge 1 commit into from
Closed

Use scroll API for CSV export job, refs #11534 #625

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
58 changes: 31 additions & 27 deletions lib/job/arInformationObjectCsvExportJob.class.php
View file
Open in desktop
Expand Up @@ -47,8 +47,8 @@ public function runJob($parameters)
$this->archivalStandard = 'rad';
}

// Create query increasing limit from default
$this->search = new arElasticSearchPluginQuery(1000000000);
// Create query setting limit for scrolling
$this->search = new arElasticSearchPluginQuery(1000);

if ($this->params['params']['fromClipboard'])
{
Expand Down Expand Up @@ -124,41 +124,45 @@ protected function exportResults($path)
array_unshift($writer->columnNames, 'referenceCode');
array_unshift($writer->standardColumns, 'referenceCode');

$resultSet = QubitSearch::getInstance()->index->getType('QubitInformationObject')->search($this->search->getQuery(false, false));
$search = QubitSearch::getInstance()->index->getType('QubitInformationObject')->createSearch($this->search->getQuery(false, false));
$scroll = new \Elastica\Scroll($search);

foreach ($resultSet as $hit)
foreach ($scroll as $resultSet)
{
$resource = QubitInformationObject::getById($hit->getId());

// If ElasticSearch document is stale (corresponding MySQL data deleted), ignore
if ($resource !== null)
foreach ($resultSet as $hit)
{
// Don't export draft descriptions with public option.
// Don't export records if level of description is not in list of selected LODs.
if (($public && $resource->getPublicationStatus()->statusId == QubitTerm::PUBLICATION_STATUS_DRAFT_ID) ||
(0 < $numLevels && !array_key_exists($resource->levelOfDescriptionId, $levels)))
$resource = QubitInformationObject::getById($hit->getId());

// If ElasticSearch document is stale (corresponding MySQL data deleted), ignore
if ($resource !== null)
{
continue;
}
// Don't export draft descriptions with public option.
// Don't export records if level of description is not in list of selected LODs.
if (($public && $resource->getPublicationStatus()->statusId == QubitTerm::PUBLICATION_STATUS_DRAFT_ID) ||
(0 < $numLevels && !array_key_exists($resource->levelOfDescriptionId, $levels)))
{
continue;
}

$writer->exportResource($resource);
$writer->exportResource($resource);

// export descendants if configured
if (!$this->params['current-level-only'])
{
foreach ($resource->getDescendantsForExport($this->params) as $descendant)
// export descendants if configured
if (!$this->params['current-level-only'])
{
$writer->exportResource($descendant);
foreach ($resource->getDescendantsForExport($this->params) as $descendant)
{
$writer->exportResource($descendant);
}
}
}

// Log progress every 1000 rows
if ($itemsExported && ($itemsExported % 1000 == 0))
{
$this->info($this->i18n->__('%1 items exported.', array('%1' => $itemsExported)));
}
// Log progress every 1000 rows
if ($itemsExported && ($itemsExported % 1000 == 0))
{
$this->info($this->i18n->__('%1 items exported.', array('%1' => $itemsExported)));
}

$itemsExported++;
$itemsExported++;
}
}
}

Expand Down