Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move preview repair to query #23073

Closed
wants to merge 2 commits into from
Closed

Move preview repair to query #23073

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1bf3379
Move preview repair to query
rullzer Sep 28, 2020
656e58a
Ignore NotFoundException when fetching the old folder
MorrisJobke Oct 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
256 changes: 133 additions & 123 deletions core/Command/Preview/Repair.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\ILogger;
use OCP\Lock\ILockingProvider;
use OCP\Lock\LockedException;
Expand All @@ -58,12 +59,17 @@ class Repair extends Command {
private $memoryTreshold;
/** @var ILockingProvider */
private $lockingProvider;
/**
* @var IDBConnection
*/
private $db;

public function __construct(IConfig $config, IRootFolder $rootFolder, ILogger $logger, IniGetWrapper $phpIni, ILockingProvider $lockingProvider) {
public function __construct(IConfig $config, IRootFolder $rootFolder, ILogger $logger, IniGetWrapper $phpIni, ILockingProvider $lockingProvider, IDBConnection $db) {
$this->config = $config;
$this->rootFolder = $rootFolder;
$this->logger = $logger;
$this->lockingProvider = $lockingProvider;
$this->db = $db;

$this->memoryLimit = $phpIni->getBytes('memory_limit');
$this->memoryTreshold = $this->memoryLimit - 25 * 1024 * 1024;
Expand Down Expand Up @@ -109,46 +115,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
/** @var \OCP\Files\Folder $currentPreviewFolder */
$currentPreviewFolder = $this->rootFolder->get("appdata_$instanceId/preview");

$directoryListing = $currentPreviewFolder->getDirectoryListing();

$total = count($directoryListing);
/**
* by default there could be 0-9 a-f and the old-multibucket folder which are all fine
*/
if ($total < 18) {
$directoryListing = array_filter($directoryListing, function ($dir) {
if ($dir->getName() === 'old-multibucket') {
return false;
}

// a-f can't be a file ID -> removing from migration
if (preg_match('!^[a-f]$!', $dir->getName())) {
return false;
}

if (preg_match('!^[0-9]$!', $dir->getName())) {
// ignore folders that only has folders in them
if ($dir instanceof Folder) {
foreach ($dir->getDirectoryListing() as $entry) {
if (!$entry instanceof Folder) {
return true;
}
}
return false;
}
}
return true;
});
$total = count($directoryListing);
}

if ($total === 0) {
$output->writeln("All previews are already migrated.");
return 0;
}

$output->writeln("A total of $total preview files need to be migrated.");
$output->writeln("");
$output->writeln("The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This fill finish the current batch and then stop the migration. This migration can then just be started and it will continue.");

if ($input->getOption('batch')) {
Expand All @@ -162,6 +128,15 @@ protected function execute(InputInterface $input, OutputInterface $output): int
}
}

$qb = $this->db->getQueryBuilder();
$cursor = $qb->select($qb->func()->count('*'))
->from('filecache')
->where($qb->expr()->eq('parent', $qb->createNamedParameter($currentPreviewFolder->getId())))
->orderBy('fileid', 'ASC')
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if you just count, don't order

->execute();
$total = (int)$cursor->fetchColumn(0);
$cursor->closeCursor();

// register the SIGINT listener late in here to be able to exit in the early process of this command
pcntl_signal(SIGINT, [$this, 'sigIntHandler']);

Expand All @@ -176,107 +151,142 @@ protected function execute(InputInterface $input, OutputInterface $output): int
$progressBar->maxSecondsBetweenRedraws(0.2);
$progressBar->start();

foreach ($directoryListing as $oldPreviewFolder) {
pcntl_signal_dispatch();
$name = $oldPreviewFolder->getName();
$time = (new \DateTime())->format('H:i:s');
$section1->writeln("$time Migrating previews of file with fileId $name …");
$progressBar->display();
$limit = 1000;

if ($this->stopSignalReceived) {
$section1->writeln("$time Stopping migration …");
return 0;
}
if (!$oldPreviewFolder instanceof Folder) {
$section1->writeln(" Skipping non-folder $name …");
$progressBar->advance();
continue;
}
if ($name === 'old-multibucket') {
$section1->writeln(" Skipping fallback mount point $name …");
$progressBar->advance();
continue;
}
if (in_array($name, ['a', 'b', 'c', 'd', 'e', 'f'])) {
$section1->writeln(" Skipping hex-digit folder $name …");
$progressBar->advance();
continue;
}
if (!preg_match('!^\d+$!', $name)) {
$section1->writeln(" Skipping non-numeric folder $name …");
$progressBar->advance();
continue;
}
$hasEntries = true;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
$hasEntries = true;
$hasEntries = true;
$previousHighestFileId = 0;

do {
$qb = $this->db->getQueryBuilder();
$cursor = $qb->select('name')->from('filecache')
->where($qb->expr()->eq('parent', $qb->createNamedParameter($currentPreviewFolder->getId())))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't it make sense to remember the highest fileID from the previous loop run and put it in here as well?

Suggested change
->where($qb->expr()->eq('parent', $qb->createNamedParameter($currentPreviewFolder->getId())))
->where($qb->expr()->eq('parent', $qb->createNamedParameter($currentPreviewFolder->getId())))
->andWhere($qb->expr()->gt('fileid', $qb->createNamedParameter($previousHighestFileId)))

->orderBy('fileid', 'ASC')
->setMaxResults($limit)
->execute();

$newFoldername = Root::getInternalFolder($name);
$hasEntries = false;
while ($row = $cursor->fetch()) {
$oldPreviewFolder = $currentPreviewFolder->get($row['name']);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
$oldPreviewFolder = $currentPreviewFolder->get($row['name']);
$oldPreviewFolder = $currentPreviewFolder->get($row['name']);
$previousHighestFileId = $oldPreviewFolder->getId();


$memoryUsage = memory_get_usage();
if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) {
$section1->writeln("");
$section1->writeln("");
$section1->writeln("");
$section1->writeln(" Stopped process 25 MB before reaching the memory limit to avoid a hard crash.");
pcntl_signal_dispatch();
$name = $oldPreviewFolder->getName();
$time = (new \DateTime())->format('H:i:s');
$section1->writeln("$time Reached memory limit and stopped to avoid hard crash.");
return 1;
}

$lockName = 'occ preview:repair lock ' . $oldPreviewFolder->getId();
try {
$section1->writeln(" Locking \"$lockName\" …", OutputInterface::VERBOSITY_VERBOSE);
$this->lockingProvider->acquireLock($lockName, ILockingProvider::LOCK_EXCLUSIVE);
} catch (LockedException $e) {
$section1->writeln(" Skipping because it is locked - another process seems to work on this …");
continue;
}
$section1->writeln("$time Migrating previews of file with fileId $name …");
$progressBar->display();

$previews = $oldPreviewFolder->getDirectoryListing();
if ($previews !== []) {
try {
$this->rootFolder->get("appdata_$instanceId/preview/$newFoldername");
} catch (NotFoundException $e) {
$section1->writeln(" Create folder preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE);
if (!$dryMode) {
$this->rootFolder->newFolder("appdata_$instanceId/preview/$newFoldername");
}
if ($this->stopSignalReceived) {
$section1->writeln("$time Stopping migration …");
return 0;
}
if (!$oldPreviewFolder instanceof Folder) {
$section1->writeln(" Skipping non-folder $name …");
$progressBar->advance();
continue;
}
if ($name === 'old-multibucket') {
$section1->writeln(" Skipping fallback mount point $name …");
$progressBar->advance();
continue;
}
if (in_array($name, ['a', 'b', 'c', 'd', 'e', 'f'])) {
$section1->writeln(" Skipping hex-digit folder $name …");
$progressBar->advance();
continue;
}
if (!preg_match('!^\d+$!', $name)) {
$section1->writeln(" Skipping non-numeric folder $name …");
$progressBar->advance();
continue;
}

foreach ($previews as $preview) {
pcntl_signal_dispatch();
$previewName = $preview->getName();
if ($oldPreviewFolder instanceof Folder) {
$listing = $oldPreviewFolder->getDirectoryListing();
$hasFiles = false;
foreach ($listing as $node) {
if (!($node instanceof Folder)) {
$hasFiles = true;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
$hasFiles = true;
$hasFiles = true;
continue;

}
}

if ($preview instanceof Folder) {
$section1->writeln(" Skipping folder $name/$previewName …");
if (!$hasFiles) {
$section1->writeln(" Skipping since folder contains folders: $name …");
$progressBar->advance();
continue;
}
$section1->writeln(" Move preview/$name/$previewName to preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE);
}

$hasEntries = true;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This only triggers the while loop to continue if there are folders in the current set, that are processed. But what if there are only skipped folders in the current set? Then it would never reach the end. I can't come up with anything that would hold 1000 non-numeric folders, so this might be a theoretical issue.


$newFoldername = Root::getInternalFolder($name);

$memoryUsage = memory_get_usage();
if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) {
$section1->writeln("");
$section1->writeln("");
$section1->writeln("");
$section1->writeln(" Stopped process 25 MB before reaching the memory limit to avoid a hard crash.");
$time = (new \DateTime())->format('H:i:s');
$section1->writeln("$time Reached memory limit and stopped to avoid hard crash.");
return 1;
}

$lockName = 'occ preview:repair lock ' . $oldPreviewFolder->getId();
try {
$section1->writeln(" Locking \"$lockName\" …", OutputInterface::VERBOSITY_VERBOSE);
$this->lockingProvider->acquireLock($lockName, ILockingProvider::LOCK_EXCLUSIVE);
} catch (LockedException $e) {
$section1->writeln(" Skipping because it is locked - another process seems to work on this …");
continue;
}

$previews = $oldPreviewFolder->getDirectoryListing();
if ($previews !== []) {
try {
$this->rootFolder->get("appdata_$instanceId/preview/$newFoldername");
} catch (NotFoundException $e) {
$section1->writeln(" Create folder preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE);
if (!$dryMode) {
$this->rootFolder->newFolder("appdata_$instanceId/preview/$newFoldername");
}
}

foreach ($previews as $preview) {
pcntl_signal_dispatch();
$previewName = $preview->getName();

if ($preview instanceof Folder) {
$section1->writeln(" Skipping folder $name/$previewName …");
$progressBar->advance();
continue;
}
$section1->writeln(" Move preview/$name/$previewName to preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE);
if (!$dryMode) {
try {
$preview->move("appdata_$instanceId/preview/$newFoldername/$previewName");
} catch (\Exception $e) {
$this->logger->logException($e, ['app' => 'core', 'message' => "Failed to move preview from preview/$name/$previewName to preview/$newFoldername"]);
}
}
}
}
if ($oldPreviewFolder->getDirectoryListing() === []) {
$section1->writeln(" Delete empty folder preview/$name", OutputInterface::VERBOSITY_VERBOSE);
if (!$dryMode) {
try {
$preview->move("appdata_$instanceId/preview/$newFoldername/$previewName");
$oldPreviewFolder->delete();
} catch (\Exception $e) {
$this->logger->logException($e, ['app' => 'core', 'message' => "Failed to move preview from preview/$name/$previewName to preview/$newFoldername"]);
$this->logger->logException($e, ['app' => 'core', 'message' => "Failed to delete empty folder preview/$name"]);
}
}
}
}
if ($oldPreviewFolder->getDirectoryListing() === []) {
$section1->writeln(" Delete empty folder preview/$name", OutputInterface::VERBOSITY_VERBOSE);
if (!$dryMode) {
try {
$oldPreviewFolder->delete();
} catch (\Exception $e) {
$this->logger->logException($e, ['app' => 'core', 'message' => "Failed to delete empty folder preview/$name"]);
}
}
}

$this->lockingProvider->releaseLock($lockName, ILockingProvider::LOCK_EXCLUSIVE);
$section1->writeln(" Unlocked", OutputInterface::VERBOSITY_VERBOSE);
$this->lockingProvider->releaseLock($lockName, ILockingProvider::LOCK_EXCLUSIVE);
$section1->writeln(" Unlocked", OutputInterface::VERBOSITY_VERBOSE);

$section1->writeln(" Finished migrating previews of file with fileId $name …");
$progressBar->advance();
}
$section1->writeln(" Finished migrating previews of file with fileId $name …");
$progressBar->advance();
}

$cursor->closeCursor();
} while ($hasEntries === true);

$progressBar->finish();
$output->writeln("");
Expand Down