Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

rework to support multiple concurrent instances #1

Open
wants to merge 1 commit into
base: master
from
Open
Changes from all commits
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.

Always

Just for now

@@ -13,7 +13,7 @@ The first time you install this app, before using a cron job, you properly want
</description>
<licence>AGPL</licence>
<author>Roeland Jago Douma</author>
<version>2.1.0</version>
<version>2.2.0</version>
<namespace>PreviewGenerator</namespace>
<category>multimedia</category>
<website>https://github.com/rullzer/previewgenerator</website>
@@ -12,6 +12,7 @@
'OCA\\PreviewGenerator\\Command\\PreGenerate' => $baseDir . '/../lib/Command/PreGenerate.php',
'OCA\\PreviewGenerator\\Command\\TimestampFormatter' => $baseDir . '/../lib/Command/TimestampFormatter.php',
'OCA\\PreviewGenerator\\Migration\\Version020000Date20180823071939' => $baseDir . '/../lib/Migration/Version020000Date20180823071939.php',
'OCA\\PreviewGenerator\\Migration\\Version020200Date20190608205303' => $baseDir . '/../lib/Migration/Version020200Date20190608205303.php',
'OCA\\PreviewGenerator\\SizeHelper' => $baseDir . '/../lib/SizeHelper.php',
'OCA\\PreviewGenerator\\Watcher' => $baseDir . '/../lib/Watcher.php',
);
@@ -27,6 +27,7 @@ class ComposerStaticInitPreviewGenerator
'OCA\\PreviewGenerator\\Command\\PreGenerate' => __DIR__ . '/..' . '/../lib/Command/PreGenerate.php',
'OCA\\PreviewGenerator\\Command\\TimestampFormatter' => __DIR__ . '/..' . '/../lib/Command/TimestampFormatter.php',
'OCA\\PreviewGenerator\\Migration\\Version020000Date20180823071939' => __DIR__ . '/..' . '/../lib/Migration/Version020000Date20180823071939.php',
'OCA\\PreviewGenerator\\Migration\\Version020200Date20190608205303' => __DIR__ . '/..' . '/../lib/Migration/Version020200Date20190608205303.php',
'OCA\\PreviewGenerator\\SizeHelper' => __DIR__ . '/..' . '/../lib/SizeHelper.php',
'OCA\\PreviewGenerator\\Watcher' => __DIR__ . '/..' . '/../lib/Watcher.php',
);
@@ -31,6 +31,7 @@
use OCP\Files\IRootFolder;
use OCP\Files\NotFoundException;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\IPreview;
use OCP\IUser;
use OCP\IUserManager;
@@ -54,6 +55,9 @@ class Generate extends Command {
/** @var IConfig */
protected $config;
/** @var IDBConnection */
protected $connection;
/** @var OutputInterface */
protected $output;
@@ -67,13 +71,15 @@ public function __construct(IRootFolder $rootFolder,
IUserManager $userManager,
IPreview $previewGenerator,
IConfig $config,
IDBConnection $connection,
IManager $encryptionManager) {
parent::__construct();
$this->userManager = $userManager;
$this->rootFolder = $rootFolder;
$this->previewGenerator = $previewGenerator;
$this->config = $config;
$this->connection = $connection;
$this->encryptionManager = $encryptionManager;
}
@@ -142,38 +148,77 @@ private function generatePathPreviews(IUser $user, string $path) {
return;
}
$pathFolder = $userFolder->get($relativePath);
$this->parseFolder($pathFolder);
$this->processFolder($pathFolder, $user);
}
private function generateUserPreviews(IUser $user) {
\OC_Util::tearDownFS();
\OC_Util::setupFS($user->getUID());
$userFolder = $this->rootFolder->getUserFolder($user->getUID());
$this->parseFolder($userFolder);
$this->processFolder($userFolder, $user);
}
private function parseFolder(Folder $folder) {
private function processFolder(Folder $folder, IUser $user) {
// Respect the '.nomedia' file. If present don't traverse the folder
if ($folder->nodeExists('.nomedia')) {
$this->output->writeln('Skipping folder ' . $folder->getPath());
return;
}
// random sleep between 0 and 50ms to avoid collision between 2 processes
usleep(rand(0,50000));
$this->output->writeln('Scanning folder ' . $folder->getPath());
$nodes = $folder->getDirectoryListing();
foreach ($nodes as $node) {
if ($node instanceof Folder) {
$this->parseFolder($node);
$this->processFolder($node, $user);
} else if ($node instanceof File) {
$this->parseFile($node);
$is_locked = false;
$qb = $this->connection->getQueryBuilder();
$row = $qb->select('*')
->from('preview_generation')
->where($qb->expr()->eq('file_id', $qb->createNamedParameter($node->getId())))
->setMaxResults(1)
->execute()
->fetch();
if ($row !== false) {
if ($row['locked'] == 1) {
// already being processed
$is_locked = true;
} else {
$qb->update('preview_generation')
->where($qb->expr()->eq('file_id', $qb->createNamedParameter($node->getId())))
->set('locked', $qb->createNamedParameter(true))
->execute();
}
} else {
$qb->insert('preview_generation')
->values([
'uid' => $qb->createNamedParameter($user->getUID()),
'file_id' => $qb->createNamedParameter($node->getId()),
'locked' => $qb->createNamedParameter(true),
])
->execute();
}
if ($is_locked === false) {
try {
$this->processFile($node);
} finally {
$qb->delete('preview_generation')
->where($qb->expr()->eq('file_id', $qb->createNamedParameter($node->getId())))
->execute();
}
}
}
}
}
private function parseFile(File $file) {
private function processFile(File $file) {
if ($this->previewGenerator->isMimeSupported($file->getMimeType())) {
if ($this->output->getVerbosity() > OutputInterface::VERBOSITY_VERBOSE) {
$this->output->writeln('Generating previews for ' . $file->getPath());
@@ -116,13 +116,6 @@ protected function execute(InputInterface $input, OutputInterface $output) {
return 1;
}
if ($this->checkAlreadyRunning()) {
$output->writeln('Command is already running.');
return 2;
}
$this->setPID();
// Set timestamp output
$formatter = new TimestampFormatter($this->config, $output->getFormatter());
$output->setFormatter($formatter);
@@ -131,37 +124,36 @@ protected function execute(InputInterface $input, OutputInterface $output) {
$this->sizes = SizeHelper::calculateSizes($this->config);
$this->startProcessing();
$this->clearPID();
return 0;
}
private function startProcessing() {
// random sleep between 0 and 50ms to avoid collision between 2 processes
usleep(rand(0,50000));
while(true) {
$qb = $this->connection->getQueryBuilder();
$qb->select('*')
$row = $qb->select('*')
->from('preview_generation')
->orderBy('id')
->setMaxResults(1000);
$cursor = $qb->execute();
$rows = $cursor->fetchAll();
$cursor->closeCursor();
->where($qb->expr()->eq('locked', $qb->createNamedParameter(false)))
->setMaxResults(1)
->execute()
->fetch();
if ($rows === []) {
if ($row === false) {
break;
}
foreach ($rows as $row) {
/*
* First delete the row so that if preview generation fails for some reason
* the next run can just continue
*/
$qb = $this->connection->getQueryBuilder();
$qb->delete('preview_generation')
->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])));
$qb->execute();
$qb->update('preview_generation')
->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])))
->set('locked', $qb->createNamedParameter(true))
->execute();
try {
$this->processRow($row);
} finally {
$qb->delete('preview_generation')
->where($qb->expr()->eq('id', $qb->createNamedParameter($row['id'])))
->execute();
}
}
}
@@ -215,40 +207,15 @@ private function processFile(File $file) {
$this->previewGenerator->getPreview($file, $width, -1, false);
}
} catch (NotFoundException $e) {
// Maybe log that previews could not be generated?
if ($this->output->getVerbosity() > OutputInterface::VERBOSITY_VERBOSE) {
$error = $e->getMessage();
$this->output->writeln("<error>${error} " . $file->getPath() . " not found.</error>");
}
} catch (\InvalidArgumentException $e) {
$error = $e->getMessage();
$this->output->writeln("<error>${error}</error>");
}
}
}
private function setPID() {
$this->config->setAppValue($this->appName, 'pid', posix_getpid());
}
private function clearPID() {
$this->config->deleteAppValue($this->appName, 'pid');
}
private function getPID() {
return (int)$this->config->getAppValue($this->appName, 'pid', -1);
}
private function checkAlreadyRunning() {
$pid = $this->getPID();
// No PID set so just continue
if ($pid === -1) {
return false;
}
// Get get the gid of non running processes so continue
if (posix_getpgid($pid) === false) {
return false;
}
// Seems there is already a running process generating previews
return true;
}
}
@@ -0,0 +1,53 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyleft (c) 2019, Ignacio Nunez <nacho@ownyourbits.com>
*
* @author Ignacio Nunez <nacho@ownyourbits.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OCA\PreviewGenerator\Migration;
use OCP\DB\ISchemaWrapper;
use OCP\Migration\SimpleMigrationStep;
use OCP\Migration\IOutput;
use Doctrine\DBAL\Types\Type;
class Version020200Date20190608205303 extends SimpleMigrationStep {
/**
* @param IOutput $output
* @param \Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
* @param array $options
* @return null|ISchemaWrapper
*/
public function changeSchema(IOutput $output, \Closure $schemaClosure, array $options) {
/** @var ISchemaWrapper $schema */
$schema = $schemaClosure();
$table = $schema->getTable('preview_generation');
if (!$table->hasColumn('locked')) {
$table->addColumn('locked', Type::BOOLEAN, [
'notnull' => true,
'default' => 0,
]);
}
return $schema;
}
}
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.