Skip to content

Commit

Permalink
[POC] Introduce background worker
Browse files Browse the repository at this point in the history
  • Loading branch information
Toflar committed Oct 18, 2022
1 parent 09696f5 commit c347813
Show file tree
Hide file tree
Showing 27 changed files with 558 additions and 120 deletions.
2 changes: 2 additions & 0 deletions composer.json
Expand Up @@ -109,6 +109,7 @@
"symfony/dependency-injection": "^5.4 || ^6.0",
"symfony/deprecation-contracts": "^2.1 || ^3.0",
"symfony/doctrine-bridge": "^5.4 || ^6.0",
"symfony/doctrine-messenger": "^5.4 || ^6.0",
"symfony/dom-crawler": "^5.4 || ^6.0",
"symfony/dotenv": "^5.4 || ^6.0",
"symfony/error-handler": "^5.4 || ^6.0",
Expand All @@ -126,6 +127,7 @@
"symfony/lock": "^5.4 || ^6.0",
"symfony/mailer": "^5.4 || ^6.0",
"symfony/maker-bundle": "^1.1",
"symfony/messenger": "^5.4 || ^6.0",
"symfony/mime": "^5.4 || ^6.0",
"symfony/monolog-bridge": "^5.4 || ^6.0",
"symfony/monolog-bundle": "^3.1",
Expand Down
2 changes: 2 additions & 0 deletions core-bundle/composer.json
Expand Up @@ -105,6 +105,7 @@
"symfony/dependency-injection": "^5.4 || ^6.0",
"symfony/deprecation-contracts": "^2.1 || ^3.0",
"symfony/doctrine-bridge": "^5.4 || ^6.0",
"symfony/doctrine-messenger": "^5.4 || ^6.0",
"symfony/dom-crawler": "^5.4 || ^6.0",
"symfony/event-dispatcher": "^5.4 || ^6.0",
"symfony/event-dispatcher-contracts": "^2.0 || ^3.0",
Expand All @@ -119,6 +120,7 @@
"symfony/intl": "^5.4 || ^6.0",
"symfony/lock": "^5.4 || ^6.0",
"symfony/mailer": "^5.4 || ^6.0",
"symfony/messenger": "^5.4 || ^6.0",
"symfony/mime": "^5.4 || ^6.0",
"symfony/monolog-bridge": "^5.4 || ^6.0",
"symfony/password-hasher": "^5.4 || ^6.0",
Expand Down
4 changes: 2 additions & 2 deletions core-bundle/config/listener.yaml
Expand Up @@ -37,7 +37,6 @@ services:
class: Contao\CoreBundle\EventListener\CommandSchedulerListener
arguments:
- '@contao.cron'
- '@contao.framework'
- '@database_connection'
- '%fragment.path%'
tags:
Expand Down Expand Up @@ -205,6 +204,7 @@ services:
class: Contao\CoreBundle\EventListener\DoctrineSchemaListener
arguments:
- '@contao.doctrine.dca_schema_provider'
- '@messenger.receiver_locator'
tags:
- { name: doctrine.event_listener, event: postGenerateSchema }

Expand Down Expand Up @@ -463,7 +463,7 @@ services:
contao.listener.search_index:
class: Contao\CoreBundle\EventListener\SearchIndexListener
arguments:
- '@contao.search.indexer'
- '@messenger.bus.default'
- '%fragment.path%'
tags:
- kernel.event_listener
Expand Down
15 changes: 15 additions & 0 deletions core-bundle/config/services.yaml
Expand Up @@ -95,8 +95,12 @@ services:
arguments:
- !service_closure '@contao.repository.cron_job'
- !service_closure '@doctrine.orm.entity_manager'
- '@cache.app'
- '@?logger'

contao.cron.messenger:
class: Contao\CoreBundle\Cron\MessengerCron

contao.cron.purge_expired_data:
class: Contao\CoreBundle\Cron\PurgeExpiredDataCron
arguments:
Expand Down Expand Up @@ -453,6 +457,17 @@ services:
arguments:
- '@contao.menu.matcher'

contao.messenger.message_handler.search_index:
class: Contao\CoreBundle\Messenger\MessageHandler\SearchIndexMessageHandler
arguments:
- '@contao.search.indexer'

contao.messenger.transport.cron_fallback_transportFactory:
class: Contao\CoreBundle\Messenger\Transport\CronFallbackTransportFactory
arguments:
- '@contao.cron'
- '@messenger.receiver_locator'

contao.model_argument_resolver:
class: Contao\CoreBundle\HttpKernel\ModelArgumentResolver
arguments:
Expand Down
1 change: 0 additions & 1 deletion core-bundle/contao/config/default.php
Expand Up @@ -100,7 +100,6 @@
$GLOBALS['TL_CONFIG']['backendTheme'] = 'flexible';
$GLOBALS['TL_CONFIG']['doNotCollapse'] = false;
$GLOBALS['TL_CONFIG']['minPasswordLength'] = 8;
$GLOBALS['TL_CONFIG']['disableCron'] = false;
$GLOBALS['TL_CONFIG']['defaultFileChmod'] = 0644;
$GLOBALS['TL_CONFIG']['defaultFolderChmod'] = 0755;
$GLOBALS['TL_CONFIG']['maxPaginationLinks'] = 7;
44 changes: 44 additions & 0 deletions core-bundle/src/Cron/AbstractConsoleCron.php
@@ -0,0 +1,44 @@
<?php

declare(strict_types=1);

namespace Contao\CoreBundle\Cron;

use Symfony\Component\Process\PhpExecutableFinder;
use Symfony\Component\Process\Process;

abstract class AbstractConsoleCron
{
private string|null $phpBinary = null;

public function __construct(private string $consolePath)
{
}

protected function createProcess(string $command, string ...$commandArguments): Process
{
$arguments = [];
$arguments[] = $this->getPhpBinary();
$arguments[] = $this->consolePath;
$arguments = array_merge($arguments, $this->getPhpArguments());
$arguments[] = $command;
$arguments = array_merge($arguments, $commandArguments);

return new Process($arguments);
}

protected function getPhpArguments(): array
{
return [];
}

private function getPhpBinary(): string
{
if (null === $this->phpBinary) {
$executableFinder = new PhpExecutableFinder();
$this->phpBinary = $executableFinder->find();
}

return $this->phpBinary;
}
}
17 changes: 16 additions & 1 deletion core-bundle/src/Cron/Cron.php
Expand Up @@ -16,10 +16,12 @@
use Contao\CoreBundle\Repository\CronJobRepository;
use Cron\CronExpression;
use Doctrine\ORM\EntityManagerInterface;
use Psr\Cache\CacheItemPoolInterface;
use Psr\Log\LoggerInterface;

class Cron
{
private const MINUTELY_CACHE_KEY = 'contao.cron.minutely_run';
final public const SCOPE_WEB = 'web';
final public const SCOPE_CLI = 'cli';

Expand All @@ -32,7 +34,7 @@ class Cron
* @param \Closure():CronJobRepository $repository
* @param \Closure():EntityManagerInterface $entityManager
*/
public function __construct(private \Closure $repository, private \Closure $entityManager, private LoggerInterface|null $logger = null)
public function __construct(private \Closure $repository, private \Closure $entityManager, private CacheItemPoolInterface $cachePool, private LoggerInterface|null $logger = null)
{
}

Expand All @@ -49,6 +51,13 @@ public function getCronJobs(): array
return $this->cronJobs;
}

public function hasMinutelyCliCron(): bool
{
$item = $this->cachePool->getItem(self::MINUTELY_CACHE_KEY);

return $item->isHit();
}

/**
* Run all the registered Contao cron jobs.
*/
Expand Down Expand Up @@ -83,6 +92,12 @@ private function doRun(array $cronJobs, string $scope, bool $force = false): voi
throw new \InvalidArgumentException('Invalid scope "'.$scope.'"');
}

if (self::SCOPE_CLI === $scope) {
$cacheItem = $this->cachePool->getItem(self::MINUTELY_CACHE_KEY);
$cacheItem->expiresAfter(60); // 60 seconds
$this->cachePool->save($cacheItem);
}

/** @var CronJobRepository $repository */
$repository = ($this->repository)();

Expand Down
71 changes: 71 additions & 0 deletions core-bundle/src/Cron/MessengerCron.php
@@ -0,0 +1,71 @@
<?php

declare(strict_types=1);

/*
* This file is part of Contao.
*
* (c) Leo Feyer
*
* @license LGPL-3.0-or-later
*/

namespace Contao\CoreBundle\Cron;

use Contao\CoreBundle\DependencyInjection\Attribute\AsCronJob;

#[AsCronJob('minutely')]
class MessengerCron extends AbstractConsoleCron
{
public function __construct(string $consolePath, private int $numberOfWorkers = 0)
{
parent::__construct($consolePath);
}

public function __invoke(string $scope): void
{
if (Cron::SCOPE_WEB === $scope || $this->numberOfWorkers < 1) {
return;
}

$processes = [];

for ($i = 0; $i < $this->numberOfWorkers; ++$i) {
$process = $this->createProcess(
'messenger:consume',
'--time-limit=60', // Minutely cronjob running for one minute max
'contao_prio_high',
'contao_prio_normal',
'contao_prio_low'
);
$process->setTimeout(65);

// Start the job asynchronously
$process->start();
$processes[] = $process;
}

// Now we need to sleep to keep the parent process open. Otherwise, this script will end and thus kill
// our child processes.
// All jobs run for 60 seconds, so we don't need to check every second yet
sleep(55);

// Now we check every second if all processes are done
while (true) {
$allDone = true;

foreach ($processes as $process) {
if ($process->isRunning()) {
$allDone = false;
break;
}
}

if ($allDone) {
break;
}

sleep(1);
}
}
}
19 changes: 19 additions & 0 deletions core-bundle/src/DependencyInjection/Configuration.php
Expand Up @@ -104,6 +104,7 @@ static function (array $options): array {
->always(static fn (string $value): string => Path::canonicalize($value))
->end()
->end()
->append($this->addWorkerNode())
->append($this->addImageNode())
->append($this->addSecurityNode())
->append($this->addSearchNode())
Expand All @@ -119,6 +120,24 @@ static function (array $options): array {
return $treeBuilder;
}

private function addWorkerNode(): NodeDefinition
{
return (new TreeBuilder('worker'))
->getRootNode()
->addDefaultsIfNotSet()
->children()
->scalarNode('console_path')
->info('The path to the Symfony console.')
->defaultValue('%kernel.project_dir%/bin/console')
->end()
->scalarNode('quantity')
->info('The number of workers to run. Use 0 to disable the worker.')
->defaultValue(1)
->end()
->end()
;
}

private function addImageNode(): NodeDefinition
{
return (new TreeBuilder('image'))
Expand Down
12 changes: 12 additions & 0 deletions core-bundle/src/DependencyInjection/ContaoCoreExtension.php
Expand Up @@ -132,6 +132,7 @@ public function load(array $configs, ContainerBuilder $container): void
$container->setParameter('contao.insert_tags.allowed_tags', $config['insert_tags']['allowed_tags']);
$container->setParameter('contao.sanitizer.allowed_url_protocols', $config['sanitizer']['allowed_url_protocols']);

$this->handleWorkerConfig($config, $container);
$this->handleSearchConfig($config, $container);
$this->handleCrawlConfig($config, $container);
$this->setPredefinedImageSizes($config, $container);
Expand Down Expand Up @@ -223,6 +224,17 @@ public function configureFilesystem(FilesystemConfiguration $config): void
;
}

private function handleWorkerConfig(array $config, ContainerBuilder $container): void
{
if (!$container->hasDefinition('contao.cron.messenger')) {
return;
}

$cron = $container->getDefinition('contao.cron.messenger');
$cron->setArgument(0, $config['worker']['console_path']);
$cron->setArgument(1, $config['worker']['quantity']);
}

private function handleSearchConfig(array $config, ContainerBuilder $container): void
{
$container
Expand Down
14 changes: 7 additions & 7 deletions core-bundle/src/EventListener/CommandSchedulerListener.php
Expand Up @@ -12,9 +12,7 @@

namespace Contao\CoreBundle\EventListener;

use Contao\Config;
use Contao\CoreBundle\Cron\Cron;
use Contao\CoreBundle\Framework\ContaoFramework;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\Exception;
use Symfony\Component\HttpFoundation\Request;
Expand All @@ -27,7 +25,6 @@ class CommandSchedulerListener
{
public function __construct(
private Cron $cron,
private ContaoFramework $framework,
private Connection $connection,
private string $fragmentPath = '_fragment',
) {
Expand All @@ -38,7 +35,12 @@ public function __construct(
*/
public function __invoke(TerminateEvent $event): void
{
if ($this->framework->isInitialized() && $this->canRunCron($event->getRequest())) {
// If we have a real minutely CLI cron, we don't need this listener.
if ($this->cron->hasMinutelyCliCron()) {
return;
}

if ($this->canRunCron($event->getRequest())) {
$this->cron->run(Cron::SCOPE_WEB);
}
}
Expand All @@ -52,9 +54,7 @@ private function canRunCron(Request $request): bool
return false;
}

$config = $this->framework->getAdapter(Config::class);

return !$config->get('disableCron') && $this->canRunDbQuery();
return $this->canRunDbQuery();
}

/**
Expand Down

0 comments on commit c347813

Please sign in to comment.