From e8069cd4f39a19f10fafc362f3020ee1c3fdb816 Mon Sep 17 00:00:00 2001 From: Maksim Kotlyar Date: Fri, 15 Dec 2017 12:23:54 +0200 Subject: [PATCH 1/2] Add ability to complerly disable doctrine listener Add queue listener doc. --- DependencyInjection/Configuration.php | 1 + DependencyInjection/FOSElasticaExtension.php | 2 +- .../doc/cookbook/doctrine-queue-listener.md | 64 +++++++++++++++++ .../doc/cookbook/speed-up-populate-command.md | 2 +- Resources/doc/index.md | 7 +- .../FOSElasticaExtensionTest.php | 69 +++++++++++++++++++ 6 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 Resources/doc/cookbook/doctrine-queue-listener.md diff --git a/DependencyInjection/Configuration.php b/DependencyInjection/Configuration.php index 475c4699f..564a635e7 100644 --- a/DependencyInjection/Configuration.php +++ b/DependencyInjection/Configuration.php @@ -537,6 +537,7 @@ protected function getPersistenceNode() ->arrayNode('listener') ->addDefaultsIfNotSet() ->children() + ->booleanNode('enabled')->defaultTrue()->end() ->scalarNode('insert')->defaultTrue()->end() ->scalarNode('update')->defaultTrue()->end() ->scalarNode('delete')->defaultTrue()->end() diff --git a/DependencyInjection/FOSElasticaExtension.php b/DependencyInjection/FOSElasticaExtension.php index 2a26e201e..ab9bac96a 100644 --- a/DependencyInjection/FOSElasticaExtension.php +++ b/DependencyInjection/FOSElasticaExtension.php @@ -334,7 +334,7 @@ private function loadTypePersistenceIntegration(array $typeConfig, ContainerBuil if (isset($typeConfig['finder'])) { $this->loadTypeFinder($typeConfig, $container, $elasticaToModelTransformerId, $typeRef, $indexName, $typeName); } - if (isset($typeConfig['listener'])) { + if (isset($typeConfig['listener']) && $typeConfig['listener']['enabled']) { $this->loadTypeListener($typeConfig, $container, $objectPersisterId, $indexName, $typeName); } } diff --git a/Resources/doc/cookbook/doctrine-queue-listener.md b/Resources/doc/cookbook/doctrine-queue-listener.md new file mode 100644 index 000000000..b74eb9445 --- /dev/null +++ b/Resources/doc/cookbook/doctrine-queue-listener.md @@ -0,0 +1,64 @@ +# Doctrine queue listener + +FOSElasticaBundle subscribes on Doctrine events, such as insert, update, remove to adjust the index accordingly. +The listener might start consuming more and more resources, most importantly time of http response. +Or, Sometimes it fails, bringing the whole your app down too, because of ElasticSearch server is out of order or some bug in the code. +Keep reading if you want to improve http response time or strive for better fault tolerance. + +Instead of doing everything in one single process the listener just sends a message to a worker (via [message queue](https://en.wikipedia.org/wiki/Message_queue)). +The work does the actual synchronization job in background. +For queuing it uses [EnqueueBundle](https://github.com/php-enqueue/enqueue-dev/blob/master/docs/bundle/quick_tour.md) which supports a lot of MQ transports out of the box. + +## Installation + +I assume you already have `FOSElasticaBundle` installed, if not here's the [setup doc](../setup.md). +So, we only have to install `EnqueueElasticaBundle` and one of the MQ transports. +I am going to install the bundle and filesystem transport by way of example. + +```bash +$ composer require enqueue/elastica-bundle:^0.8.1 enqueue/fs:^0.8 +``` + +_**Note:** As long as you are on Symfony Flex you are done. If not, you have to do some extra things, like registering the bundle in your `AppKernel` class._ + +## Usage + +The usage is simple, you have to disable the default listener: + +```yaml +fos_elastica: + indexes: + enqueue: + types: + blog: + persistence: + driver: orm + model: AppBundle\Entity\Blog + listener: { enabled: false } +``` + +and enable the queue one: + +``` +enqueue_elastica: + doctrine: + queue_listeners: + - + index_name: 'enqueue' + type_name: 'blog' + model_class: 'AppBundle\Entity\Blog' +``` + +Don't forget to run some queue consumers (the more you run the better performance you might get): + +```bash +$ ./bin/console enqueue:consume --setup-broker -vvv +``` + +or (use it only if you cannot use the solution above): + +```bash +$ ./bin/console enqueue:transport:consume enqueue_elastica.doctrine.sync_index_with_object_change_processor -vvv +``` + +[back to index](../index.md) diff --git a/Resources/doc/cookbook/speed-up-populate-command.md b/Resources/doc/cookbook/speed-up-populate-command.md index d9690e186..7dfc00c41 100644 --- a/Resources/doc/cookbook/speed-up-populate-command.md +++ b/Resources/doc/cookbook/speed-up-populate-command.md @@ -19,7 +19,7 @@ For queuing it uses [EnqueueBundle](https://github.com/php-enqueue/enqueue-dev/b ## Installation I assume you already have `FOSElasticaBundle` installed, if not here's the [setup doc](../setup.md). -So, we only have to install `EnqueueBundle` and one of the MQ transports. +So, we only have to install `EnqueueElasticaBundle` and one of the MQ transports. I am going to install the bundle and filesystem transport by way of example. ```bash diff --git a/Resources/doc/index.md b/Resources/doc/index.md index b6d3ee9f5..c6c1b7a62 100644 --- a/Resources/doc/index.md +++ b/Resources/doc/index.md @@ -18,7 +18,6 @@ Cookbook Entries * [Custom Repositories](cookbook/custom-repositories.md) * [Pre Transform Event](cookbook/pre-transform-event.md) * [HTTP Headers for Elastica](cookbook/elastica-client-http-headers.md) -* Performance - [Logging](cookbook/logging.md) - [Compression](cookbook/compression.md) * [Manual Providers](cookbook/manual-provider.md) * [Clustering - Multiple Connections](cookbook/multiple-connections.md) * [Suppressing server errors](cookbook/suppress-server-errors.md) @@ -26,4 +25,8 @@ Cookbook Entries * [Multi type search](cookbook/multi-type-search.md) * [Attachments Handling](cookbook/attachments.md) * [Populate Events](cookbook/populate-events.md) -* [Speed up populate command](cookbook/speed-up-populate-command.md) +* Performance + - [Logging](cookbook/logging.md) + - [Compression](cookbook/compression.md) + - [Speed up populate command](cookbook/speed-up-populate-command.md) + - [Doctrine queue listener](cookbook/doctrine-queue-listener.md) diff --git a/Tests/DependencyInjection/FOSElasticaExtensionTest.php b/Tests/DependencyInjection/FOSElasticaExtensionTest.php index f3bf72c89..93c7fc749 100644 --- a/Tests/DependencyInjection/FOSElasticaExtensionTest.php +++ b/Tests/DependencyInjection/FOSElasticaExtensionTest.php @@ -4,6 +4,7 @@ use Doctrine\ODM\MongoDB\DocumentManager; use FOS\ElasticaBundle\DependencyInjection\FOSElasticaExtension; +use FOS\ElasticaBundle\Doctrine\Listener; use FOS\ElasticaBundle\Doctrine\RegisterListenersService; use FOS\ElasticaBundle\Doctrine\MongoDBPagerProvider; use FOS\ElasticaBundle\Doctrine\ORMPagerProvider; @@ -521,4 +522,72 @@ public function testShouldRegisterPagerPersisterRegisterService() $this->assertSame(PagerPersisterRegistry::class, $listener->getClass()); $this->assertSame([], $listener->getArgument(0)); } + + public function testShouldRegisterDoctrineORMListener() + { + $container = new ContainerBuilder(); + $container->setParameter('kernel.debug', true); + + $extension = new FOSElasticaExtension(); + $extension->load([ + 'fos_elastica' => [ + 'clients' => [ + 'default' => ['host' => 'a_host', 'port' => 'a_port'], + ], + 'indexes' => [ + 'acme_index' => [ + 'types' => [ + 'acme_type' => [ + 'properties' => ['text' => null], + 'persistence' => [ + 'driver' => 'orm', + 'model' => 'theModelClass', + 'provider' => ['pager_provider' => true], + 'listener' => null, + 'finder' => null, + ] + ] + ] + ] + ] + ] + ], $container); + + $this->assertTrue($container->hasDefinition('fos_elastica.listener.acme_index.acme_type')); + } + + public function testShouldNotRegisterDoctrineORMListenerIfDisabled() + { + $container = new ContainerBuilder(); + $container->setParameter('kernel.debug', true); + + $extension = new FOSElasticaExtension(); + $extension->load([ + 'fos_elastica' => [ + 'clients' => [ + 'default' => ['host' => 'a_host', 'port' => 'a_port'], + ], + 'indexes' => [ + 'acme_index' => [ + 'types' => [ + 'acme_type' => [ + 'properties' => ['text' => null], + 'persistence' => [ + 'driver' => 'orm', + 'model' => 'theModelClass', + 'provider' => ['pager_provider' => true], + 'listener' => [ + 'enabled' => false, + ], + 'finder' => null, + ] + ] + ] + ] + ] + ] + ], $container); + + $this->assertFalse($container->hasDefinition('fos_elastica.listener.acme_index.acme_type')); + } } From 3c2117de27fe554098c0f2fa4e2baff9e042e47d Mon Sep 17 00:00:00 2001 From: Maksim Kotlyar Date: Fri, 15 Dec 2017 12:59:08 +0200 Subject: [PATCH 2/2] [doc][skip ci] correct index\type names. --- Resources/doc/cookbook/doctrine-queue-listener.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Resources/doc/cookbook/doctrine-queue-listener.md b/Resources/doc/cookbook/doctrine-queue-listener.md index b74eb9445..580e1a473 100644 --- a/Resources/doc/cookbook/doctrine-queue-listener.md +++ b/Resources/doc/cookbook/doctrine-queue-listener.md @@ -28,12 +28,12 @@ The usage is simple, you have to disable the default listener: ```yaml fos_elastica: indexes: - enqueue: + acme_index: types: - blog: + acme_type: persistence: - driver: orm - model: AppBundle\Entity\Blog + driver: 'orm' + model: 'AppBundle\Entity\Blog' listener: { enabled: false } ``` @@ -44,8 +44,8 @@ enqueue_elastica: doctrine: queue_listeners: - - index_name: 'enqueue' - type_name: 'blog' + index_name: 'acme_index' + type_name: 'acme_blog' model_class: 'AppBundle\Entity\Blog' ```