[HttpClient] Added RetryHttpClient

symfony · Sep 17, 2020 · 712ac59 · 712ac59
1 parent f1f37a8
commit 712ac59
Show file tree

Hide file tree

Showing 19 changed files with 662 additions and 6 deletions.
diff --git a/src/Symfony/Bundle/FrameworkBundle/CHANGELOG.md b/src/Symfony/Bundle/FrameworkBundle/CHANGELOG.md
@@ -10,6 +10,7 @@ CHANGELOG
    `cache_clearer`, `filesystem` and `validator` services to private.
  * Added `TemplateAwareDataCollectorInterface` and `AbstractDataCollector` to simplify custom data collector creation and leverage autoconfiguration
  * Add `cache.adapter.redis_tag_aware` tag to use `RedisCacheAwareAdapter`
+ * added `framework.http_client.retry_failing` configuration tree
 
 5.1.0
 -----

diff --git a/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php b/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/Configuration.php
@@ -17,6 +17,7 @@
 use Symfony\Bundle\FullStack;
 use Symfony\Component\Asset\Package;
 use Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition;
+use Symfony\Component\Config\Definition\Builder\NodeBuilder;
 use Symfony\Component\Config\Definition\Builder\TreeBuilder;
 use Symfony\Component\Config\Definition\ConfigurationInterface;
 use Symfony\Component\Config\Definition\Exception\InvalidConfigurationException;
@@ -1367,6 +1368,25 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
                     ->info('HTTP Client configuration')
                     ->{!class_exists(FullStack::class) && class_exists(HttpClient::class) ? 'canBeDisabled' : 'canBeEnabled'}()
                     ->fixXmlConfig('scoped_client')
+                    ->beforeNormalization()
+                        ->always(function ($config) {
+                            if (empty($config['scoped_clients']) || !\is_array($config['default_options']['retry_failed'] ?? null)) {
+                                return $config;
+                            }
+
+                            foreach ($config['scoped_clients'] as &$scopedConfig) {
+                                if (!isset($scopedConfig['retry_failed']) || true === $scopedConfig['retry_failed']) {
+                                    $scopedConfig['retry_failed'] = $config['default_options']['retry_failed'];
+                                    continue;
+                                }
+                                if (\is_array($scopedConfig['retry_failed'])) {
+                                    $scopedConfig['retry_failed'] = $scopedConfig['retry_failed'] + $config['default_options']['retry_failed'];
+                                }
+                            }
+
+                            return $config;
+                        })
+                    ->end()
                     ->children()
                         ->integerNode('max_host_connections')
                             ->info('The maximum number of connections to a single host.')
@@ -1452,6 +1472,7 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
                                         ->variableNode('md5')->end()
                                     ->end()
                                 ->end()
+                                ->append($this->addHttpClientRetrySection())
                             ->end()
                         ->end()
                         ->scalarNode('mock_response_factory')
@@ -1594,6 +1615,7 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
                                             ->variableNode('md5')->end()
                                         ->end()
                                     ->end()
+                                    ->append($this->addHttpClientRetrySection())
                                 ->end()
                             ->end()
                         ->end()
@@ -1603,6 +1625,50 @@ private function addHttpClientSection(ArrayNodeDefinition $rootNode)
         ;
     }
 
+    private function addHttpClientRetrySection()
+    {
+        $root = new NodeBuilder();
+
+        return $root
+            ->arrayNode('retry_failed')
+                ->fixXmlConfig('http_code')
+                ->canBeEnabled()
+                ->addDefaultsIfNotSet()
+                ->beforeNormalization()
+                    ->always(function ($v) {
+                        if (isset($v['backoff_service']) && (isset($v['delay']) || isset($v['multiplier']) || isset($v['max_delay']))) {
+                            throw new \InvalidArgumentException('The "backoff_service" option cannot be used along with the "delay", "multiplier" or "max_delay" options.');
+                        }
+                        if (isset($v['decider_service']) && (isset($v['http_codes']))) {
+                            throw new \InvalidArgumentException('The "decider_service" option cannot be used along with the "http_codes" options.');
+                        }
+
+                        return $v;
+                    })
+                ->end()
+                ->children()
+                    ->scalarNode('backoff_service')->defaultNull()->info('service id to override the retry backoff')->end()
+                    ->scalarNode('decider_service')->defaultNull()->info('service id to override the retry decider')->end()
+                    ->arrayNode('http_codes')
+                        ->performNoDeepMerging()
+                        ->beforeNormalization()
+                            ->ifArray()
+                            ->then(function ($v) {
+                                return array_filter(array_values($v));
+                            })
+                        ->end()
+                        ->prototype('integer')->end()
+                        ->info('A list of HTTP status code that triggers a retry')
+                        ->defaultValue([423, 425, 429, 500, 502, 503, 504, 507, 510])
+                    ->end()
+                    ->integerNode('max_retries')->defaultValue(3)->min(0)->end()
+                    ->integerNode('delay')->defaultValue(1000)->min(0)->info('Time in ms to delay (or the initial value when multiplier is used)')->end()
+                    ->floatNode('multiplier')->defaultValue(2)->min(1)->info('If greater than 1, delay will grow exponentially for each retry: (delay * (multiple ^ retries))')->end()
+                    ->integerNode('max_delay')->defaultValue(0)->min(0)->info('Max time in ms that a retry should ever be delayed (0 = infinite)')->end()
+                ->end()
+            ;
+    }
+
     private function addMailerSection(ArrayNodeDefinition $rootNode)
     {
         $rootNode

diff --git a/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php b/src/Symfony/Bundle/FrameworkBundle/DependencyInjection/FrameworkExtension.php
@@ -64,6 +64,7 @@
 use Symfony\Component\Form\FormTypeGuesserInterface;
 use Symfony\Component\Form\FormTypeInterface;
 use Symfony\Component\HttpClient\MockHttpClient;
+use Symfony\Component\HttpClient\RetryableHttpClient;
 use Symfony\Component\HttpClient\ScopingHttpClient;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpKernel\CacheClearer\CacheClearerInterface;
@@ -1979,7 +1980,10 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
     {
         $loader->load('http_client.php');
 
-        $container->getDefinition('http_client')->setArguments([$config['default_options'] ?? [], $config['max_host_connections'] ?? 6]);
+        $options = $config['default_options'] ?? [];
+        $retryOptions = $options['retry_failed'] ?? ['enabled' => false];
+        unset($options['retry_failed']);
+        $container->getDefinition('http_client')->setArguments([$options, $config['max_host_connections'] ?? 6]);
 
         if (!$hasPsr18 = interface_exists(ClientInterface::class)) {
             $container->removeDefinition('psr18.http_client');
@@ -1990,15 +1994,20 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
             $container->removeDefinition(HttpClient::class);
         }
 
-        $httpClientId = $this->isConfigEnabled($container, $profilerConfig) ? '.debug.http_client.inner' : 'http_client';
+        if ($this->isConfigEnabled($container, $retryOptions)) {
+            $this->registerHttpClientRetry($retryOptions, 'http_client', $container);
+        }
 
+        $httpClientId = $retryOptions['enabled'] ?? false ? 'http_client.retry.inner' : ($this->isConfigEnabled($container, $profilerConfig) ? '.debug.http_client.inner' : 'http_client');
         foreach ($config['scoped_clients'] as $name => $scopeConfig) {
             if ('http_client' === $name) {
                 throw new InvalidArgumentException(sprintf('Invalid scope name: "%s" is reserved.', $name));
             }
 
             $scope = $scopeConfig['scope'] ?? null;
             unset($scopeConfig['scope']);
+            $retryOptions = $scopeConfig['retry_failed'] ?? ['enabled' => false];
+            unset($scopeConfig['retry_failed']);
 
             if (null === $scope) {
                 $baseUri = $scopeConfig['base_uri'];
@@ -2016,6 +2025,10 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
                 ;
             }
 
+            if ($this->isConfigEnabled($container, $retryOptions)) {
+                $this->registerHttpClientRetry($retryOptions, $name, $container);
+            }
+
             $container->registerAliasForArgument($name, HttpClientInterface::class);
 
             if ($hasPsr18) {
@@ -2033,6 +2046,44 @@ private function registerHttpClientConfiguration(array $config, ContainerBuilder
         }
     }
 
+    private function registerHttpClientRetry(array $retryOptions, string $name, ContainerBuilder $container)
+    {
+        if (!class_exists(RetryableHttpClient::class)) {
+            throw new LogicException('Retry failed request support cannot be enabled as version 5.2+ of the HTTP Client component is required.');
+        }
+
+        if (null !== $retryOptions['backoff_service']) {
+            $backoffReference = new Reference($retryOptions['backoff_service']);
+        } else {
+            $retryServiceId = $name.'.retry.exponential_backoff';
+            $retryDefinition = new ChildDefinition('http_client.retry.abstract_exponential_backoff');
+            $retryDefinition
+                ->replaceArgument(0, $retryOptions['delay'])
+                ->replaceArgument(1, $retryOptions['multiplier'])
+                ->replaceArgument(2, $retryOptions['max_delay']);
+            $container->setDefinition($retryServiceId, $retryDefinition);
+
+            $backoffReference = new Reference($retryServiceId);
+        }
+        if (null !== $retryOptions['decider_service']) {
+            $deciderReference = new Reference($retryOptions['decider_service']);
+        } else {
+            $retryServiceId = $name.'.retry.decider';
+            $retryDefinition = new ChildDefinition('http_client.retry.abstract_httpstatuscode_decider');
+            $retryDefinition
+                ->replaceArgument(0, $retryOptions['http_codes']);
+            $container->setDefinition($retryServiceId, $retryDefinition);
+
+            $deciderReference = new Reference($retryServiceId);
+        }
+
+        $container
+            ->register($name.'.retry', RetryableHttpClient::class)
+            ->setDecoratedService($name)
+            ->setArguments([new Reference($name.'.retry.inner'), $deciderReference, $backoffReference, $retryOptions['max_retries'], new Reference('logger')])
+            ->addTag('monolog.logger', ['channel' => 'http_client']);
+    }
+
     private function registerMailerConfiguration(array $config, ContainerBuilder $container, PhpFileLoader $loader)
     {
         if (!class_exists(Mailer::class)) {

diff --git a/src/Symfony/Bundle/FrameworkBundle/Resources/config/http_client.php b/src/Symfony/Bundle/FrameworkBundle/Resources/config/http_client.php
@@ -17,6 +17,8 @@
 use Symfony\Component\HttpClient\HttpClient;
 use Symfony\Component\HttpClient\HttplugClient;
 use Symfony\Component\HttpClient\Psr18Client;
+use Symfony\Component\HttpClient\Retry\ExponentialBackOff;
+use Symfony\Component\HttpClient\Retry\HttpStatusCodeDecider;
 use Symfony\Contracts\HttpClient\HttpClientInterface;
 
 return static function (ContainerConfigurator $container) {
@@ -48,5 +50,19 @@
                 service(ResponseFactoryInterface::class)->ignoreOnInvalid(),
                 service(StreamFactoryInterface::class)->ignoreOnInvalid(),
             ])
+
+        // retry
+        ->set('http_client.retry.abstract_exponential_backoff', ExponentialBackOff::class)
+            ->abstract()
+            ->args([
+                abstract_arg('delay ms'),
+                abstract_arg('multiplier'),
+                abstract_arg('max delay ms'),
+            ])
+        ->set('http_client.retry.abstract_httpstatuscode_decider', HttpStatusCodeDecider::class)
+            ->abstract()
+            ->args([
+                abstract_arg('http codes'),
+            ])
     ;
 };
diff --git a/src/Symfony/Bundle/FrameworkBundle/Resources/config/schema/symfony-1.0.xsd b/src/Symfony/Bundle/FrameworkBundle/Resources/config/schema/symfony-1.0.xsd
@@ -519,6 +519,7 @@
             <xsd:element name="resolve" type="http_resolve" minOccurs="0" maxOccurs="unbounded" />
             <xsd:element name="header" type="http_header" minOccurs="0" maxOccurs="unbounded" />
             <xsd:element name="peer-fingerprint" type="fingerprint" minOccurs="0" maxOccurs="unbounded" />
+            <xsd:element name="retry-failed" type="http_client_retry_failed" minOccurs="0" maxOccurs="1" />
         </xsd:choice>
         <xsd:attribute name="max-redirects" type="xsd:integer" />
         <xsd:attribute name="http-version" type="xsd:string" />
@@ -535,7 +536,6 @@
         <xsd:attribute name="local-pk" type="xsd:string" />
         <xsd:attribute name="passphrase" type="xsd:string" />
         <xsd:attribute name="ciphers" type="xsd:string" />
-
     </xsd:complexType>
 
     <xsd:complexType name="http_client_scope_options" mixed="true">
@@ -544,6 +544,7 @@
             <xsd:element name="resolve" type="http_resolve" minOccurs="0" maxOccurs="unbounded" />
             <xsd:element name="header" type="http_header" minOccurs="0" maxOccurs="unbounded" />
             <xsd:element name="peer-fingerprint" type="fingerprint" minOccurs="0" maxOccurs="unbounded" />
+            <xsd:element name="retry-failed" type="http_client_retry_failed" minOccurs="0" maxOccurs="1" />
         </xsd:choice>
         <xsd:attribute name="name" type="xsd:string" />
         <xsd:attribute name="scope" type="xsd:string" />
@@ -574,6 +575,20 @@
         </xsd:choice>
     </xsd:complexType>
 
+    <xsd:complexType name="http_client_retry_failed">
+        <xsd:sequence>
+            <xsd:element name="http-code" type="xsd:integer" minOccurs="0" maxOccurs="unbounded" />
+        </xsd:sequence>
+        <xsd:attribute name="enabled" type="xsd:boolean" />
+        <xsd:attribute name="backoff-service" type="xsd:string" />
+        <xsd:attribute name="decider-service" type="xsd:string" />
+        <xsd:attribute name="max-retries" type="xsd:integer" />
+        <xsd:attribute name="delay" type="xsd:integer" />
+        <xsd:attribute name="multiplier" type="xsd:float" />
+        <xsd:attribute name="max-delay" type="xsd:float" />
+        <xsd:attribute name="response_header" type="xsd:boolean" />
+    </xsd:complexType>
+
     <xsd:complexType name="http_query" mixed="true">
         <xsd:attribute name="key" type="xsd:string" />
     </xsd:complexType>

diff --git a/...mfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/php/http_client_retry.php b/...mfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/php/http_client_retry.php
@@ -0,0 +1,23 @@
+<?php
+
+$container->loadFromExtension('framework', [
+    'http_client' => [
+        'default_options' => [
+            'retry_failed' => [
+                'backoff_service' => null,
+                'decider_service' => null,
+                'http_codes' => [429, 500],
+                'max_retries' => 2,
+                'delay' => 100,
+                'multiplier' => 2,
+                'max_delay' => 0,
+            ]
+        ],
+        'scoped_clients' => [
+            'foo' => [
+                'base_uri' => 'http://example.com',
+                'retry_failed' => ['multiplier' => 4],
+            ],
+        ],
+    ],
+]);
diff --git a/...mfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/xml/http_client_retry.xml b/...mfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/xml/http_client_retry.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<container xmlns="http://symfony.com/schema/dic/services"
+           xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+           xmlns:framework="http://symfony.com/schema/dic/symfony"
+           xsi:schemaLocation="http://symfony.com/schema/dic/services http://symfony.com/schema/dic/services/services-1.0.xsd
+        http://symfony.com/schema/dic/symfony http://symfony.com/schema/dic/symfony/symfony-1.0.xsd">
+
+    <framework:config>
+        <framework:http-client>
+            <framework:default-options>
+                <framework:retry-failed
+                        delay="100"
+                        max-delay="0"
+                        max-retries="2"
+                        multiplier="2">
+                    <framework:http-code>429</framework:http-code>
+                    <framework:http-code>500</framework:http-code>
+                </framework:retry-failed>
+            </framework:default-options>
+            <framework:scoped-client name="foo" base-uri="http://example.com">
+                <framework:retry-failed multiplier="4"/>
+            </framework:scoped-client>
+        </framework:http-client>
+    </framework:config>
+</container>
diff --git a/...mfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/yml/http_client_retry.yml b/...mfony/Bundle/FrameworkBundle/Tests/DependencyInjection/Fixtures/yml/http_client_retry.yml
@@ -0,0 +1,16 @@
+framework:
+    http_client:
+        default_options:
+            retry_failed:
+                backoff_service: null
+                decider_service: null
+                http_codes: [429, 500]
+                max_retries: 2
+                delay: 100
+                multiplier: 2
+                max_delay: 0
+        scoped_clients:
+            foo:
+                base_uri: http://example.com
+                retry_failed:
+                    multiplier: 4
diff --git a/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTest.php b/src/Symfony/Bundle/FrameworkBundle/Tests/DependencyInjection/FrameworkExtensionTest.php
@@ -36,11 +36,13 @@
 use Symfony\Component\DependencyInjection\ContainerBuilder;
 use Symfony\Component\DependencyInjection\ContainerInterface;
 use Symfony\Component\DependencyInjection\Definition;
+use Symfony\Component\DependencyInjection\Exception\LogicException;
 use Symfony\Component\DependencyInjection\Loader\ClosureLoader;
 use Symfony\Component\DependencyInjection\ParameterBag\EnvPlaceholderParameterBag;
 use Symfony\Component\DependencyInjection\Reference;
 use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 use Symfony\Component\HttpClient\MockHttpClient;
+use Symfony\Component\HttpClient\RetryableHttpClient;
 use Symfony\Component\HttpClient\ScopingHttpClient;
 use Symfony\Component\HttpKernel\DependencyInjection\LoggerPass;
 use Symfony\Component\Messenger\Transport\TransportFactory;
@@ -1482,6 +1484,23 @@ public function testHttpClientOverrideDefaultOptions()
         $this->assertSame($expected, $container->getDefinition('foo')->getArgument(2));
     }
 
+    public function testHttpClientRetry()
+    {
+        if (!class_exists(RetryableHttpClient::class)) {
+            $this->expectException(LogicException::class);
+        }
+        $container = $this->createContainerFromFile('http_client_retry');
+
+        $this->assertSame([429, 500], $container->getDefinition('http_client.retry.decider')->getArgument(0));
+        $this->assertSame(100, $container->getDefinition('http_client.retry.exponential_backoff')->getArgument(0));
+        $this->assertSame(2, $container->getDefinition('http_client.retry.exponential_backoff')->getArgument(1));
+        $this->assertSame(0, $container->getDefinition('http_client.retry.exponential_backoff')->getArgument(2));
+        $this->assertSame(2, $container->getDefinition('http_client.retry')->getArgument(3));
+
+        $this->assertSame(RetryableHttpClient::class, $container->getDefinition('foo.retry')->getClass());
+        $this->assertSame(4, $container->getDefinition('foo.retry.exponential_backoff')->getArgument(1));
+    }
+
     public function testHttpClientWithQueryParameterKey()
     {
         $container = $this->createContainerFromFile('http_client_xml_key');

diff --git a/src/Symfony/Component/HttpClient/CHANGELOG.md b/src/Symfony/Component/HttpClient/CHANGELOG.md
@@ -10,6 +10,7 @@ CHANGELOG
  * added `MockResponse::getRequestMethod()` and `getRequestUrl()` to allow inspecting which request has been sent
  * added `EventSourceHttpClient` a Server-Sent events stream implementing the [EventSource specification](https://www.w3.org/TR/eventsource/#eventsource)
  * added option "extra.curl" to allow setting additional curl options in `CurlHttpClient`
+ * added `RetryableHttpClient` to automatically retry failed HTTP requests.
 
 5.1.0
 -----