Feature/retry on 503

README.md

@@ -196,6 +196,19 @@ On which methods should the connection be retried in case of socket hang up.
 
 By default: `['GET', 'HEAD', 'OPTIONS', 'TRACE' ]`
 
+This plugin will always retry on 503 errors, _unless_ `retryMethods` does not contain `GET`.
+
+---
+
+#### `maxRetriesOn503`
+
+This plugin will always retry on `GET` requests that returns 503 errors, _unless_ `retryMethods` does not contain `GET`.
+
+This option set the limit on how many times the plugin should retry the request, specifically for 503 errors.
+
+By Default: 10
+
+
 ---
 
 ### `reply.from(source, [opts])`

index.js

@@ -43,6 +43,7 @@ module.exports = fp(function from (fastify, opts, next) {
     const getUpstream = opts.getUpstream || upstreamNoOp
     const onError = opts.onError || onErrorDefault
     const retriesCount = opts.retriesCount || 0
+    const maxRetriesOn503 = opts.maxRetriesOn503 || 10
 
     if (!source) {
       source = req.url
@@ -118,9 +119,8 @@ module.exports = fp(function from (fastify, opts, next) {
     const requestHeaders = rewriteRequestHeaders(req, headers)
     const contentLength = requestHeaders['content-length']
     let requestImpl
-
-    if (retriesCount && retryMethods.has(req.method) && !contentLength) {
-      requestImpl = createRequestRetry(request, this, retriesCount, retryOnError)
+    if (retryMethods.has(req.method) && !contentLength) {
+      requestImpl = createRequestRetry(request, this, retriesCount, retryOnError, maxRetriesOn503)
     } else {
       requestImpl = request
     }
@@ -213,25 +213,38 @@ function isFastifyMultipartRegistered (fastify) {
   return fastify.hasContentTypeParser('multipart') && fastify.hasRequestDecorator('multipart')
 }
 
-function createRequestRetry (requestImpl, reply, retriesCount, retryOnError) {
+function createRequestRetry (requestImpl, reply, retriesCount, retryOnError, maxRetriesOn503) {
   function requestRetry (req, cb) {
     let retries = 0
 
     function run () {
       requestImpl(req, function (err, res) {
-        if (err && !reply.sent && retriesCount > retries) {
-          if (err.code === retryOnError) {
-            retries += 1
+        // Magic number, so why not 42? We might want to make this configurable.
+        let retryAfter = 42 * Math.random() * (retries + 1)
 
-            run()
-            return
+        if (res && res.headers['retry-after']) {
+          retryAfter = res.headers['retry-after']
+        }
+        if (!reply.sent) {
+          // always retry on 503 errors
+          if (res && res.statusCode === 503 && req.method === 'GET') {
+            if (retriesCount === 0 && retries < maxRetriesOn503) {
+              // we should stop at some point
+              return retry(retryAfter)
+            }
+          } else if (retriesCount > retries && err && err.code === retryOnError) {
+            return retry(retryAfter)
           }
         }
-
         cb(err, res)
       })
     }
 
+    function retry (after) {
+      retries += 1
+      setTimeout(run, after)
+    }
+
     run()
   }
 

test/retry-on-503.js

@@ -0,0 +1,101 @@
+'use strict'
+
+const { test } = require('tap')
+const Fastify = require('fastify')
+const From = require('..')
+const http = require('http')
+const got = require('got')
+
+function createTargetServer (withRetryAfterHeader, stopAfter = 1) {
+  let requestCount = 0
+  return http.createServer((req, res) => {
+    if (requestCount++ < stopAfter) {
+      res.statusCode = 503
+      res.setHeader('Content-Type', 'text/plain')
+      if (withRetryAfterHeader) {
+        res.setHeader('Retry-After', 100)
+      }
+      return res.end('This Service is Unavailable')
+    }
+    res.statusCode = 205
+    res.setHeader('Content-Type', 'text/plain')
+    return res.end(`Hello World ${requestCount}!`)
+  })
+}
+
+test('Should retry on 503 HTTP error', async function (t) {
+  t.plan(3)
+  const target = createTargetServer()
+  await target.listen(0)
+  t.teardown(target.close.bind(target))
+
+  const instance = Fastify()
+
+  instance.register(From, {
+    base: `http://localhost:${target.address().port}`
+  })
+
+  instance.get('/', (request, reply) => {
+    reply.from()
+  })
+
+  t.teardown(instance.close.bind(instance))
+  await instance.listen(0)
+
+  const res = await got.get(`http://localhost:${instance.server.address().port}`, { retry: 0 })
+  t.equal(res.headers['content-type'], 'text/plain')
+  t.equal(res.statusCode, 205)
+  t.equal(res.body.toString(), 'Hello World 2!')
+})
+
+test('Should retry on 503 HTTP error with Retry-After response header', async function (t) {
+  t.plan(3)
+  const target = createTargetServer(true)
+  await target.listen(0)
+  t.teardown(target.close.bind(target))
+
+  const instance = Fastify()
+
+  instance.register(From, {
+    base: `http://localhost:${target.address().port}`
+  })
+
+  instance.get('/', (request, reply) => {
+    reply.from()
+  })
+
+  t.teardown(instance.close.bind(instance))
+  await instance.listen(0)
+
+  const res = await got.get(`http://localhost:${instance.server.address().port}`, { retry: 0 })
+  t.equal(res.headers['content-type'], 'text/plain')
+  t.equal(res.statusCode, 205)
+  t.equal(res.body.toString(), 'Hello World 2!')
+})
+
+test('Should abort if server is always returning 503', async function (t) {
+  t.plan(2)
+  const target = createTargetServer(true, Number.MAX_SAFE_INTEGER)
+  await target.listen(0)
+  t.teardown(target.close.bind(target))
+
+  const instance = Fastify()
+
+  instance.register(From, {
+    base: `http://localhost:${target.address().port}`
+  })
+
+  instance.get('/', (request, reply) => {
+    reply.from()
+  })
+
+  t.teardown(instance.close.bind(instance))
+  await instance.listen(0)
+  try {
+    await got.get(`http://localhost:${instance.server.address().port}`, { retry: 0 })
+    t.fail()
+  } catch (err) {
+    t.equal(err.response.statusCode, 503)
+    t.equal(err.response.body.toString(), 'This Service is Unavailable')
+  }
+})