fix(blocking): Send block for all requests if memcache is down

lib/server.js

@@ -73,9 +73,9 @@ module.exports = function createServer(config, log) {
     return P.all(
       [
         // store records ignoring errors
-        mc.setAsync(email, emailRecord, LIFETIME).catch(ignore),
-        mc.setAsync(ip, ipRecord, LIFETIME).catch(ignore),
-        mc.setAsync(ip + email, ipEmailRecord, LIFETIME).catch(ignore)
+        mc.setAsync(email, emailRecord, LIFETIME),
+        mc.setAsync(ip, ipRecord, LIFETIME),
+        mc.setAsync(ip + email, ipEmailRecord, LIFETIME)
       ]
     )
   }
@@ -133,7 +133,17 @@ module.exports = function createServer(config, log) {
           },
           function (err) {
             log.error({ op: 'request.check', email: email, ip: ip, action: action, err: err })
-            res.send(500, err)
+
+            // Temporarily block request if memcache related error
+            if( err.name && err.name === 'RejectionError' ) {
+              log.error({ op: 'memcachedError', err: err })
+              res.send({
+                block: true,
+                retryAfter: LIFETIME // A very big number
+              })
+            } else  {
+              res.send(500, err)
+            }
           }
         )
         .done(next, next)

test/remote/memcache_error_tests.js

@@ -0,0 +1,58 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+var test = require('tap').test
+var restify = require('restify')
+var TestServer = require('../test_server')
+
+var TEST_EMAIL = 'test@example.com'
+var TEST_IP = '192.0.2.1'
+
+var config = {
+  listen: {
+    port: 7000
+  }
+}
+
+process.env.MEMCACHE_ADDRESS = '128.0.0.1:12131'
+
+var testServer = new TestServer(config)
+
+test(
+  'startup',
+  function (t) {
+    testServer.start(function (err) {
+      t.type(testServer.server, 'object', 'test server was started')
+      t.notOk(err, 'no errors were returned')
+      t.end()
+    })
+  }
+)
+
+
+var client = restify.createJsonClient({
+  url: 'http://127.0.0.1:' + config.listen.port
+})
+
+test(
+  'request with disconnected memcache',
+  function (t) {
+    client.post('/check', { email: TEST_EMAIL, ip: TEST_IP, action: 'someRandomAction' },
+      function (err, req, res, obj) {
+        t.equal(res.statusCode, 200, 'check worked')
+        t.equal(obj.block, true, 'request was blocked')
+        t.equal(obj.retryAfter, 900, 'retry after')
+        t.end()
+      }
+    )
+  }
+)
+
+test(
+  'teardown',
+  function (t) {
+    testServer.stop()
+    t.equal(testServer.server.killed, true, 'test server has been killed')
+    t.end()
+  }
+)