Attempt a stop-gap solution for mongo dropped connections.

I don't think this was globally happening, but is perhaps more related
to specific server environments dropping persistent connections:
https://support.mongolab.com/entries/23009358-handling-dropped-connections-on-windows-azure

lib/gatekeeper/middleware/api_key_validator.js

@@ -1,7 +1,9 @@
 'use strict';
 
 var _ = require('lodash'),
+    async = require('async'),
     config = require('../../config'),
+    logger = require('../../logger'),
     mergeOverwriteArrays = require('object-extend'),
     utils = require('../utils');
 
@@ -20,9 +22,39 @@ _.extend(ApiKeyValidatorRequest.prototype, {
     if(apiKey) {
       request.apiUmbrellaGatekeeper.apiKey = apiKey;
 
-      this.validator.users.findOne({
-        api_key: request.apiUmbrellaGatekeeper.apiKey,
-      }, this.handleUser.bind(this, request));
+      // FIXME: We're seeing mongo dropped connections on one of our servers
+      // that's erroneously leading to bad api key errors. This seems sporadic
+      // and related to mongo dropping connections:
+      // https://support.mongolab.com/entries/23009358-handling-dropped-connections-on-windows-azure
+      //
+      // For now, let's try retrying the user lookup several times on failure
+      // to allow the chance for the mongo connection to re-establish.
+      var retriesCount = 0;
+      var retriesError;
+      var retriesUser;
+      async.doWhilst(
+        function(callback) {
+          this.validator.users.findOne({ api_key: request.apiUmbrellaGatekeeper.apiKey }, function(error, user) {
+            if(retriesCount > 0) {
+              logger.warning('MongoDB find user retry: ', retriesCount);
+            }
+
+            retriesError = error;
+            retriesCount++;
+            if(error) {
+              logger.error('MongoDB find user error (retrying...): ', error);
+              setTimeout(callback, 5); // Retry in 5ms.
+            } else {
+              retriesUser = user;
+              callback();
+            }
+          });
+        }.bind(this), function() {
+          // Keep retrying while there's an error up to 10 times.
+          return (retriesError && retriesCount < 10);
+        }, function(error) {
+          this.handleUser(request, error, retriesUser);
+        }.bind(this));
     } else {
       if(request.apiUmbrellaGatekeeper.settings && request.apiUmbrellaGatekeeper.settings.disable_api_key) {
         next();
@@ -65,6 +97,10 @@ _.extend(ApiKeyValidatorRequest.prototype, {
   },
 
   handleUser: function(request, error, user) {
+    if(error) {
+      logger.error('Failed to find user: ', error);
+    }
+
     if(user) {
       if(!user.disabled_at) {
         this.request.apiUmbrellaGatekeeper.user = user;