ZAPI-763 Connections refused by VMAPI on first boot

Reviewed by: Josh Wilsdon <jwilsdon@joyent.com>
Reviewed by: Trent Mick <trent.mick@joyent.com>
Approved by: Julien Gilli <julien.gilli@joyent.com>

lib/apis/moray.js

@@ -13,15 +13,16 @@
  */
 
 
-var EventEmitter = require('events').EventEmitter;
-var sprintf = require('sprintf').sprintf;
 var assert = require('assert-plus');
-var restify = require('restify');
-var util = require('util');
-var Logger = require('bunyan');
 var async = require('async');
+var backoff = require('backoff');
+var bunyan = require('bunyan');
+var EventEmitter = require('events').EventEmitter;
 var jsprint = require('jsprim');
 var ldapjs = require('ldap-filter');
+var restify = require('restify');
+var sprintf = require('sprintf').sprintf;
+var util = require('util');
 
 var errors = require('../errors');
 var common = require('./../common');
@@ -106,7 +107,7 @@ function Moray(options) {
     EventEmitter.call(this);
     // this.log = options.log;
     // this.log.level(options.logLevel || 'info');
-    this.log = new Logger({
+    this.log = new bunyan.createLogger({
         name: 'moray',
         level: options.logLevel || 'info',
         serializers: restify.bunyan.serializers
@@ -141,7 +142,23 @@ Moray.prototype.connect = function () {
         })
     });
 
+
     connection.on('connect', function () {
+        /*
+         * Even though only one 'connect' event can be emitted throughout the
+         * lifetime of a given moray client, clear a potentially existing
+         * buckets setup backoff instance if Moray.prototype.connect is called
+         * more than once. This should also prevent the same problem from
+         * happening when we upgrade node-moray to a version that emits the
+         * 'connect' event more than once, e.g when it reconnects after losing
+         * its connection to the moray service.
+         */
+        if (self._bucketsSetupBackoff) {
+            self._bucketsSetupBackoff.reset();
+        }
+
+        self._bucketsSetupBackoff = backoff.exponential();
+
         log.info({ moray: connection.toString() }, 'moray: connected');
         self.emit('moray-connected');
 
@@ -151,14 +168,29 @@ Moray.prototype.connect = function () {
             log.error(err, 'moray client error');
         });
 
-        self._setupBuckets(function (err) {
-            if (err) {
-                self.log.error({ err: err }, 'Buckets were not loaded');
-            } else {
-                self.emit('moray-ready');
-                self.log.info('Buckets have been loaded');
-            }
-        });
+        self._bucketsSetupBackoff.on('backoff',
+                function onBucketSetupBackoff(number, delay) {
+                    log.info({
+                        number: number,
+                        delay: delay
+                    }, 'backing off buckets setup');
+            });
+
+        self._bucketsSetupBackoff.on('ready',
+            function onBucketSetupBackoffReady() {
+                self._setupBuckets(function (err) {
+                    if (err) {
+                        self.log.error({ err: err }, 'Buckets were not loaded');
+                        self._bucketsSetupBackoff.backoff();
+                    } else {
+                        self.log.info('Buckets have been loaded');
+                        self._bucketsSetupBackoff.reset();
+                        self.emit('moray-ready');
+                    }
+                });
+            });
+
+        self._bucketsSetupBackoff.backoff();
     });
 };
 

package.json

@@ -7,6 +7,7 @@
   "dependencies": {
     "assert-plus": "1.0.0",
     "async": "0.7.0",
+    "backoff": "2.5.0",
     "bunyan": "1.8.1",
     "changefeed": "1.2.2",
     "deep-diff": "0.3.3",