Merge pull request metabrainz#460 from paramsingh/mer

Merge production into master

listenbrainz/webserver/__init__.py

@@ -103,7 +103,11 @@ def gen_app(config_path=None, debug=None):
     create_influx(app)
 
     # RabbitMQ connection
-    create_rabbitmq(app)
+    try:
+        create_rabbitmq(app)
+    except ConnectionError:
+        app.logger.critical("RabbitMQ service is not up!", exc_info=True)
+
 
     # Database connection
     from listenbrainz import db

listenbrainz/webserver/rabbitmq_connection.py

@@ -1,4 +1,3 @@
-import sys
 import queue
 from time import sleep
 import pika
@@ -20,8 +19,12 @@ def init_rabbitmq_connection(app):
     if _rabbitmq is not None:
         return
 
+    # if RabbitMQ config values are not in the config file
+    # raise an error. This is caught in create_app, so the app will continue running.
+    # Consul will bring the values back into config once the RabbitMQ service comes up.
     if "RABBITMQ_HOST" not in app.config:
-        raise ConnectionError("Cannot connect to RabbitMQ: host and port not defined")
+        app.logger.critical("RabbitMQ host:port not defined, cannot create RabbitMQ connection...")
+        raise ConnectionError("RabbitMQ service is not up!")
 
     connection_parameters = pika.ConnectionParameters(
         host=app.config['RABBITMQ_HOST'],
@@ -92,7 +95,11 @@ def __exit__(self, type, value, traceback):
 
     @property
     def is_open(self):
-        return self.connection.is_open
+        try:
+            self.connection.process_data_events()
+            return True
+        except pika.exceptions.ConnectionClosed as e:
+            return False
 
     def close(self):
         self.connection.close()

listenbrainz/webserver/test_rabbitmq_connection.py

@@ -1,9 +1,12 @@
+from flask import Flask
 from unittest import TestCase
 from unittest.mock import patch, MagicMock
 from pika.exceptions import ConnectionClosed
 
+import listenbrainz.webserver.rabbitmq_connection as rabbitmq_connection
+
+from listenbrainz.webserver.rabbitmq_connection import RabbitMQConnectionPool, CONNECTION_RETRIES, init_rabbitmq_connection
 
-from listenbrainz.webserver.rabbitmq_connection import RabbitMQConnectionPool, CONNECTION_RETRIES
 
 
 class RabbitMQConnectionPoolTestCase(TestCase):
@@ -19,3 +22,12 @@ def test_connection_closed_while_creating(self, mock_sleep, mock_blocking_connec
             connection = self.pool.create()
             self.pool.log.critical.assert_called_once()
             self.assertEqual(self.mock_sleep.call_count, CONNECTION_RETRIES - 1)
+
+    def test_connection_error_when_rabbitmq_down(self):
+        # create an app with no RabbitMQ config
+        # as will be the case when RabbitMQ is down in production
+        app = Flask(__name__)
+
+        with self.assertRaises(ConnectionError):
+            rabbitmq_connection._rabbitmq = None
+            init_rabbitmq_connection(app)