diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..e9a13746 --- /dev/null +++ b/.env.example @@ -0,0 +1 @@ +STATUS_TOKEN= diff --git a/docker-compose.yml b/docker-compose.yml index 32011e2a..6ad2bb94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,20 +2,24 @@ db: image: postgres ports: - "5432" + redis: image: redis ports: - "6379" + elastic: image: elasticsearch command: elasticsearch -Des.network.host=0.0.0.0 ports: - "9200" + stats: image: kamon/grafana_graphite ports: - "8125/udp:8125/udp" - "8071:80" + web: build: . mem_limit: 384m @@ -45,6 +49,7 @@ web: CELERY_RESULT_BACKEND: redis://redis:6379/4 BROKER_URL: redis://redis:6379/4 HAYSTACK_URL: elastic:9200 + env_file: .env ports: - "8070:8070" links: @@ -52,6 +57,7 @@ web: - redis - elastic - stats + celery: image: lore_web mem_limit: 384m diff --git a/lore/settings.py b/lore/settings.py index 5793e03d..2a065d3c 100644 --- a/lore/settings.py +++ b/lore/settings.py @@ -98,6 +98,7 @@ def get_var(name, default): 'search', 'roles', 'xanalytics', + 'server_status', ) MIDDLEWARE_CLASSES = ( @@ -374,6 +375,7 @@ def get_var(name, default): } # Celery +USE_CELERY = True BROKER_URL = get_var("BROKER_URL", get_var("REDISCLOUD_URL", None)) CELERY_RESULT_BACKEND = get_var( "CELERY_RESULT_BACKEND", get_var("REDISCLOUD_URL", None) @@ -396,7 +398,8 @@ def get_var(name, default): XANALYTICS_URL = get_var('XANALYTICS_URL', "") -# Token required to access the status page. +# server-status +HEALTH_CHECK = ['CELERY', 'REDIS', 'POSTGRES', 'ELASTIC_SEARCH'] STATUS_TOKEN = get_var( "STATUS_TOKEN", "7E17C32A63B2810F0053DE454FC8395CA3262CCB8392D2307887C5E67F132550" diff --git a/requirements.txt b/requirements.txt index 8e8b9592..1203da26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,3 +42,4 @@ beautifulsoup4==4.4.1 # Application monitoring requirements newrelic==2.58.1.44 +django-server-status==0.3 \ No newline at end of file diff --git a/status/__init__.py b/status/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/status/tests/__init__.py b/status/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/status/tests/test_status.py b/status/tests/test_status.py deleted file mode 100644 index b8d3d152..00000000 --- a/status/tests/test_status.py +++ /dev/null @@ -1,117 +0,0 @@ -""" -Tests for the status module. -""" -from __future__ import unicode_literals - -from copy import deepcopy -import json -import logging - -from django.conf import settings -from django.core.urlresolvers import reverse -from django.test import Client -from django.test.testcases import TestCase - -log = logging.getLogger(__name__) - -HTTP_OK = 200 -SERVICE_UNAVAILABLE = 503 - - -class TestStatus(TestCase): - """Test output of status page.""" - - def setUp(self): - """ - Set up client and remember settings for PostgreSQL, Redis, - and Elasticsearch. Changes made to django.conf.settings during - tests persist beyond the test scope, so if they are changed - we need to restore them. - """ - - # Create test client. - self.client = Client() - self.url = reverse("status") - super(TestStatus, self).setUp() - - def get(self, expected_status=HTTP_OK): - """Get the page.""" - resp = self.client.get(self.url, data={"token": settings.STATUS_TOKEN}) - self.assertEqual(resp.status_code, expected_status) - return json.loads(resp.content.decode('utf-8')) - - def test_view(self): - """Get normally.""" - resp = self.get() - for key in ("postgresql", "redis", "elasticsearch"): - self.assertTrue(resp[key]["status"] == "up") - - def test_no_settings(self): - """Missing settings.""" - (broker_url, databases, haystack_connections) = ( - settings.BROKER_URL, - settings.DATABASES, - settings.HAYSTACK_CONNECTIONS - ) - try: - del settings.BROKER_URL - del settings.DATABASES - del settings.HAYSTACK_CONNECTIONS - resp = self.get() - for key in ("postgresql", "redis", "elasticsearch"): - self.assertTrue(resp[key]["status"] == "no config found") - finally: - ( - settings.BROKER_URL, - settings.DATABASES, - settings.HAYSTACK_CONNECTIONS, - ) = (broker_url, databases, haystack_connections) - - def test_broken_settings(self): - """Settings that couldn't possibly work.""" - junk = " not a chance " - broker_url = junk - databases = deepcopy(settings.DATABASES) - databases['default'] = junk - haystack_connections = deepcopy(settings.HAYSTACK_CONNECTIONS) - haystack_connections["default"]["URL"] = junk - with self.settings( - BROKER_URL=broker_url, - DATABASES=databases, - HAYSTACK_CONNECTIONS=haystack_connections - ): - resp = self.get(SERVICE_UNAVAILABLE) - for key in ("postgresql", "redis", "elasticsearch"): - self.assertTrue(resp[key]["status"] == "down") - - def test_invalid_settings(self): - """ - Settings that look right, but aren't (if service is actually down). - """ - broker_url = "redis://bogus:6379/4" - databases = deepcopy(settings.DATABASES) - databases["default"]["HOST"] = "monkey" - haystack_connections = deepcopy(settings.HAYSTACK_CONNECTIONS) - haystack_connections["default"]["URL"] = "pizza:2300" - with self.settings( - BROKER_URL=broker_url, - DATABASES=databases, - HAYSTACK_CONNECTIONS=haystack_connections - ): - resp = self.get(SERVICE_UNAVAILABLE) - for key in ("postgresql", "redis", "elasticsearch"): - self.assertTrue(resp[key]["status"] == "down") - - def test_token(self): - """ - Caller must have correct token, or no dice. Having a good token - is tested in all the other tests. - """ - - # No token. - resp = self.client.get(self.url) - self.assertTrue(resp.status_code == 404) - - # Invalid token. - resp = self.client.get(self.url, {"token": "gibberish"}) - self.assertTrue(resp.status_code == 404) diff --git a/status/views.py b/status/views.py deleted file mode 100644 index a3eb45be..00000000 --- a/status/views.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -Status checks for LORE. - -Notes: - -* Useful messages are logged, but NO_CONFIG is returned whether - settings are missing or invalid, to prevent information leakage. -* Different services provide different information, but all should return - "up," DOWN, or NO_CONFIG for the "status" key. -""" - -from datetime import datetime -import logging - -from django.conf import settings -from django.http import JsonResponse, Http404 -from elasticsearch import Elasticsearch, ConnectionError as ESConnectionError -from kombu.utils.url import _parse_url as parse_redis_url -from psycopg2 import connect, OperationalError -from redis import ( - StrictRedis, - ConnectionError as RedisConnectionError, - ResponseError as RedisResponseError, -) - -log = logging.getLogger(__name__) - -UP = "up" -DOWN = "down" -NO_CONFIG = "no config found" -HTTP_OK = 200 -SERVICE_UNAVAILABLE = 503 -TIMEOUT_SECONDS = 5 - - -def get_pg_info(): - """Check PostgreSQL connection.""" - log.debug("entered get_pg_info") - try: - conf = settings.DATABASES['default'] - database = conf["NAME"] - user = conf["USER"] - host = conf["HOST"] - port = conf["PORT"] - password = conf["PASSWORD"] - except (AttributeError, KeyError): - log.error("No PostgreSQL connection info found in settings.") - return {"status": NO_CONFIG} - except TypeError: - return {"status": DOWN} - log.debug("got past getting conf") - try: - start = datetime.now() - connection = connect( - database=database, user=user, host=host, - port=port, password=password, connect_timeout=TIMEOUT_SECONDS, - ) - log.debug("at end of context manager") - micro = (datetime.now() - start).microseconds - connection.close() - except (OperationalError, KeyError): - log.error("Invalid PostgreSQL connection info in settings: %s", conf) - return {"status": DOWN} - log.debug("got to end of postgres check successfully") - return {"status": UP, "response_microseconds": micro} - - -def get_redis_info(): - """Check Redis connection.""" - try: - url = settings.BROKER_URL - _, host, port, _, password, db, _ = parse_redis_url(url) - except AttributeError: - log.error("No valid Redis connection info found in settings.") - return {"status": NO_CONFIG} - - start = datetime.now() - try: - rdb = StrictRedis( - host=host, port=port, db=db, - password=password, socket_timeout=TIMEOUT_SECONDS, - ) - info = rdb.info() - except (RedisConnectionError, TypeError) as ex: - log.error("Error making Redis connection: %s", ex.args) - return {"status": DOWN} - except RedisResponseError as ex: - log.error("Bad Redis response: %s", ex.args) - return {"status": DOWN, "message": "auth error"} - micro = (datetime.now() - start).microseconds - del rdb # the redis package does not support Redis's QUIT. - ret = { - "status": UP, "response_microseconds": micro, - } - fields = ("uptime_in_seconds", "used_memory", "used_memory_peak") - ret.update({x: info[x] for x in fields}) - return ret - - -def get_elasticsearch_info(): - """Check Elasticsearch connection.""" - try: - url = settings.HAYSTACK_CONNECTIONS["default"]["URL"] - except (AttributeError, KeyError): - log.error("No Elasticsearch connection info in settings.") - return {"status": NO_CONFIG} - start = datetime.now() - try: - search = Elasticsearch(url, request_timeout=TIMEOUT_SECONDS) - info = search.info() - except ESConnectionError: - return {"status": DOWN} - del search # The elasticsearch library has no "close" or "disconnect." - micro = (datetime.now() - start).microseconds - return { - "status": UP, "response_microseconds": micro, - "status_code": info["status"], - } - - -def status(request): # pylint: disable=unused-argument - """Status""" - token = request.GET.get("token", "") - if token != settings.STATUS_TOKEN: - raise Http404() - info = {} - log.debug("going to get redis") - info["redis"] = get_redis_info() - log.debug("redis done, going for elasticsearch") - info["elasticsearch"] = get_elasticsearch_info() - log.debug("elasticsearch done, hunting postgres") - info["postgresql"] = get_pg_info() - code = HTTP_OK - for key in info: - if info[key]["status"] == "down": - code = SERVICE_UNAVAILABLE - break - resp = JsonResponse(info) - resp.status_code = code - return resp diff --git a/ui/urls.py b/ui/urls.py index dcfcffd5..9aa79762 100644 --- a/ui/urls.py +++ b/ui/urls.py @@ -20,7 +20,6 @@ from django.conf.urls import include, url from django.contrib import admin -from status.views import status from ui.views import ( welcome, create_repo, @@ -54,7 +53,7 @@ ), url(r'^repositories/(?P[-\w]+)/import/$', upload, name='upload'), - url(r'^status/$', status, name='status'), + url(r'^status/', include('server_status.urls')), ] if (settings.DEFAULT_FILE_STORAGE ==