Merge branch 'a190638523095652_improve_health_check'

setup.py

@@ -26,6 +26,7 @@
 test_requires = requires + [
     'webtest',
     'python-coveralls',
+    'mock',
 ]
 docs_requires = requires + [
     'sphinxcontrib-httpdomain',

src/openprocurement/api/__init__.py

@@ -177,7 +177,8 @@ def main(global_config, **settings):
             plugin(config.registry)
 
     config.registry.server_id = settings.get('id', '')
-    config.registry.health_threshold = float(settings.get('health_threshold', 99))
+    config.registry.health_threshold = float(settings.get('health_threshold', 512))
+    config.registry.health_threshold_func = settings.get('health_threshold_func', 'all')
     config.registry.update_after = asbool(settings.get('update_after', True))
     return config.make_wsgi_app()
 

src/openprocurement/api/tests/health.py

@@ -1,10 +1,96 @@
 # -*- coding: utf-8 -*-
-
+from uuid import uuid4
 from openprocurement.api.tests.base import BaseWebTest
+from mock import Mock, MagicMock
+from couchdb import Server as CouchdbServer
+from copy import copy
+
+REPLICATION = {
+    "pid": "<0.17282.20>",
+    "checkpoint_interval": 5000,
+    "checkpointed_source_seq": 0,
+    "continuous": True,
+    "doc_id": "de1a666d189db087bcd6151d2c0014a2",
+    "doc_write_failures": 0,
+    "docs_read": 231078,
+    "docs_written": 231078,
+    "missing_revisions_found": 231076,
+    "progress": 100,
+    "replication_id":"c54108a5a3c6f23208936eb961611021+continuous+create_target",
+    "revisions_checked": 1000,
+    "source":"http://op_db_reader:*****@source/openprocurement/",
+    "source_seq": 1000,
+    "started_on": 1476261621,
+    "target": "http://target:*****@target/openprocurement/",
+    "type": "replication",
+    "updated_on":1476366001
+}
+
+REPLICATION_OK = copy(REPLICATION)
+REPLICATION_OK['checkpointed_source_seq'] = REPLICATION_OK['source_seq']
+
 
+class HealthTestBase(BaseWebTest):
 
-class HealthTest(BaseWebTest):
+    return_value = []
+
+    def setUp(self):
+        self.db_name += uuid4().hex
+        # self.couchdb_server.create(self.db_name)
+        couchdb_server = Mock(spec=CouchdbServer)
+        couchdb_server.tasks = MagicMock(return_value=self.return_value)
+        self.app.app.registry.couchdb_server = couchdb_server
+        self.db_name = self.db.name
+        self.app.authorization = ('Basic', ('token', ''))
 
     def test_health_view(self):
         response = self.app.get('/health', status=503)
         self.assertEqual(response.status, '503 Service Unavailable')
+
+        response = self.app.get('/health?health_threshold_func=all', status=503)
+        self.assertEqual(response.status, '503 Service Unavailable')
+
+        response = self.app.get('/health?health_threshold_func=any', status=503)
+        self.assertEqual(response.status, '503 Service Unavailable')
+
+class HealthTest503(HealthTestBase):
+    return_value = [REPLICATION]
+
+    def test_health_view(self):
+        response = self.app.get('/health?health_threshold=10000', status=200)
+        self.assertEqual(response.status, '200 OK')
+
+class HealthTest200(HealthTestBase):
+    return_value = [REPLICATION_OK]
+
+
+    def test_health_view(self):
+        response = self.app.get('/health', status=200)
+        self.assertEqual(response.status, '200 OK')
+
+        response = self.app.get('/health?health_threshold_func=all', status=200)
+        self.assertEqual(response.status, '200 OK')
+
+        response = self.app.get('/health?health_threshold_func=any', status=200)
+        self.assertEqual(response.status, '200 OK')
+
+
+class HealthTest_all(HealthTestBase):
+    return_value = [REPLICATION_OK, REPLICATION_OK]
+
+
+    def test_health_view(self):
+        response = self.app.get('/health', status=200)
+        self.assertEqual(response.status, '200 OK')
+
+        response = self.app.get('/health?health_threshold_func=all', status=200)
+        self.assertEqual(response.status, '200 OK')
+
+
+class HealthTest_any(HealthTestBase):
+    return_value = [REPLICATION_OK, REPLICATION]
+
+
+    def test_health_view(self):
+        response = self.app.get('/health?health_threshold_func=any', status=200)
+        self.assertEqual(response.status, '200 OK')

src/openprocurement/api/views/health.py

@@ -3,13 +3,21 @@
 from pyramid.response import Response
 
 health = Service(name='health', path='/health', renderer='json')
-
+HEALTH_THRESHOLD_FUNCTIONS = {
+    'any': any,
+    'all': all
+}
 
 @health.get()
 def get_spore(request):
     tasks = getattr(request.registry, 'admin_couchdb_server', request.registry.couchdb_server).tasks()
     output = {task['replication_id']: task['progress'] for task in tasks if 'type' in task and task['type'] == 'replication'}
-    if not(output and all([True if progress >= request.registry.health_threshold else False
-                          for progress in output.values()])):
+    health_threshold = request.params.get('health_threshold', request.registry.health_threshold)
+    health_threshold_func_name = request.params.get('health_threshold_func', request.registry.health_threshold_func)
+    health_threshold_func = HEALTH_THRESHOLD_FUNCTIONS.get(health_threshold_func_name, all)
+    if not(output and health_threshold_func(
+            [True if (task['source_seq'] - task['checkpointed_source_seq']) <= health_threshold else False
+             for task in tasks if 'type' in task and task['type'] == 'replication']
+    )):
         return Response(json_body=output, status=503)
     return output

versions.cfg

@@ -19,6 +19,7 @@ waitress = 0.8.10
 zc.recipe.egg = 2.0.1
 zope.deprecation = 4.1.1
 zope.interface = 4.1.1
+mock = 1.0.1
 
 # Required by:
 # couchdb-schematics==1.1.1