feat(medcat-trainer): CU-869adnv5w k8s improvements. Health checks, fix load_examples, fix solr logging

medcat-trainer/nginx/sites-enabled/medcattrainer

@@ -25,6 +25,10 @@ server {
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     }
-
+    location /nginx/health/live {
+        access_log off;
+        return 200 'OK';
+        add_header Content-Type text/plain;
+    }
 }
 

medcat-trainer/webapp/api/api/solr_utils.py

@@ -31,6 +31,7 @@ def _cache_solr_collection_schema_types(collection):
 
 def collections_available(cdbs: List[int]):
     url = f'http://{SOLR_HOST}:{SOLR_PORT}/solr/admin/collections?action=LIST'
+    logger.info(f'Retrieving all SOLR collections: {url}')
     resp = requests.get(url)
     if resp.status_code == 200:
         collections = json.loads(resp.text)['collections']

medcat-trainer/webapp/api/api/views.py

@@ -675,8 +675,9 @@ def concept_search_index_available(request):
     try:
         return collections_available(cdb_ids)
     except Exception as e:
+        logger.error("Failed to search for concept_search_index. Solr Search Service not available", exc_info=e)
         return HttpResponseServerError("Solr Search Service not available check the service is up, running "
-                                       "and configured correctly. %s", e)
+                                       "and configured correctly", e)
 
 
 @api_view(http_method_names=['GET'])

medcat-trainer/webapp/api/core/settings.py

@@ -65,6 +65,9 @@
     'django_filters',
     'background_task',
     'api',
+    'health_check',
+    'health_check.contrib.db_heartbeat',
+    'health_check.contrib.migrations',
 ]
 
 MIDDLEWARE = [
@@ -85,7 +88,7 @@
         'DIRS': [
             os.path.join(BASE_DIR, "..", "frontend", "dist"),
             os.path.join(BASE_DIR, "..", "templates", "registration")
-            ],
+        ],
         'APP_DIRS': True,
         'OPTIONS': {
             'context_processors': [
@@ -158,6 +161,15 @@
         }
     }
 
+# https://pypi.org/project/django-health-check/
+HEALTH_CHECK = {
+    "SUBSETS": {
+        'startup': ['DatabaseHeartBeatCheck', 'MigrationsHealthCheck'],
+        'live': [],
+        'ready': ['DatabaseHeartBeatCheck'],
+    },
+}
+
 # Password validation
 # https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
 
@@ -235,4 +247,4 @@
 EMAIL_PORT = os.environ.get('EMAIL_PORT')
 EMAIL_HOST_USER = os.environ.get('EMAIL_USER')
 EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_PASS')
-EMAIL_USE_TLS = True
+EMAIL_USE_TLS = True

medcat-trainer/webapp/api/core/urls.py

@@ -25,6 +25,7 @@
 
 urlpatterns = [
     path('api/', include(router.urls)),
+    path('api/health/', include('health_check.urls')),
     path('api/anno-conf/', api.views.get_anno_tool_conf),
     path('api/search-concepts/', api.views.search_solr),
     path('api/prepare-documents/', api.views.prepare_documents),

medcat-trainer/webapp/requirements.txt

@@ -8,3 +8,4 @@ django-background-tasks-updated==1.2.*
 openpyxl==3.1.2
 medcat==1.16.*
 psycopg[binary,pool]==3.2.9
+django-health-check==3.20.0

medcat-trainer/webapp/scripts/load_examples.py

@@ -62,7 +62,7 @@ def main(port=8000,
                         f.write(vocab_file.content)
                     print("Downloading example dataset")
                     ds = requests.get(
-                        'https://raw.githubusercontent.com/CogStack/cogstack-nlp/09e32429/medcat-trainer/notebook_docs/example_data/ortho.csvv')
+                        'https://raw.githubusercontent.com/CogStack/cogstack-nlp/09e32429/medcat-trainer/notebook_docs/example_data/ortho.csv')
                     with open(dataset_tmp_file, 'w') as f:
                         f.write(ds.text)