monitoring: REDIS, ES indicies

* Renames blueprints to match reroils.
* Adds REDIS and ES indicies monitoring.
* Corrects ES-DB monitoring for projects.

Co-Authored-by: Peter Weber <peter.weber@rero.ch>

sonar/monitoring/api/data_integrity.py

@@ -36,7 +36,7 @@ def get_db_count(self, doc_type, with_deleted=False):
         :param with_deleted: Count also deleted items.
         :returns: Items count.
         """
-        if not sonar.endpoints.get(doc_type):
+        if not sonar.endpoints.get(doc_type) or doc_type == 'proj':
             raise Exception(
                 'No endpoint configured for "{type}"'.format(type=doc_type))
 

sonar/monitoring/api/database.py

@@ -75,13 +75,14 @@ def format_row(row):
             """Format returned row from DB."""
             return {
                 'application_name': row['application_name'],
-                'client_address': row['client_addr'],
+                'client_addr': row['client_addr'],
                 'client_port': row['client_port'],
-                'query': row['left'],
+                'backend_start': row['backend_start'],
+                'xact_start': row['xact_start'],
                 'query_start': row['query_start'],
-                'state': row['state'],
                 'wait_event': row['wait_event'],
-                'transaction_start': row['xact_start']
+                'state': row['state'],
+                'left': row['left'],
             }
 
         return list(map(format_row, db.session.execute(query).fetchall()))

sonar/monitoring/views/__init__.py

@@ -19,9 +19,10 @@
 
 from functools import wraps
 
-from flask import Blueprint, abort, jsonify, request
+from flask import Blueprint, current_app, jsonify, request
 from flask_security import current_user
 from invenio_search import current_search_client
+from redis import Redis
 
 from sonar.modules.permissions import superuser_access_permission
 from sonar.monitoring.api.data_integrity import DataIntegrityMonitoring
@@ -52,7 +53,7 @@ def check_for_superuser():
     """Check if user is superuser before each request, with decorator."""
 
 
-@api_blueprint.route('/db/connections/count')
+@api_blueprint.route('/db_connection_counts')
 def db_connection_count():
     """Information about current database connections."""
     try:
@@ -62,7 +63,7 @@ def db_connection_count():
         return jsonify({'error': str(exception)}), 500
 
 
-@api_blueprint.route('/db/activity')
+@api_blueprint.route('db_connections')
 def db_activity():
     """Current database activity."""
     try:
@@ -72,21 +73,21 @@ def db_activity():
         return jsonify({'error': str(exception)}), 500
 
 
-@api_blueprint.route('/data/status')
+@api_blueprint.route('/es_db_status')
 def data_status():
     """Status of data integrity."""
     try:
         data_monitoring = DataIntegrityMonitoring()
         return jsonify({
             'data': {
-                'status': 'green' if not data_monitoring.has_error() else 'red'
+                'status': 'red' if data_monitoring.has_error() else 'green'
             }
         })
     except Exception as exception:
         return jsonify({'error': str(exception)}), 500
 
 
-@api_blueprint.route('/data/info')
+@api_blueprint.route('/es_db_counts')
 def data_info():
     """Info of data integrity."""
     try:
@@ -99,6 +100,18 @@ def data_info():
         return jsonify({'error': str(exception)}), 500
 
 
+@api_blueprint.route('/redis')
+def redis():
+    """Displays redis info.
+
+    :return: jsonified redis info.
+    """
+    url = current_app.config.get('ACCOUNTS_SESSION_REDIS_URL',
+                                 'redis://localhost:6379')
+    redis = Redis.from_url(url)
+    info = redis.info()
+    return jsonify({'data': info})
+
 @api_blueprint.route('/es')
 def elastic_search():
     """Displays elastic search cluster info.
@@ -110,3 +123,15 @@ def elastic_search():
         return jsonify({'data': info})
     except Exception as exception:
         return jsonify({'error': str(exception)}), 500
+
+
+@api_blueprint.route('/es_indices')
+def elastic_search_indices():
+    """Displays Elasticsearch indices info.
+
+    :return: jsonified Elasticsearch indices info.
+    """
+    info = current_search_client.cat.indices(
+        bytes='b', format='json', s='index')
+    info = {data['index']: data for data in info}
+    return jsonify({'data': info})

tests/api/monitoring/test_monitoring_views.py

@@ -103,14 +103,15 @@ def fetchall(self):
     assert response.json == {
         'data': [{
             'application_name': '',
-            'client_address': '10.233.92.25',
+            'backend_start': 'Mon, 08 Feb 2021 10:46:55 GMT',
+            'client_addr': '10.233.92.25',
             'client_port': 33382,
-            'query':
+            'left':
             '\n        SELECT\n            pid, application_name, client',
             'query_start': 'Mon, 08 Feb 2021 10:46:55 GMT',
             'state': 'active',
-            'transaction_start': 'Mon, 08 Feb 2021 10:46:55 GMT',
-            'wait_event': None
+            'wait_event': None,
+            'xact_start': 'Mon, 08 Feb 2021 10:46:55 GMT'
         }]
     }