Rework health check, using redis when applicable (#731)

* Rework health check, using redis when applicable

Creates unit tests for the queries and sets up unit test mocks

* Fix tests

* Remove unused .ini and add docstring

* Fix env vars for tests

* Fix tests

discovery-provider/.test.env

@@ -0,0 +1,3 @@
+# TODO: dummy ganache keys for local setup; should wire with dynamically generated keys
+audius_delegate_owner_wallet=0x1D9c77BcfBfa66D37390BF2335f0140979a6122B
+audius_delegate_private_key=0x3873ed01bfb13621f9301487cc61326580614a5b99f3c33cf39c6f9da3a19cad

discovery-provider/compose/docker-compose.base.override.yml

@@ -14,6 +14,8 @@ services:
   redis-server:
     image: redis:3.0-alpine
     command: redis-server --save '' 
+    ports:
+      - '5379:6379'
     networks:
       - audius_dev
 

discovery-provider/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+addopts =  -s -v --fulltrace
+env_files =
+  .test.env

discovery-provider/requirements.txt

@@ -5,15 +5,17 @@ SQLAlchemy==1.2.5
 alembic==1.0.0
 celery[redis]==4.2.0
 redis==3.2.0
-pytest==3.7.4
+pytest==6.0.1
 SQLAlchemy-Utils==0.33.3
 chance==0.110
 pylint==2.3.1
 ipfshttpclient==0.4.12
 pytest-cov==2.6.0
+pytest-dotenv==0.5.2
 base58==1.0.0
 flask-cors==3.0.6
 gunicorn==19.9.0
 jsonschema==3.2.0
 flask-restx==0.2.0
 hashids==1.2.0
+fakeredis==1.4.2

discovery-provider/scripts/test.sh

@@ -69,4 +69,8 @@ docker-compose -f docker-compose.base.yml up -d
 # docker-compose -f docker-compose.ipfs.yml up -d
 sleep 5
 
-pytest -s -v --fulltrace 
+# Unit tests
+pytest src
+
+# Integration tests
+pytest tests

discovery-provider/src/conftest.py

@@ -0,0 +1,62 @@
+"""
+Test fixtures to support unit testing
+"""
+
+from unittest.mock import MagicMock
+import pytest
+import fakeredis
+import src.utils.redis_connection
+import src.utils.web3_provider
+import src.utils.db_session
+
+
+# Test fixture to mock a postgres database using an in-memory alternative
+@pytest.fixture()
+def db_mock(monkeypatch):
+    db = src.utils.db_session.SessionManager(
+        'sqlite://',
+        {}
+    )
+
+    def get_db_read_replica():
+        return db
+
+    monkeypatch.setattr(
+        src.utils.db_session,
+        'get_db_read_replica',
+        get_db_read_replica
+    )
+
+    return db
+
+
+# Test fixture to mock a web3 provider
+@pytest.fixture()
+def web3_mock(monkeypatch):
+    web3 = MagicMock()
+
+    def get_web3():
+        return web3
+
+    monkeypatch.setattr(
+        src.utils.web3_provider,
+        'get_web3',
+        get_web3
+    )
+    return web3
+
+
+# Test fixture to mock a redis connection
+@pytest.fixture()
+def redis_mock(monkeypatch):
+    redis = fakeredis.FakeStrictRedis()
+
+    def get_redis():
+        return redis
+
+    monkeypatch.setattr(
+        src.utils.redis_connection,
+        'get_redis',
+        get_redis
+    )
+    return redis

discovery-provider/src/queries/get_health.py

@@ -0,0 +1,146 @@
+import logging
+import os
+import sqlalchemy
+
+from src.models import Block
+from src.utils import helpers, redis_connection, web3_provider, db_session
+from src.utils.config import shared_config
+from src.utils.redis_constants import latest_block_redis_key, \
+    latest_block_hash_redis_key, most_recent_indexed_block_hash_redis_key, most_recent_indexed_block_redis_key
+
+
+logger = logging.getLogger(__name__)
+disc_prov_version = helpers.get_discovery_provider_version()
+
+default_healthy_block_diff = int(
+    shared_config["discprov"]["healthy_block_diff"])
+
+# Returns DB block state & diff
+def _get_db_block_state():
+    db = db_session.get_db_read_replica()
+    with db.scoped_session() as session:
+        # Fetch latest block from DB
+        db_block_query = session.query(Block).filter(
+            Block.is_current == True).all()
+        assert len(db_block_query) == 1, "Expected SINGLE row marked as current"
+        return helpers.model_to_dictionary(db_block_query[0])
+
+
+# Returns number of and info on open db connections
+def _get_db_conn_state():
+    db = db_session.get_db_read_replica()
+    with db.scoped_session() as session:
+        # Query number of open DB connections
+        num_connections = session.execute(
+            sqlalchemy.text("select sum(numbackends) from pg_stat_database;")
+        ).fetchall()
+
+        if not (num_connections and num_connections[0][0]):
+            return 'pg_stat_database query failed', True
+        num_connections = num_connections[0][0]
+
+        # Query connection info
+        connection_info = session.execute(sqlalchemy.text(
+            "select datname, state, query, wait_event_type, wait_event from pg_stat_activity where state is not null;"
+        )).fetchall()
+        connection_info = [dict(row) for row in connection_info]
+
+    return {"open_connections": num_connections, "connection_info": connection_info}, False
+
+
+def get_health(args, use_redis_cache=True):
+    """
+    Gets health status for the service
+
+    :param args: dictionary
+    :param args.verbose: bool
+        if True, returns db connection information
+    :param args.healthy_block_diff: int
+        determines the point at which a block difference is considered unhealthy
+    :param args.enforce_block_diff: bool
+        if true and the block difference is unhealthy an error is returned
+
+    :rtype: (dictionary, bool)
+    :return: tuple of health results and a boolean indicating an error
+    """
+    redis = redis_connection.get_redis()
+    web3 = web3_provider.get_web3()
+
+    verbose = args.get("verbose")
+    enforce_block_diff = args.get("enforce_block_diff")
+    qs_healthy_block_diff = args.get("healthy_block_diff")
+
+    # If healthy block diff is given in url and positive, override config value
+    healthy_block_diff = qs_healthy_block_diff if qs_healthy_block_diff is not None \
+        and qs_healthy_block_diff >= 0 else default_healthy_block_diff
+
+    latest_block_num = None
+    latest_block_hash = None
+
+    # Get latest web block info
+    if use_redis_cache:
+        stored_latest_block_num = redis.get(latest_block_redis_key)
+        if stored_latest_block_num is not None:
+            latest_block_num = int(stored_latest_block_num)
+
+        stored_latest_blockhash = redis.get(latest_block_hash_redis_key)
+        if stored_latest_blockhash is not None:
+            latest_block_hash = stored_latest_blockhash.decode("utf-8")
+
+    if latest_block_num is None or latest_block_hash is None:
+        latest_block = web3.eth.getBlock("latest", True)
+        latest_block_num = latest_block.number
+        latest_block_hash = latest_block.hash.hex()
+
+    latest_indexed_block_num = None
+    latest_indexed_block_hash = None
+
+    # Get latest indexed block info
+    if use_redis_cache:
+        latest_indexed_block_num = redis.get(
+            most_recent_indexed_block_redis_key)
+        if latest_indexed_block_num is not None:
+            latest_indexed_block_num = int(latest_indexed_block_num)
+
+        latest_indexed_block_hash = redis.get(
+            most_recent_indexed_block_hash_redis_key)
+        if latest_indexed_block_hash is not None:
+            latest_indexed_block_hash = latest_indexed_block_hash.decode(
+                "utf-8")
+
+    if latest_indexed_block_num is None or latest_indexed_block_hash is None:
+        db_block_state = _get_db_block_state()
+        latest_indexed_block_num = db_block_state["number"] or 0
+        latest_indexed_block_hash = db_block_state["blockhash"]
+
+    health_results = {
+        "web": {
+            "blocknumber": latest_block_num,
+            "blockhash": latest_block_hash,
+        },
+        "db": {
+            "number": latest_indexed_block_num,
+            "blockhash": latest_indexed_block_hash
+        },
+        "git": os.getenv("GIT_SHA"),
+    }
+
+    block_difference = abs(
+        health_results["web"]["blocknumber"] - health_results["db"]["number"]
+    )
+    health_results["block_difference"] = block_difference
+    health_results["maximum_healthy_block_difference"] = default_healthy_block_diff
+    health_results.update(disc_prov_version)
+
+    if verbose:
+        # DB connections check
+        db_connections_json, error = _get_db_conn_state()
+        health_results["db_connections"] = db_connections_json
+        if error:
+            return health_results, error
+
+    # Return error on unhealthy block diff if requested.
+    if enforce_block_diff and health_results["block_difference"] > healthy_block_diff:
+        return health_results, True
+
+    return health_results, False