Deployment documentation (#427)

* Celery configuration updated and documented, adding `CELERYD_CONCURRENCY` env var
* Use environment variable to set celery concurrency.
* Update helm templates and charts. should be easier to debug future update on the templates.
* Consistency between helm configmap template and setting.py expected environment variables.
* helm values.yaml updated to be a functioning minimal deployment.

azure-pipelines.yml

@@ -183,7 +183,6 @@ stages:
         cd deployment/entity-service
         
         helm lint --debug --namespace $(NAMESPACE) $(DEPLOYMENT) . \
-        -f values.yaml -f minimal-values.yaml \
         --set api.app.debug=true \
         --set global.postgresql.postgresqlPassword=notaproductionpassword \
         --set api.ingress.enabled=false \
@@ -194,7 +193,6 @@ stages:
         
         echo "Dry run"
         helm upgrade --install --dry-run --wait --namespace $(NAMESPACE) $(DEPLOYMENT) . \
-        -f values.yaml -f minimal-values.yaml \
         --set api.app.debug=true \
         --set global.postgresql.postgresqlPassword=notaproductionpassword \
         --set api.ingress.enabled=false \
@@ -211,7 +209,6 @@ stages:
         echo "Start all the entity service pods"
         cd deployment/entity-service
         helm upgrade --install --wait --namespace $(NAMESPACE) $(DEPLOYMENT) . \
-        -f values.yaml -f minimal-values.yaml \
         --set api.app.debug=true \
         --set global.postgresql.postgresqlPassword=notaproductionpassword \
         --set api.ingress.enabled=false \

backend/entityservice/__init__.py

@@ -50,7 +50,9 @@ def initdb_command():
 
 @app.before_first_request
 def before_first_request():
-    db.init_db_pool()
+    db_min_connections = config.FLASK_DB_MIN_CONNECTIONS
+    db_max_connections = config.FLASK_DB_MAX_CONNECTIONS
+    db.init_db_pool(db_min_connections, db_max_connections)
 
 
 @app.before_request

backend/entityservice/async_worker.py

@@ -20,6 +20,9 @@
 celery.conf.result_backend_transport_options = Config.CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS
 celery.conf.worker_prefetch_multiplier = Config.CELERYD_PREFETCH_MULTIPLIER
 celery.conf.worker_max_tasks_per_child = Config.CELERYD_MAX_TASKS_PER_CHILD
+if Config.CELERYD_CONCURRENCY > 0:
+    # If set to 0, let celery choose the default, which is the number of available CPUs on the machine.
+    celery.conf.worker_concurrency = Config.CELERYD_CONCURRENCY
 
 
 # Set up our logging
@@ -31,7 +34,9 @@
 
 @worker_process_init.connect()
 def init_worker(**kwargs):
-    init_db_pool()
+    db_min_connections = Config.CELERY_DB_MIN_CONNECTIONS
+    db_max_connections = Config.CELERY_DB_MAX_CONNECTIONS
+    init_db_pool(db_min_connections, db_max_connections)
 
 
 @worker_process_shutdown.connect()

backend/entityservice/database/util.py

@@ -33,24 +33,22 @@ def init_db(delay=0.5):
 
     @param float delay: Number of seconds to wait before executing the SQL.
     """
-    init_db_pool()
+    init_db_pool(1, 3)
     time.sleep(delay)
     with DBConn() as db:
         with current_app.open_resource('init-db-schema.sql', mode='r') as f:
             logger.warning("Initialising database")
             db.cursor().execute(f.read())
 
 
-def init_db_pool():
+def init_db_pool(db_min_connections, db_max_connections):
     """
     Initializes the database connection pool required by the application to connect to the database.
     """
     db = config.DATABASE
     host = config.DATABASE_SERVER
     user = config.DATABASE_USER
     pw = config.DATABASE_PASSWORD
-    db_min_connections = config.DATABASE_MIN_CONNECTIONS
-    db_max_connections = config.DATABASE_MAX_CONNECTIONS
 
     global connection_pool
     if connection_pool is None:

backend/entityservice/settings.py

@@ -3,15 +3,15 @@
 Config shared between the application backend and the celery workers.
 """
 import os
-import math
-import logging
 
 
 class Config(object):
     """
-    Hard coded default configuration which can be overwritten with environment variables
+    Hard coded default configuration which can be overwritten with environment variables.
     """
 
+    # If adding or deleting any, please ensure that the changelog will mention them.
+
     DEBUG = os.getenv("DEBUG", "false") == "true"
 
     CONNEXION_STRICT_VALIDATION = os.getenv("CONNEXION_STRICT_VALIDATION", "true").lower() == "true"
@@ -33,14 +33,18 @@ class Config(object):
     DATABASE = os.getenv('DATABASE', 'postgres')
     DATABASE_USER = os.getenv('DATABASE_USER', 'postgres')
     DATABASE_PASSWORD = os.getenv('DATABASE_PASSWORD', '')
-    DATABASE_MIN_CONNECTIONS = os.getenv('DATABASE_MIN_CONNECTIONS', '1')
-    DATABASE_MAX_CONNECTIONS = os.getenv('DATABASE_MAX_CONNECTIONS', '3')
+
+    FLASK_DB_MIN_CONNECTIONS = os.getenv('FLASK_DB_MIN_CONNECTIONS', '1')
+    FLASK_DB_MAX_CONNECTIONS = os.getenv('FLASK_DB_MAX_CONNECTIONS', '10')
 
     CELERY_BROKER_URL = os.getenv(
         'CELERY_BROKER_URL',
         ('sentinel://:{}@{}:26379/0' if REDIS_USE_SENTINEL else 'redis://:{}@{}:6379/0').format(REDIS_PASSWORD, REDIS_SERVER)
     )
 
+    CELERY_DB_MIN_CONNECTIONS = os.getenv('CELERY_DB_MIN_CONNECTIONS', '1')
+    CELERY_DB_MAX_CONNECTIONS = os.getenv('CELERY_DB_MAX_CONNECTIONS', '3')
+
     CELERY_BROKER_TRANSPORT_OPTIONS = {'master_name': "mymaster"} if REDIS_USE_SENTINEL else {}
 
     CELERY_RESULT_BACKEND = CELERY_BROKER_URL
@@ -61,6 +65,7 @@ class Config(object):
 
     CELERYD_PREFETCH_MULTIPLIER = int(os.getenv('CELERYD_PREFETCH_MULTIPLIER', '1'))
     CELERYD_MAX_TASKS_PER_CHILD = int(os.getenv('CELERYD_MAX_TASKS_PER_CHILD', '4'))
+    CELERYD_CONCURRENCY = int(os.getenv("CELERYD_CONCURRENCY", '0'))
     CELERY_ACKS_LATE = os.getenv('CELERY_ACKS_LATE', 'false') == 'true'
 
     # Number of comparisons per chunk (on average).

deployment/entity-service/.gitignore

@@ -0,0 +1 @@
+charts/*

deployment/entity-service/Chart.yaml

@@ -1,6 +1,6 @@
 name: entity-service
 appVersion: 1.11.2
-version: 1.11.2
+version: 1.12.0
 description: Privacy preserving record linkage service
 sources:
   - https://github.com/data61/anonlink-entity-service