Run migrations outside discovery web server (#1389)

* First pass

* fix tests?

* force build

* not working on ci

* khaled

* maybe this?

* this one?

* here

* run migrations by hand in test

* Run migrations in conftest.py

* lint

* default test db

* create db if not exists

* Revert "create db if not exists"

This reverts commit 18ad527.

* undo this

* Cleanup

* Forgot this

* More cleanup

* does this work?

discovery-provider/alembic/env.py

@@ -36,6 +36,11 @@ def run_migrations_offline():
 
     """
     url = config.get_main_option("sqlalchemy.url")
+
+    audius_db_url = os.getenv('audius_db_url')
+    if audius_db_url:
+        url = audius_db_url
+
     context.configure(
         url=url, target_metadata=target_metadata, literal_binds=True)
 
@@ -50,6 +55,11 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
+    audius_db_url = os.getenv('audius_db_url')
+
+    if audius_db_url:
+        config.set_main_option('sqlalchemy.url', audius_db_url)
+
     connectable = engine_from_config(
         config.get_section(config.config_ini_section),
         prefix='sqlalchemy.',

discovery-provider/scripts/circle-ci.sh

@@ -10,6 +10,13 @@ function main {
   echo 'Writing flask config'
   node_modules/.bin/truffle exec scripts/_contractsLocalSetup.js -run --network test_local
   cd_discprov_repo
+
+  # run migrations
+  echo 'Running alembic migrations'
+  export PYTHONPATH='.'
+  alembic upgrade head
+  echo 'Finished running migrations'
+
   pytest -s -v --fulltrace
 }
 

discovery-provider/scripts/dev-server.sh

@@ -8,5 +8,14 @@ export FLASK_APP=src
 # enables flask development environment, including interactive debugger and reloader
 # - (http://flask.pocoo.org/docs/1.0/server/)
 export FLASK_ENV=development
+
+# run db migrations
+if [ "$audius_db_run_migrations" != false ] ; then
+  echo "Running alembic migrations"
+  export PYTHONPATH='.'
+  alembic upgrade head
+  echo "Finished running migrations"
+fi
+
 exec flask run --host=0.0.0.0
 

discovery-provider/scripts/prod-server.sh

@@ -23,6 +23,14 @@ else
   THREADS="${audius_gunicorn_threads}"
 fi
 
+# run db migrations
+if [ "$audius_db_run_migrations" != false ] ; then
+  echo "Running alembic migrations"
+  export PYTHONPATH='.'
+  alembic upgrade head
+  echo "Finished running migrations"
+fi
+
 # If a worker class is specified, use that. Otherwise, use sync workers.
 if [[ -z "${audius_gunicorn_worker_class}" ]]
 then

discovery-provider/src/__init__.py

@@ -17,9 +17,6 @@
 from flask.json import JSONEncoder
 from flask_cors import CORS
 
-import alembic
-import alembic.config  # pylint: disable=E0611
-
 from src import exceptions
 from src.queries import queries, search, search_queries, health_check, notifications
 from src.api.v1 import api as api_v1
@@ -248,20 +245,6 @@ def default(self, o):
 
         i += 1
 
-    # Conditionally perform alembic database upgrade to HEAD during
-    # flask initialization
-    if mode == "app":
-        alembic_dir = os.getcwd()
-        alembic_config = alembic.config.Config(f"{alembic_dir}/alembic.ini")
-        alembic_config.set_main_option("sqlalchemy.url", str(database_url))
-        with helpers.cd(alembic_dir):
-            # run db migrations unless `run_migrations` overridden to false. default value is true
-            # need to call with getboolean because configparser doesn't allow you to
-            # store non string types and it coerces into bool for you
-            if shared_config.getboolean("db", "run_migrations"):
-                logger.info('running alembic db migrations')
-                alembic.command.upgrade(alembic_config, "head")
-
     if test_config is not None:
         # load the test config if passed in
         app.config.update(test_config)

discovery-provider/tests/conftest.py

@@ -1,10 +1,13 @@
 from __future__ import absolute_import
+import os
 import pytest
 from sqlalchemy_utils import database_exists, drop_database
 from web3 import HTTPProvider, Web3
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from pytest_postgresql import factories
+import alembic
+import alembic.config
 from src import create_app, create_celery
 from src.utils import helpers
 from src.models import Base
@@ -38,6 +41,14 @@ def app():
 
     # Create application for testing
     discovery_provider_app = create_app(TEST_CONFIG_OVERRIDE)
+
+    # run db migrations because the db gets dropped at the start of the tests
+    alembic_dir = os.getcwd()
+    alembic_config = alembic.config.Config(f"{alembic_dir}/alembic.ini")
+    alembic_config.set_main_option("sqlalchemy.url", str(DB_URL))
+    with helpers.cd(alembic_dir):
+        alembic.command.upgrade(alembic_config, "head")
+
     yield discovery_provider_app