PXP-1913 PPS-251 Use Alembic for database migrations (#354)

uc-cdis · Mar 27, 2023 · fb21317 · fb21317
1 parent f9533a4
commit fb21317
Show file tree

Hide file tree

Showing 25 changed files with 1,663 additions and 803 deletions.
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -112,6 +112,77 @@
         "line_number": 107
       }
     ],
+    "alembic.ini": [
+      {
+        "type": "Basic Auth Credentials",
+        "filename": "alembic.ini",
+        "hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684",
+        "is_verified": false,
+        "line_number": 58
+      }
+    ],
+    "bin/creds.json": [
+      {
+        "type": "Secret Keyword",
+        "filename": "bin/creds.json",
+        "hashed_secret": "74c3beb5cdad8ec5540665d7b45c5504db90365e",
+        "is_verified": false,
+        "line_number": 4
+      }
+    ],
+    "deployment/Secrets/indexd_creds.json": [
+      {
+        "type": "Secret Keyword",
+        "filename": "deployment/Secrets/indexd_creds.json",
+        "hashed_secret": "bf41596f893a5f6ed0f66addb555cba581413c56",
+        "is_verified": false,
+        "line_number": 4
+      }
+    ],
+    "deployment/scripts/postgresql/postgresql_init.sql": [
+      {
+        "type": "Secret Keyword",
+        "filename": "deployment/scripts/postgresql/postgresql_init.sql",
+        "hashed_secret": "8414234c06141597b7dc1b3410b69cc49773e042",
+        "is_verified": false,
+        "line_number": 15
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "deployment/scripts/postgresql/postgresql_init.sql",
+        "hashed_secret": "c9ed73071942a54e7ec610d5a93d4a22e83e1da7",
+        "is_verified": false,
+        "line_number": 19
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "deployment/scripts/postgresql/postgresql_init.sql",
+        "hashed_secret": "8aedff83e21726bb3591555105f3d2b0c9b83e18",
+        "is_verified": false,
+        "line_number": 23
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "deployment/scripts/postgresql/postgresql_init.sql",
+        "hashed_secret": "bf41596f893a5f6ed0f66addb555cba581413c56",
+        "is_verified": false,
+        "line_number": 27
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "deployment/scripts/postgresql/postgresql_init.sql",
+        "hashed_secret": "cb93dace47db45078164ade928ba21cf27c1d8cf",
+        "is_verified": false,
+        "line_number": 31
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "deployment/scripts/postgresql/postgresql_init.sql",
+        "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
+        "is_verified": false,
+        "line_number": 34
+      }
+    ],
     "docs/drs.md": [
       {
         "type": "Hex High Entropy String",
@@ -125,7 +196,59 @@
         "filename": "docs/drs.md",
         "hashed_secret": "d46b814688b90bd6869a8a3b3220812e2ed48b1e",
         "is_verified": false,
-        "line_number": 104
+        "line_number": 80
+      }
+    ],
+    "docs/local_dev_environment.md": [
+      {
+        "type": "Secret Keyword",
+        "filename": "docs/local_dev_environment.md",
+        "hashed_secret": "91dfd9ddb4198affc5c194cd8ce6d338fde470e2",
+        "is_verified": false,
+        "line_number": 124
+      },
+      {
+        "type": "Secret Keyword",
+        "filename": "docs/local_dev_environment.md",
+        "hashed_secret": "74c3beb5cdad8ec5540665d7b45c5504db90365e",
+        "is_verified": false,
+        "line_number": 125
+      }
+    ],
+    "migrations/versions/15f2e9345ade_create_tables.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "migrations/versions/15f2e9345ade_create_tables.py",
+        "hashed_secret": "7c52f05d2823f0becb8196c04678c822ac71bcdf",
+        "is_verified": false,
+        "line_number": 14
+      }
+    ],
+    "tests/default_test_settings.py": [
+      {
+        "type": "Basic Auth Credentials",
+        "filename": "tests/default_test_settings.py",
+        "hashed_secret": "afc848c316af1a89d49826c5ae9d00ed769415f3",
+        "is_verified": false,
+        "line_number": 26
+      }
+    ],
+    "tests/postgres/migrations/test_15f2e9345ade_create_tables.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tests/postgres/migrations/test_15f2e9345ade_create_tables.py",
+        "hashed_secret": "7c52f05d2823f0becb8196c04678c822ac71bcdf",
+        "is_verified": false,
+        "line_number": 18
+      }
+    ],
+    "tests/postgres/migrations/test_legacy_schema_migration.py": [
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tests/postgres/migrations/test_legacy_schema_migration.py",
+        "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0",
+        "is_verified": false,
+        "line_number": 88
       }
     ],
     "tests/test_aliases_endpoints.py": [
@@ -150,7 +273,28 @@
         "filename": "tests/test_bundles.py",
         "hashed_secret": "fd66f51cba49640055a05a6173764b5f0241c63e",
         "is_verified": false,
-        "line_number": 151
+        "line_number": 137
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tests/test_bundles.py",
+        "hashed_secret": "168762db39e35d49d630689f2ff453b5813a9255",
+        "is_verified": false,
+        "line_number": 152
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tests/test_bundles.py",
+        "hashed_secret": "c5f0378cf93d896ecc394150943f13afa16ba766",
+        "is_verified": false,
+        "line_number": 174
+      },
+      {
+        "type": "Hex High Entropy String",
+        "filename": "tests/test_bundles.py",
+        "hashed_secret": "a2ca8b84f631b40d866b8e376d077da3527b1fe4",
+        "is_verified": false,
+        "line_number": 177
       }
     ],
     "tests/test_client.py": [
@@ -238,7 +382,7 @@
         "filename": "tests/test_deprecated_aliases_endpoints.py",
         "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0",
         "is_verified": false,
-        "line_number": 176
+        "line_number": 12
       }
     ],
     "tests/test_drs.py": [
@@ -249,16 +393,7 @@
         "is_verified": false,
         "line_number": 31
       }
-    ],
-    "tests/test_schema_migration.py": [
-      {
-        "type": "Hex High Entropy String",
-        "filename": "tests/test_schema_migration.py",
-        "hashed_secret": "5666c088b494f26cd8f63ace013992f5fc391ce0",
-        "is_verified": false,
-        "line_number": 89
-      }
     ]
   },
-  "generated_at": "2023-03-17T18:42:05Z"
+  "generated_at": "2023-03-23T20:46:39Z"
 }
diff --git a/Dockerfile b/Dockerfile
@@ -43,6 +43,7 @@ RUN poetry config virtualenvs.create false \
 RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/index/version_data.py \
     && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/index/version_data.py
 
-WORKDIR /var/www/$appname
+# directory where the app can find Alembic files
+WORKDIR /indexd
 
 CMD /dockerrun.sh
diff --git a/alembic.ini b/alembic.ini
@@ -0,0 +1,105 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# sys.path path, will be prepended to sys.path if present.
+# defaults to the current working directory.
+prepend_sys_path = .
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/bin/index_admin.py b/bin/index_admin.py
@@ -1,8 +1,14 @@
 import argparse
 import sys
+
+from alembic.config import main as alembic_main
 from cdislogging import get_logger
 
-logger = get_logger(__name__)
+from indexd.index.drivers.alchemy import Base as IndexBase
+from indexd.alias.drivers.alchemy import Base as AliasBase
+from indexd.auth.drivers.alchemy import Base as AuthBase
+
+logger = get_logger(__name__, log_level="info")
 
 
 def main(path, action=None, username=None, password=None):
@@ -13,8 +19,6 @@ def main(path, action=None, username=None, password=None):
         logger.info("Can't import local_settings, import from default")
         from indexd.default_settings import settings
     driver = settings["auth"]
-    index_driver = settings["config"]["INDEX"]["driver"]
-    alias_driver = settings["config"]["ALIAS"]["driver"]
     if action == "create":
         try:
             driver.add(username, password)
@@ -31,9 +35,17 @@ def main(path, action=None, username=None, password=None):
 
     elif action == "migrate_database":
         try:
-            logger.info("Start database migration")
-            alias_driver.migrate_alias_database()
-            index_driver.migrate_index_database()
+            engine_name = settings["config"]["INDEX"]["driver"].engine.dialect.name
+            logger.info(f"Start database migration. Engine name: {engine_name}")
+            if engine_name == "sqlite":
+                IndexBase.metadata.create_all()
+                AliasBase.metadata.create_all()
+                AuthBase.metadata.create_all()
+                settings["config"]["INDEX"]["driver"].migrate_index_database()
+                settings["config"]["ALIAS"]["driver"].migrate_alias_database()
+            else:
+                alembic_main(["--raiseerr", "upgrade", "head"])
+
         except Exception as e:
             logger.error(e)
 

diff --git a/bin/indexd b/bin/indexd
@@ -20,7 +20,7 @@ def setup_logging(log_levels, log_stream, **kwargs):
 def parse_arguments():
     parser = argparse.ArgumentParser()
 
-    parser.set_defaults(log_levels=[logging.ERROR])
+    parser.set_defaults(log_levels=[logging.INFO])
 
     parser.add_argument(
         "--debug",

diff --git a/docs/local_dev_environment.md b/docs/local_dev_environment.md
@@ -192,7 +192,7 @@ python3 -m pytest -vv --cov=indexd --cov-report xml --junitxml="test-results.xml
 You may also need to update the [test settings](./tests/default_test_settings.py) with the appropriate database connection information prior to running the tests.
 
 ```python
-settings["config"]["TEST_DB"] = "postgres://{username}:{password}@localhost:{port}/test_migration_db"
+settings["config"]["TEST_DB"] = "postgres://{username}:{password}@localhost:{port}/indexd_tests"
 ```
 
 > If you are using Azure Postgresql, you will need to include the `username@hostname` for the `username` in the connection string.  You may also need to include support for SSL in the connection string, e.g. `postgres://{username@hostname}:{password}@serverfqdn:{port}/{dbname}?sslmode=require`.