diff --git a/.circleci/config.yml b/.circleci/config.yml index 6537b15f..1ca7baf2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,6 +69,21 @@ jobs: echo 'export PATH=$PWD:$PATH' >> $BASH_ENV source $BASH_ENV fastANI -h + - run: + name: install sqlite3 + command: | + version=version-3.37.0 + wget https://github.com/sqlite/sqlite/archive/refs/tags/${version}.tar.gz + tar xzf ${version}.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite-${version}/configure ;# Run the configure script + make ;# Run the makefile. + make sqlite3.c ;# Build the "amalgamation" source file + make test ;# Run some tests (requires Tcl) + echo 'export PATH=~/repo/bld:$PATH' >> $BASH_ENV + source $BASH_ENV + echo $PATH - run: diff --git a/.gitignore b/.gitignore index 1bb86dd5..c7f50db4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ # Scratch directory for local testing scratch/ +alembic/version.txt +*/*add_third_column.py # Mac-related dreck .DS_Store @@ -72,4 +74,4 @@ venv-* # Extra documentation output classes_pyani.pdf -packages_pyani.pdf \ No newline at end of file +packages_pyani.pdf diff --git a/Makefile b/Makefile index d855e026..a1677e34 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ setup_conda: @conda install --file requirements-dev.txt --yes @conda install --file requirements.txt --yes + @conda config --add channels blaze @conda install --file requirements-thirdparty.txt --yes @conda install --file requirements-fastani.txt --yes @conda install --file requirements-pyqt-conda.txt --yes diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..a340f706 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,99 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. Valid values are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # default: use os.pathsep + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = sqlite:///.pyani/pyanidb + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 00000000..69c7ffa1 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,104 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +import os + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +dbpath = os.environ.get("PYANI_DATABASE") +script_dir = os.environ.get("ALEMBIC_MIGRATIONS_DIR") +url = f"sqlite:///{dbpath}" +config.set_main_option("sqlalchemy.url", url) +config.set_main_option("script_location", "alembic") + + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + # Track the current version of the database in a local file + # this value is used in the --dry-run option + version_file = os.path.join( + os.path.dirname(config.config_file_name), "alembic/version.txt" + ) + if os.path.exists(version_file): + current_version = open(version_file).read().strip() + else: + current_version = None + context.configure(dialect_name="sqlite", starting_rev=current_version) + + # Perform the dry run + run_migrations_offline() + + # Write 'new' version to file + end_version = context.get_revision_argument() + if end_version and end_version != current_version: + open(version_file, "w").write(end_version) + elif end_version is None: + open(version_file, "w").write("base") +else: + run_migrations_online() diff --git a/alembic/version.txt b/alembic/version.txt new file mode 100644 index 00000000..7fe54291 --- /dev/null +++ b/alembic/version.txt @@ -0,0 +1 @@ +65538af5a5e1 \ No newline at end of file diff --git a/alembic/versions/92f7f6b1626e_add_fastani_columns.py b/alembic/versions/92f7f6b1626e_add_fastani_columns.py new file mode 100644 index 00000000..8c39a1d7 --- /dev/null +++ b/alembic/versions/92f7f6b1626e_add_fastani_columns.py @@ -0,0 +1,123 @@ +"""add fastani columns + +Revision ID: 92f7f6b1626e +Revises: +Create Date: 2022-02-07 22:57:35.779356 + +""" +from alembic import op +import sqlalchemy as sa +import sys + +from sqlalchemy import ( + Column, + # Table, + # MetaData, + # ForeignKey, + Integer, + # String, + Float, + # Boolean, + UniqueConstraint, +) + +# revision identifiers, used by Alembic. +revision = "92f7f6b1626e" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # op.add_column("comparisons", sa.Column("kmersize", sa.Integer)) + # op.add_column("comparisons", sa.Column("minmatch", sa.Float)) + """ + comparisons = Table( + "comparisons", + meta, + Column("comparisons_id", Integer, primary_key=True), + Column("query_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("subject_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("aln_length", Integer), + Column("sim_errs", Integer), + Column("identity", Float), + Column("cov_query", Float), + Column("cov_subject", Float), + Column("program", String), + Column("version", String), + Column("fragsize", Integer), + Column("maxmatch", Boolean), + UniqueConstraint( + "query_id", + "subject_id", + "program", + "version", + "fragsize", + "maxmatch", + name="base_reqs", + ), + ) + """ + with op.batch_alter_table("comparisons") as batch_op: + # batch_op.add_column(sa.Column("kmersize", sa.Integer, default=None)) + # batch_op.add_column(sa.Column("minmatch", sa.Float, default=None)) + batch_op.drop_constraint("base_reqs") + batch_op.add_column(sa.Column("kmersize", sa.Integer, default=None)) + batch_op.add_column(sa.Column("minmatch", sa.Float, default=None)) + batch_op.create_unique_constraint( + "fastani_reqs", + [ + "query_id", + "subject_id", + "program", + "version", + "fragsize", + "maxmatch", + "kmersize", + "minmatch", + ], + ) + + +def downgrade(): + # op.drop_constraint("comparisons", 'kmersize') + # op.drop_column("comparisons", "kmersize") + """ + comparisons = Table( + "comparisons", + meta, + Column("comparisons_id", Integer, primary_key=True), + Column("query_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("subject_id", Integer, ForeignKey("genomes.genome_id"), nullable=False), + Column("aln_length", Integer), + Column("sim_errs", Integer), + Column("identity", Float), + Column("cov_query", Float), + Column("cov_subject", Float), + Column("program", String), + Column("version", String), + Column("fragsize", Integer), + Column("maxmatch", Boolean), + Column("kmersize", Integer), + Column("minmatch", Float), + UniqueConstraint( + "query_id", + "subject_id", + "program", + "version", + "fragsize", + "maxmatch", + "kmersize", + "minmatch", + name="fastani_reqs", + ), + ) + """ + with op.batch_alter_table("comparisons") as batch_op: + batch_op.drop_column("kmersize") + batch_op.drop_column("minmatch") + batch_op.drop_constraint("fastani_reqs") + batch_op.create_unique_constraint( + "base_reqs", + ["query_id", "subject_id", "program", "version", "fragsize", "maxmatch"], + ) diff --git a/alt_alembic_config.ini b/alt_alembic_config.ini new file mode 100644 index 00000000..1792ea18 --- /dev/null +++ b/alt_alembic_config.ini @@ -0,0 +1,103 @@ +# a multi-database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to multidb/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" +# version_locations = %(here)s/bar:%(here)s/bat:multidb/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. Valid values are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # default: use os.pathsep + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +databases = pyanidb_altdb + +[pyanidb_altdb] +sqlalchemy.url = sqlite:///.pyani/pyanidb +script_location = %(here)s/alembic + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/docs/run_fastani.rst b/docs/run_fastani.rst index a1c73537..42a74a89 100644 --- a/docs/run_fastani.rst +++ b/docs/run_fastani.rst @@ -37,7 +37,7 @@ The basic form of the command is: pyani fastani -i -o -This instructs ``pyani`` to perform fast ANI on the genome FASTA files in ````, and write any output files to ````. For example, the following command performs fastANI on genomes in the directory ``genomes`` and writes output to a new directory ``genoems_fastANI``: +This instructs ``pyani`` to perform fast ANI on the genome FASTA files in ````, and write any output files to ````. For example, the following command performs fastANI on genomes in the directory ``genomes`` and writes output to a new directory ``genomes_fastANI``: .. code-block:: bash @@ -49,7 +49,7 @@ This instructs ``pyani`` to perform fast ANI on the genome FASTA files in ````. Please see `SQLite`_ for more details. + +The expectation is that the above command is what most ``pyani`` users will need in terms of migrations, as this will perform the necessary alterations to the database to accommodate changes merged into ``pyani`` with commit `fastani_merge`_, which broke compatibility with previous database schemas. + + +-------------- +Advanced Usage +-------------- + +However, there may be cases where it is necessary to downgrade a database to a specific, *earlier*, version, or the user wants to perform a dry run. Some users may also want to specify a non-standard location for an Alembic config file, or need to use this in a project with a ``multidb`` setup. Some examples of these advanced cases can be found below. + +~~~~~~~~~~~~~~~~~~~ +Performing a dry run +~~~~~~~~~~~~~~~~~~~ + +.. NOTE:: + + This option has been implemented, but tests for it are still in the works. + +This following command creates an SQL file, ``./test_database.downgrade.YYYY-MM-DD_HH-MM-SS.sql``, (in the same directory as ``./test_database``) containing the raw SQL that would produce the necessary changes to the database to migrate it to the specified version (in this case, downgrading it to ``base``): + +.. code-block:: bash + + pyani versiondb --dbpath ./test_database --dry-run down head:base + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using a different config file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, ``Alembic``, and therefore ``pyani versiondb``, assume a config file, ``alembic.ini``, located in a standard location, which in the case of ``pyani``, is among the package files. To use a different file, add the ``-c`` or ``--config`` flag to your ``pyani`` command: + +.. code-block:: bash + + pyani versiondb --dbpath ./test_database --upgrade head --config ./config.ini + +If you need to specify additional settings for `Alembic`_, or have multiple databases in your ``pyani`` project (especially if not all should be upgraded/downgraded), this is the way you will need to use this option. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using pyani versiondb in a multidb setup +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. NOTE:: + + For information about how to set up a project with multiple databases managed by `Alembic`_, please see the `Alembic`_ documentation on `working with multiple databases `_. + +To specify a single database in a ``multidb`` setup, use the ``-n`` or ``--name`` option, along with the designation for the correct database from your ``multidb`` config file: + +.. code-block:: bash + + pyani versiondb --dbpath ./test_database --upgrade head --config ./multidb.ini --name database2 + +---------- +References +---------- + +.. _SQLite: https://www.sqlite.org/docs.html + +.. _fastani_merge: https://github.com/widdowquinn/pyani/pull/299/commits/254346cae24058b745bd9496b4205400da03fb4c + +.. _Alembic: https://alembic.sqlalchemy.org/en/latest/ diff --git a/docs/subcmd_versiondb.rst b/docs/subcmd_versiondb.rst new file mode 100644 index 00000000..19a74264 --- /dev/null +++ b/docs/subcmd_versiondb.rst @@ -0,0 +1,43 @@ +. _pyani-subcmd-versiondb: + +=============== +pyani versiondb +=============== + +The ``versiondb`` subcommand will migrate a ``pyani`` database between versions. `fastani_merge`_ + +usage: pyani versiondb [-h] [-l LOGFILE] [-v] [--debug] [--disable_tqdm] [--version] [--citation] [--dbpath DBPATH] + (--upgrade [VERSION] | --downgrade [VERSION] | --dry-run DIRECTION START:END) [--alembic_exe ALEMBIC_EXE] [-n NAME] [-c FILE] + +One of --upgrade, --downgrade, or --dry-run must be specified. + +optional arguments: + -h, --help show this help message and exit + -l LOGFILE, --logfile LOGFILE + logfile location (default: None) + -v, --verbose report verbose progress to log (default: False) + --debug report debug messages to log (default: False) + --disable_tqdm Turn off tqdm progress bar (default: False) + --version + --citation + --dbpath DBPATH path to pyani database (default: .pyani/pyanidb) + --upgrade [VERSION] update an existing database to a newer schema; if no argument is given, 'head' will be used (default: None) + --downgrade [VERSION] + revert an existing database to a older schema; if no argument is given, 'base' will be used (default: None) + --dry-run DIRECTION START:END + produce the SQL that would be run in migrations, without altering the database; a direction {upgrade or downgrade} and start and end versions e.g., {head:base, + base:head, base:} must be specified (default: None) + --alembic_exe ALEMBIC_EXE + path to alembic executable (default: alembic) + -n NAME, --name NAME used to specify an individual database in a multidb setup (default: None) + -c FILE, --config FILE + used to specify a config file for alembic (default: None) + + +---------- +References +---------- + +.. _SQLite: https://www.sqlite.org/docs.html + +.. _fastani_merge: https://github.com/widdowquinn/pyani/pull/299/commits/254346cae24058b745bd9496b4205400da03fb4c diff --git a/docs/subcommands.rst b/docs/subcommands.rst index dd12fb5e..42a31590 100644 --- a/docs/subcommands.rst +++ b/docs/subcommands.rst @@ -27,3 +27,4 @@ This document links out to detailed instructions for each of the ``pyani`` subco subcmd_classify subcmd_listdeps subcmd_fastani + subcmd_versiondb diff --git a/docs/testing.rst b/docs/testing.rst index ee40949b..995b4c2d 100644 --- a/docs/testing.rst +++ b/docs/testing.rst @@ -9,6 +9,11 @@ We are currently writing tests formatted for the `pytest`_ package, for testing .. WARNING:: Some tests are still targeted at `nosetests`_, which is in maintenance mode and, although we are still in transition, our plan is to change the test framework completely to use `pytest`_. +.. Note:: + Tests for database migration (upgrades and downgrades) using ``alembic`` require ``sqldiff``, one of the ``sqlite-tools`` that is available as part of the compiled binaries, or source code, from `sqlite downloads`_. This tool is not available through package managers, and is *only* required to run the related tests locally, not to have the database migration functionality ``alembic`` affords ``pyani`` users; as such, it has been installed as part of our maintenance testing on CircleCI, and by our devs, but it does not come as part of the ``pyani`` installation. The related tests will be skipped if ``sqldiff`` is not present on the system. + + Should you wish to run those tests locally, installation details can be found below under **Installing sqldiff**. + ------------------------ Test directory structure ------------------------ @@ -71,7 +76,36 @@ And to test only "multiple command generation" we can issue the following: nosetests -v tests/test_anim.py:TestNUCmerCmdline.test_multi_cmd_generation +------------------ +Installing sqldiff +------------------ + +The CircleCI testing setup for ``pyani`` uses ``sqlite 3.37.0``, which comes with ``sqldiff``. This version is available to download as a zip file from GitHub: `sqlite 3.37.0 download`_. + +Installation instructions are available in the `sqlite Readme`_, but will need to be tweaked for the older archive. For example, in CircleCI (a Unix environment), we do: + +.. code-block:: bash + + version=version-3.37.0 ;# Pull version number into a variable + wget https://github.com/sqlite/sqlite/archive/refs/tags/${version}.tar.gz + tar xzf ${version}.tar.gz ;# Unpack the source tree into "sqlite" + mkdir bld ;# Build will occur in a sibling directory + cd bld ;# Change to the build directory + ../sqlite-${version}/configure ;# Run the configure script + make ;# Run the makefile. + make sqlite3.c ;# Build the "amalgamation" source file + make test ;# Run some tests (requires Tcl) + echo "export PATH=~/repo/bld:$PATH" >> $BASH_ENV ;# Add to shell configuration + source $BASH_ENV ;# Effect changes + +.. Note:: + This will also place ``sqlite 3.37.0`` into your ``$PATH``. ``pyani`` installs ``sqlite3`` via a package manager, already. If having two copies is not desirable, you may wish to copy the ``sqldiff`` binary itself into somewhere already on your path, instead. .. _nosetests: https://nose.readthedocs.io/en/latest/ .. _pytest: https://docs.pytest.org/en/latest/ +.. _sqlite downloads:: https://www.sqlite.org/download.html + + +.. _sqlite 3.37.0 download:: https://github.com/sqlite/sqlite/tags +.. _sqlite Readme:: https://github.com/sqlite/sqlite diff --git a/pyani/dependencies.py b/pyani/dependencies.py index 1233ce11..139085df 100644 --- a/pyani/dependencies.py +++ b/pyani/dependencies.py @@ -59,6 +59,7 @@ "seaborn", "sqlalchemy", "tqdm", + "alembic", ] DEVELOPMENT = [ diff --git a/pyani/pyani_config.py b/pyani/pyani_config.py index caa268c0..22f70174 100644 --- a/pyani/pyani_config.py +++ b/pyani/pyani_config.py @@ -55,6 +55,9 @@ FORMATDB_DEFAULT = Path("formatdb") QSUB_DEFAULT = Path("qsub") FASTANI_DEFAULT = Path("fastANI") +ALEMBIC_DEFAULT = Path("alembic") +SQLITE_DEFAULT = Path("sqlite3") +SQLDIFF_DEFAULT = Path("sqldiff") # Stems for output files ANIM_FILESTEMS = ( diff --git a/pyani/pyani_orm.py b/pyani/pyani_orm.py index 40864afc..2feef423 100644 --- a/pyani/pyani_orm.py +++ b/pyani/pyani_orm.py @@ -53,7 +53,17 @@ from sqlalchemy import and_ # type: ignore import sqlalchemy from sqlalchemy import UniqueConstraint, create_engine, Table -from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Float, Boolean +from sqlalchemy import ( + Column, + DateTime, + ForeignKey, + Integer, + String, + Float, + Boolean, + types, + PrimaryKeyConstraint, +) from sqlalchemy.ext.declarative import declarative_base # type: ignore from sqlalchemy.orm import relationship, sessionmaker # type: ignore @@ -104,6 +114,14 @@ class LabelTuple(NamedTuple): class_label: str +class Alembic(Base): + __tablename__ = "alembic_version" + __table_args__ = (PrimaryKeyConstraint("version_num", name="alembic_version_pkc"),) + # __table_args__ = tuple([PrimaryKeyConstraint('alembic_version_pkc', name='version_num')]) + + version_num = Column(types.VARCHAR(32), primary_key=True, nullable=False) + + class Label(Base): """Describes relationship between genome, run and genome label. @@ -279,6 +297,7 @@ class Comparison(Base): "maxmatch", "kmersize", "minmatch", + name="fastani_reqs", ), ) @@ -293,7 +312,12 @@ class Comparison(Base): program = Column(String) version = Column(String) fragsize = Column(Integer) # in fastANI this is fragLength - maxmatch = Column(Boolean) # in fastANi this is Null + + # create_constraint keyword is needed for portability between sqlalchemy 1.3 and 1.4 + if float(sqlalchemy.__version__.rsplit(".", 1)[0]) < 1.4: + maxmatch = Column(Boolean) + else: + maxmatch = Column(Boolean, create_constraint=True) # in fastANi this is Null kmersize = Column(Integer) minmatch = Column(Float) @@ -347,6 +371,19 @@ def get_session(dbpath: Path) -> Any: return Session() +def add_alembic(session, version_num): + try: + db_version = Alembic(version_num=version_num) + except Exception: + raise PyaniORMException(f"Could not create Alembic() object {version_num}") + try: + session.add(db_version) + session.commit() + except Exception: + raise PyaniORMException(f"Could not add version {version_num}") + return db_version.version_num + + def get_comparison_dict(session: Any) -> Dict[Tuple, Any]: """Return a dictionary of comparisons in the session database. diff --git a/pyani/scripts/parsers/__init__.py b/pyani/scripts/parsers/__init__.py index e1fc790c..1f7b1edf 100644 --- a/pyani/scripts/parsers/__init__.py +++ b/pyani/scripts/parsers/__init__.py @@ -58,6 +58,7 @@ common_parser, run_common_parser, listdeps_parser, + versiondb_parser, ) @@ -76,6 +77,8 @@ def parse_cmdline(argv: Optional[List] = None) -> Namespace: index genome sequence files in a subdirectory, for analysis - createdb generate SQLite database for data and analysis results + - versiondb + upgrade/downgrade pyani SQLite database between schemas - anim conduct ANIm analysis - anib @@ -119,6 +122,7 @@ def parse_cmdline(argv: Optional[List] = None) -> Namespace: download_parser.build(subparsers, parents=[parser_common]) index_parser.build(subparsers, parents=[parser_common]) createdb_parser.build(subparsers, parents=[parser_common]) + versiondb_parser.build(subparsers, parents=[parser_common]) anim_parser.build( subparsers, parents=[parser_common, parser_scheduler, parser_run_common] ) diff --git a/pyani/scripts/parsers/versiondb_parser.py b/pyani/scripts/parsers/versiondb_parser.py new file mode 100644 index 00000000..b03ef1f7 --- /dev/null +++ b/pyani/scripts/parsers/versiondb_parser.py @@ -0,0 +1,111 @@ +"""Provides parser for versiondb subcommand.""" + +from argparse import ( + ArgumentDefaultsHelpFormatter, + ArgumentParser, + _SubParsersAction, + Action, +) +from pathlib import Path +from typing import List, Optional + +from pyani import pyani_config + +from pyani.scripts import subcommands + + +class DryRunAction(Action): + def __init__(self, option_strings, **kwargs): + super().__init__(option_strings, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + print("%r %r %r" % (namespace, values, option_string)) + setattr(namespace, "direction", values.pop(0)) + setattr(namespace, "dry_run", values.pop(0)) + + +def build( + subps: _SubParsersAction, parents: Optional[List[ArgumentParser]] = None +) -> None: + """Return a command-line parser for the versiondb subcommand. + + :param subps: collection of subparsers in main parser + :param parents: parsers from which arguments are inherited + + """ + parser = subps.add_parser( + "versiondb", + parents=parents, + formatter_class=ArgumentDefaultsHelpFormatter, + description="One of --upgrade, --downgrade, or --dry-run must be specified.", + ) + # Path to database (default: .pyani/pyanidb) + parser.add_argument( + "--dbpath", + action="store", + dest="dbpath", + default=Path(".pyani/pyanidb"), + type=Path, + help="path to pyani database", + ) + direction = parser.add_mutually_exclusive_group(required=True) + direction.add_argument( + "--upgrade", + action="store", + dest="upgrade", + nargs="?", + default=None, + const="head", + metavar="VERSION", + help="update an existing database to a newer schema; if no argument is given, 'head' will be used", + ) + direction.add_argument( + "--downgrade", + action="store", + dest="downgrade", + nargs="?", + default=None, + const="base", + metavar="VERSION", + help="revert an existing database to a older schema; if no argument is given, 'base' will be used", + ) + direction.add_argument( + "--dry-run", + action=DryRunAction, + dest="dry_run", + required=False, + nargs=2, + metavar=("DIRECTION", "START:END"), + default=None, + help="produce the SQL that would be run in migrations, without altering the database; a direction {upgrade or downgrade} and start and end versions e.g., {head:base, base:head, base:} must be specified", + ) + parser.add_argument( + "--alembic_exe", + action="store", + dest="alembic_exe", + required=False, + default=pyani_config.ALEMBIC_DEFAULT, + type=Path, + help="path to alembic executable", + ) + parser.add_argument( + "-n", + "--name", + action="store", + dest="dbname", + default=None, + required=False, + metavar="NAME", + help="used to specify an individual database in a multidb setup", + ) + parser.add_argument( + "-c", + "--config", + action="store", + dest="alembic_config", + default=None, + required=False, + metavar="FILE", + help="used to specify a config file for alembic", + ) + parser.set_defaults(func=subcommands.subcmd_versiondb) diff --git a/pyani/scripts/subcommands/__init__.py b/pyani/scripts/subcommands/__init__.py index 7e5e92ef..863a28ef 100644 --- a/pyani/scripts/subcommands/__init__.py +++ b/pyani/scripts/subcommands/__init__.py @@ -43,6 +43,7 @@ from .subcmd_anim import subcmd_anim from .subcmd_classify import subcmd_classify from .subcmd_createdb import subcmd_createdb +from .subcmd_versiondb import subcmd_versiondb from .subcmd_download import subcmd_download from .subcmd_index import subcmd_index from .subcmd_listdeps import subcmd_listdeps diff --git a/pyani/scripts/subcommands/subcmd_createdb.py b/pyani/scripts/subcommands/subcmd_createdb.py index ea445b1c..bf69dc4f 100644 --- a/pyani/scripts/subcommands/subcmd_createdb.py +++ b/pyani/scripts/subcommands/subcmd_createdb.py @@ -45,6 +45,8 @@ from pyani import pyani_orm +from pyani.pyani_orm import PyaniORMException, get_session + def subcmd_createdb(args: Namespace) -> int: """Create an empty pyani database. @@ -72,4 +74,28 @@ def subcmd_createdb(args: Namespace) -> int: logger.info("Creating pyani database at %s", args.dbpath) pyani_orm.create_db(args.dbpath) + # Get connection to existing database. This may or may not have data + logger.debug("Connecting to database %s", args.dbpath) + try: + session = get_session(args.dbpath) + except Exception: + logger.error( + "Could not connect to database %s (exiting)", args.dbpath, exc_info=True + ) + raise SystemExit(1) + + # Add information about the database version to the database + logger.debug("Adding database version to database %s...", args.dbpath) + try: + version_num = "92f7f6b1626e" # most current version (fastani) + pyani_orm.add_alembic(session, version_num=version_num) + except PyaniORMException: + logger.error( + "Could not add db_version %s to the database (exiting)", + version_num, + exc_info=True, + ) + raise SystemExit(1) + logger.debug("...added db_version: %s to the database", version_num) + return 0 diff --git a/pyani/scripts/subcommands/subcmd_versiondb.py b/pyani/scripts/subcommands/subcmd_versiondb.py new file mode 100644 index 00000000..e34b87e7 --- /dev/null +++ b/pyani/scripts/subcommands/subcmd_versiondb.py @@ -0,0 +1,92 @@ +import logging +import os +import platform +import re +import shutil +import subprocess +import sys +import datetime + +from logging import Logger +from pathlib import Path +from typing import List + +from pyani import pyani_config + +from argparse import Namespace + +from pyani import ( + pyani_orm, + versiondb, +) + +from pyani.pyani_tools import termcolor + + +def subcmd_versiondb(args: Namespace) -> int: + """Up/downgrade a pyani database. + + :param args: Namespace, command-line arguments + :param logger: logging object + """ + # Create logger + logger = logging.getLogger(__name__) + + # Announce what's happening + logger.info(termcolor("Database: %s", "cyan"), str(args.dbpath.resolve())) + if args.downgrade: + logger.info(termcolor("Downgrading database to %s", bold=True), args.downgrade) + else: + logger.info(termcolor("Upgrading database to %s", bold=True), args.upgrade) + + # Get current alembic version + alembic_version = versiondb.get_version(args.alembic_exe) + logger.info(termcolor("Alembic version: %s", "cyan"), alembic_version) + + # If the database doesn't exist, raise an error + if not args.dbpath.is_file(): + logger.error("Database %s does not exist (exiting)", args.dbpath) + raise SystemError(1) + + # Create environment variables for alembic to access + abs_path = args.dbpath.resolve() + os.environ["PYANI_DATABASE"] = str(abs_path) + os.environ["ALEMBIC_MIGRATIONS_DIR"] = "alembic" + + # Create a backup of the database + timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + + # Up/downgrade database + if args.dry_run: + logger.info( + "(Dry run): Migrating database from %s to %s", *args.dry_run.split(":") + ) + versiondb.migrate_database(args.direction, args, timestamp=timestamp) + elif args.downgrade: + logger.info("Downgrading database schema to: %s", args.downgrade) + shutil.copy(args.dbpath.resolve(), f"{abs_path}.{timestamp}.bak") + versiondb.migrate_database("downgrade", args, timestamp=timestamp) + elif args.upgrade: + logger.info("Upgrading database schema to: %s", args.upgrade) + shutil.copy(args.dbpath.resolve(), f"{abs_path}.{timestamp}.bak") + versiondb.migrate_database("upgrade", args, timestamp=timestamp) + + return 0 + + +# Valid SQLite URL forms are: +# sqlite:///:memory: (or, sqlite://) +# sqlite:///relative/path/to/file.db +# sqlite:////absolute/path/to/file.db + +# alembic init --package # alembic +# need to change location of sqlalchemy.url + +# - alembic.ini +# - alembic +# - __init__.py +# - env.py +# - README +# - script.py.mako +# - versions +# - __init__.py diff --git a/pyani/versiondb.py b/pyani/versiondb.py new file mode 100644 index 00000000..91e0ce23 --- /dev/null +++ b/pyani/versiondb.py @@ -0,0 +1,134 @@ +import logging +import os +import platform +import re +import shutil +import subprocess +import sys + +from typing import List + +from pathlib import Path + +from pyani import pyani_config + +from argparse import Namespace + + +def get_version(alembic_exe: Path = pyani_config.ALEMBIC_DEFAULT) -> str: + """Return ALembic package version as a string. + + :param alembic_exe: path to Alembic executable + + We expect Alembic to return a string on STDOUT as + + .. code-block:: bash + + $ alembic --version + alembic 1.7.5 + + we concatenate this with the OS name. + + The following circumstances are explicitly reported as strings: + + - a value of None given for the executable + - no executable at passed path + - non-executable file at passed path (this includes cases where the user doesn't have execute permissions on the file) + - no version info returned + """ + + try: + # Returns a TypeError if `alembic_exe` is None + try: + alembic_path = shutil.which(alembic_exe) # type:ignore + except TypeError: + return f"expected path to alembic executable; received {alembic_exe}" + # Returns a TypeError if `alembic_path` is not on the PATH + alembic_path = Path(alembic_path) + except TypeError: + return f"{alembic_exe} is not found in $PATH" + + if not alembic_path.is_file(): # no executable + return f"No alembic at {alembic_path}" + + # This should catch cases when the file can't be executed by the user + if not os.access(alembic_path, os.X_OK): # file exists but not executable + return f"alembic exists at {alembic_path} but not executable" + + cmdline = [alembic_exe, "--version"] # type: List + result = subprocess.run( + cmdline, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + + if result.stdout: + match = re.search(r"(?<=alembic\s)[0-9\.]*", str(result.stdout, "utf-8")) + + version = match.group() # type: ignore + + if 0 == len(version.strip()): + return f"alembic exists at {alembic_path} but could not retrieve version" + + return f"{platform.system()}_{version} ({alembic_path})" + + +def get_optional_args(args: Namespace): + opts = [] + if args.dbname: + opts.extend(["-n", args.dbname]) + if args.alembic_config: + opts.extend(["-c", args.alembic_config]) + return opts + + +def construct_alembic_cmdline( + direction, + args: Namespace, + alembic_exe=pyani_config.ALEMBIC_DEFAULT, +): + if args.dry_run: + return [ + str(alembic_exe), + direction, + args.dry_run, + "--sql", + *get_optional_args(args), + ] # FAILED: downgrade with --sql requires : + elif direction == "upgrade": + return [str(alembic_exe), *get_optional_args(args), direction, args.upgrade] + elif direction == "downgrade": + return [str(alembic_exe), *get_optional_args(args), direction, args.downgrade] + + +def log_output_and_errors(result, direction, args: Namespace, timestamp=None): + logger = logging.getLogger(__name__) + if result.stdout: + logger.info("Alembic stdout:") + for line in str(result.stdout, "utf-8").split("\n"): + if line: + logger.info(line) + if args.dry_run: + abs_path = args.dbpath.resolve() + with open(f"{abs_path}.{direction}.{timestamp}.sql", "w") as sqlfile: + for line in str(result.stdout, "utf-8").split("\n"): + if line: + sqlfile.write(f"{line}\n") + + if result.stderr: + logger.info("Alembic stderr:") + for line in str(result.stderr, "utf-8").split("\n"): + if line: + logger.info(line) + + +def migrate_database(direction, args: Namespace, timestamp=None): + cmdline = construct_alembic_cmdline(direction, args) + + result = subprocess.run( + cmdline, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + log_output_and_errors(result, direction, args, timestamp) diff --git a/requirements-pip.txt b/requirements-pip.txt index 5467acdc..e9331f95 100644 --- a/requirements-pip.txt +++ b/requirements-pip.txt @@ -1,2 +1,3 @@ pytest-ordering sphinx-rtd-theme +sqldiff diff --git a/requirements-thirdparty.txt b/requirements-thirdparty.txt index bfe39e58..d294fd9d 100644 --- a/requirements-thirdparty.txt +++ b/requirements-thirdparty.txt @@ -1,3 +1,5 @@ blast blast-legacy mummer +fastani +sqlite3 diff --git a/requirements.txt b/requirements.txt index dd5ab215..74cc2f5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ scipy seaborn sqlalchemy tqdm +alembic diff --git a/setup.py b/setup.py index ee8e3ef6..26177663 100644 --- a/setup.py +++ b/setup.py @@ -99,7 +99,14 @@ ] }, packages=setuptools.find_packages(), - package_data={"pyani": ["tests/test_JSpecies/*.tab"]}, + package_data={ + "pyani": [ + "tests/test_JSpecies/*.tab", + "alembic.ini", + "alembic/*", + "alembic/**/*", + ] + }, include_package_date=True, install_requires=[ "biopython", diff --git a/tests/conftest.py b/tests/conftest.py index 4b88f323..dd150efd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -142,6 +142,18 @@ def dir_graphics_in(): return FIXTUREPATH / "graphics" +@pytest.fixture +def dir_versiondb_in(): + """Input files for versiondb tests.""" + return FIXTUREPATH / "versiondb" + + +@pytest.fixture +def dir_versiondb_out(): + """Output files for versiondb tests.""" + return TESTSPATH / "test_output/subcmd_versiondb" + + @pytest.fixture def dir_seq(): """Sequence files for tests.""" diff --git a/tests/fixtures/versiondb/base_pyanidb b/tests/fixtures/versiondb/base_pyanidb new file mode 100644 index 00000000..2b6634f1 Binary files /dev/null and b/tests/fixtures/versiondb/base_pyanidb differ diff --git a/tests/fixtures/versiondb/head_pyanidb b/tests/fixtures/versiondb/head_pyanidb new file mode 100644 index 00000000..904de8a7 Binary files /dev/null and b/tests/fixtures/versiondb/head_pyanidb differ diff --git a/tests/test_cli_parsing.py b/tests/test_cli_parsing.py index ecdeaa40..a064dd27 100644 --- a/tests/test_cli_parsing.py +++ b/tests/test_cli_parsing.py @@ -81,7 +81,11 @@ def test_createdb(args_createdb, monkeypatch): def mock_return_none(*args, **kwargs): return None + def mock_add_alembic(*args, **kwargs): + return "TEST VERSION" + monkeypatch.setattr(pyani_orm, "create_db", mock_return_none) + monkeypatch.setattr(pyani_orm, "add_alembic", mock_add_alembic) pyani_script.run_main(args_createdb) diff --git a/tests/test_subcmd_10_versiondb.py b/tests/test_subcmd_10_versiondb.py new file mode 100644 index 00000000..ec5d862e --- /dev/null +++ b/tests/test_subcmd_10_versiondb.py @@ -0,0 +1,58 @@ +import logging +import unittest + +from argparse import Namespace +from typing import NamedTuple +from pathlib import Path + +from pyani.scripts import subcommands + + +# Convenience struct with paths to third-party executables +class ThirdPartyExes(NamedTuple): + alembic_exe: Path + + +# Convenience struct with paths to working directories +class DirPaths(NamedTuple): + indir: Path + outdir: Path + + +class TestVersiondbSubcommand(unittest.TestCase): + + """Class defining tests of the pyani versiondb subcommand.""" + + def setUp(self): + """Configure parameters for tests.""" + testdir = Path("tests") + self.dirpaths = DirPaths( + testdir / "test_input" / "subcmd_versiondb", + testdir / "test_output" / "subcmd_versiondb", + ) + self.dirpaths.outdir.Mkdir(exist_ok=True) + self.dbpath = testdir / "test_input" / "subcmd_versiondb" / "pyanidb" + self.exes = ThirdPartyExes("alembic") + + # Null logger instance + self.logger = logging.getLogger("TestVersiondbSubcommand logger") + self.logger.addHandler(logging.NullHandler()) + + # Command line namespaces + self.argsdict = { + "versiondb": Namespace( + indir=self.dirpaths.indir, + outdir=self.dirpaths.outdir, + dbpath=self.dbpath, + name="test_versiondb", + cmdline="alembic test suite", + alembic_exe=self.exes.alembic_exe, + workers=None, + jobprefix="alembicTest", + ) + } + + def test_versiondb(self): + """Test versiondb run.""" + print(self.argsdict["versiondb"]) + subcommands.subcmd_versiondb(self.argsdict["versiondb"]) diff --git a/tests/test_versiondb.py b/tests/test_versiondb.py new file mode 100644 index 00000000..d0e643e1 --- /dev/null +++ b/tests/test_versiondb.py @@ -0,0 +1,415 @@ +"""Test versiondb.py module. + +These tests are intended to be run from the repository root using: + +pytest -v +""" + +import os +import sys +import subprocess +import platform + +from argparse import Namespace +from pathlib import Path +from typing import List, NamedTuple, Tuple + +import pandas as pd +import pytest +import unittest +import shutil + +from pandas.util.testing import assert_frame_equal + +from pyani import versiondb, pyani_files, pyani_tools + +from pyani.pyani_orm import PyaniORMException, get_session, add_alembic +from tools import modify_namespace + +from pyani import pyani_config + + +# Create environment variables for alembic to access +def setenv(dir_versiondb_in, dbfile: Path): + abs_path = Path(dir_versiondb_in / dbfile).resolve() + os.environ["PYANI_DATABASE"] = str(abs_path) + + +# Test get_version() +# Test case 0: no executable location is specified +def test_get_version_nonetype(): + """Test behaviour when no location for the executable is given.""" + test_file_0 = None + + assert ( + versiondb.get_version(test_file_0) + == f"expected path to alembic executable; received {test_file_0}" + ) + + +# Test case 1: no such file exists +def test_get_version_random_string(): + """Test behaviour when the given 'file' is not one.""" + test_file_1 = "string" + + assert versiondb.get_version(test_file_1) == f"{test_file_1} is not found in $PATH" + + +# Test case 2: there is no executable +def test_get_version_no_exe(executable_missing, monkeypatch): + """Test behaviour when there is no file at the specified executable location.""" + test_file_2 = Path("/non/existent/alembic") + assert versiondb.get_version(test_file_2) == f"No alembic at {test_file_2}" + + +# Test case 3: there is a file, but it is not executable +def test_get_version_exe_not_executable(executable_not_executable, monkeypatch): + """Test behaviour when the file at the executable location is not executable.""" + test_file_3 = Path("/non/executable/alembic") + assert ( + versiondb.get_version(test_file_3) + == f"alembic exists at {test_file_3} but not executable" + ) + + +# Test case 4: there is an executable file, but the version can't be retrieved +def test_get_version_exe_no_version(executable_without_version, monkeypatch): + """Test behaviour when the version for the executable can not be retrieved.""" + test_file_4 = Path("/missing/version/alembic") + assert ( + versiondb.get_version(test_file_4) + == f"alembic exists at {test_file_4} but could not retrieve version" + ) + + +@pytest.fixture +def generic_versiondb_namespace(dir_versiondb_in): + """Generic namespace for the pyani versiondb subcommand.""" + return Namespace( + dbpath="pyanidb_upgrade", + upgrade="head", + downgrade=None, + dry_run=None, + direction="upgrade", + dbname=None, + alembic_config=None, + start=dir_versiondb_in / "base_pyanidb", + target=dir_versiondb_in / "head_pyanidb", + ) + + +@pytest.fixture +def downgrade_namespace(generic_versiondb_namespace, dir_versiondb_in): + """Namespace for pyani versiondb downgrade.""" + return modify_namespace( + generic_versiondb_namespace, + dbpath="pyanidb_downgrade", + upgrade=None, + downgrade="base", + direction="downgrade", + start=dir_versiondb_in / "head_pyanidb", + target=dir_versiondb_in / "base_pyanidb", + ) + + +@pytest.fixture +def altdb_namespace(generic_versiondb_namespace): + """Namespace for pyani versiondb -n altdb.""" + return modify_namespace( + generic_versiondb_namespace, + dbpath="pyanidb_altdb", + dbname="pyanidb_altdb", + alembic_config="alt_alembic_config.ini", + ) + + +@pytest.fixture +def dry_up_namespace(generic_versiondb_namespace): + """Namespace for pyani versiondb dry-run upgrade.""" + return modify_namespace( + generic_versiondb_namespace, + dbpath="pyanidb_dry_up", + upgrade=None, + dry_run="base:head", + ) + + +@pytest.fixture +def dry_down_namespace(generic_versiondb_namespace): + """Namespace for pyani versiondb dry-run downgrade.""" + return modify_namespace( + generic_versiondb_namespace, + dbpath="pyanidb_dry_down", + direction="downgrade", + downgrade=None, + dry_run="head:base", + ) + + +def expected_diffs(namespace): + """Expected (acceptable) differences between output and target databases.""" + return { + "upgrade": "", + "downgrade": "DROP TABLE alembic_version;\n", + "altdb": "", + }.get(namespace, None) + + +# Create database dump—a version that can be edited using sed +def dumpdb(abs_dbpath): + """Dump contents of database to a plain-text file.""" + + cmdline = [pyani_config.SQLITE_DEFAULT, f"{abs_dbpath}", ".dump"] + with open(f"{abs_dbpath}.sql", "w") as outfile: + subprocess.run( + cmdline, + shell=False, + stdout=outfile, + stderr=subprocess.PIPE, + ) + return f"{abs_dbpath}.sql" + + +def name_base_reqs(startdb_dump): + """Name unique constraint in comparisons table of old database schema.""" + old_constraint = ( + "UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch)," + ) + new_constraint = "CONSTRAINT base_reqs UNIQUE (query_id, subject_id, program, version, fragsize, maxmatch)," + + # Edit .dump file so that the unique constraint is named + # This is required in order to subsequently modify it + # In-place usage differs on macOs vs Linux + if platform.system() == "Darwin": + sed_cmd = [ + "sed", + "-i", + ".bak", + f"s/{old_constraint}/{new_constraint}/", + startdb_dump, + ] + else: + sed_cmd = [ + "sed", + "-i", + f"s/{old_constraint}/{new_constraint}/", + startdb_dump, + ] + subprocess.run( + sed_cmd, + shell=False, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + +def cleanup(abs_dbpath, test, dir_versiondb_out, args): + """Remove files created for test.""" + + dir_versiondb_out.mkdir(exist_ok=True) + Path(f"{dir_versiondb_out}/{test}").mkdir(exist_ok=True) + + # The files must be moved, but shutil.move() does not work + # if the file already exists, so this is a two-step process + # Copy files to new location + shutil.copy(abs_dbpath, dir_versiondb_out / test) + shutil.copy(f"{args.start}.sql", dir_versiondb_out / test) + + # Remove old files + os.remove(abs_dbpath) + os.remove(f"{args.start}.sql") + + # This file is not generated in the downgrade test + try: + shutil.copy(f"{args.start}.sql.bak", dir_versiondb_out / test) + os.remove(f"{args.start}.sql.bak") + except FileNotFoundError: + pass + + +# Test alembic command generation +def test_alembic_cmdline_generation( + generic_versiondb_namespace, + downgrade_namespace, + altdb_namespace, + dir_versiondb_in, +): + """Generate alembic command lines.""" + + alembic_cmds = [] + upgrade_args = generic_versiondb_namespace + alembic_cmds.append( + " ".join( + versiondb.construct_alembic_cmdline(upgrade_args.direction, upgrade_args) + ) + ) + + downgrade_args = downgrade_namespace + alembic_cmds.append( + " ".join( + versiondb.construct_alembic_cmdline( + downgrade_args.direction, downgrade_args + ) + ) + ) + + altdb_args = altdb_namespace + alembic_cmds.append( + " ".join(versiondb.construct_alembic_cmdline(altdb_args.direction, altdb_args)) + ) + + assert alembic_cmds == [ + "alembic upgrade head", + "alembic downgrade base", + f"alembic -n {altdb_args.dbname} -c {altdb_args.alembic_config} upgrade head", + ] + + +# Test upgrade +@pytest.mark.skip_if_exe_missing("sqldiff") +def test_versiondb_upgrade( + generic_versiondb_namespace, + dir_versiondb_in, + dir_versiondb_out, +): + """Test upgrade of database.""" + # Test setup + # Retrieve test namespace and + # Set environment variables and resolve absolute path of database + args = generic_versiondb_namespace + setenv(dir_versiondb_in, args.dbpath) + abs_dbpath = os.environ.get("PYANI_DATABASE") + + # Create and edit dump file to fix constraint problem + startdb_dump = dumpdb(args.start) + name_base_reqs(startdb_dump) + + # Run `sqlite3 -init + init_cmd = [pyani_config.SQLITE_DEFAULT, abs_dbpath] + subprocess.run( + init_cmd, + stdin=open(startdb_dump), + shell=False, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + # Run test migration + versiondb.migrate_database(args.direction, args, timestamp="testing") + + # Run diff + diff_cmd = [pyani_config.SQLDIFF_DEFAULT, "--schema", abs_dbpath, args.target] + result = subprocess.run( + diff_cmd, + shell=False, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + expected_diff = "" + + sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") + sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") + + # Move files + cleanup(abs_dbpath, "upgrade", dir_versiondb_out, args) + + assert result.stdout.decode() == expected_diff + + +@pytest.mark.skip_if_exe_missing("sqldiff") +def test_versiondb_downgrade(downgrade_namespace, dir_versiondb_in, dir_versiondb_out): + """Test downgrade of database.""" + # Test setup + # Retrieve test namespace and + # Set environment variables and resolve absolute path of database + args = downgrade_namespace + setenv(dir_versiondb_in, args.dbpath) + abs_dbpath = os.environ.get("PYANI_DATABASE") + + # Create dump file + startdb_dump = dumpdb(args.start) + + # Run `sqlite3 -init + init_cmd = [pyani_config.SQLITE_DEFAULT, abs_dbpath] + subprocess.run( + init_cmd, + stdin=open(startdb_dump), + shell=False, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + # Run test migration + versiondb.migrate_database(args.direction, args, timestamp="testing") + + # Run diff + diff_cmd = [pyani_config.SQLDIFF_DEFAULT, "--schema", abs_dbpath, args.target] + result = subprocess.run( + diff_cmd, + shell=False, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + expected_diff = "DROP TABLE alembic_version;\n" + + sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") + sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") + + # Move output files + cleanup(abs_dbpath, "downgrade", dir_versiondb_out, args) + + assert result.stdout.decode() == expected_diff + + +# Test alternate dbname +@pytest.mark.skip_if_exe_missing("sqldiff") +def test_versiondb_altdb(altdb_namespace, dir_versiondb_in, dir_versiondb_out): + """Test upgrade of database using an alternate database name and config file, such as in a multidb situation.""" + # Test setup + # Retrieve test namespace and + # Set environment variables and resolve absolute path of database + args = altdb_namespace + setenv(dir_versiondb_in, args.dbpath) + abs_dbpath = os.environ.get("PYANI_DATABASE") + + # Create dump file + startdb_dump = dumpdb(args.start) + name_base_reqs(startdb_dump) + + # assert False + # Run `sqlite3 -init + init_cmd = [pyani_config.SQLITE_DEFAULT, abs_dbpath] + subprocess.run( + init_cmd, + stdin=open(startdb_dump), + shell=False, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + # Run test migration + versiondb.migrate_database(args.direction, args, timestamp="testing") + + # Run diff + diff_cmd = [pyani_config.SQLDIFF_DEFAULT, "--schema", abs_dbpath, args.target] + result = subprocess.run( + diff_cmd, + shell=False, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + expected_diff = "" + + sys.stdout.write(f"Expected_diff: {expected_diff}\n\n") + sys.stdout.write(f"Actual diff: {result.stdout.decode()}\n\n") + + # Move files + cleanup(abs_dbpath, "altdb", dir_versiondb_out, args) + + assert result.stdout.decode() == expected_diff + + +# Dry-run tests still to be done