diff --git a/.travis.yml b/.travis.yml index a24588f..cab76c5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,9 @@ language: python cache: - pip +services: + - docker + matrix: fast_finish: true include: diff --git a/MANIFEST.in b/MANIFEST.in index 6bfd147..4957d62 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -11,6 +11,9 @@ include *.sh include *.yaml include pytest.ini recursive-include reana_db *.py +recursive-include reana_db *.mako +recursive-include reana_db .gitkeep +recursive-include reana_db *.ini recursive-include docs *.py recursive-include docs *.png recursive-include docs *.rst diff --git a/reana_db/alembic.ini b/reana_db/alembic.ini new file mode 100644 index 0000000..c5130ba --- /dev/null +++ b/reana_db/alembic.ini @@ -0,0 +1,58 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# version location specification; this defaults +# to reana_db/alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +version_locations = %(here)s/alembic/versions + +file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +hooks=black +black.type=console_scripts +black.entrypoint=black + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/reana_db/alembic/env.py b/reana_db/alembic/env.py new file mode 100644 index 0000000..b618917 --- /dev/null +++ b/reana_db/alembic/env.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2020 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""REANA DB alembic's environment context.""" + +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +from reana_db.config import SQLALCHEMY_DATABASE_URI +from reana_db.models import Base + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def _include_object(object_, name, *args): + # We ignore non-reana tables in migrations + schema = object_.schema if hasattr(object_, "schema") else object_.table.schema + if name == "alembic_version" or schema != "__reana": + return False + return True + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + context.configure( + url=SQLALCHEMY_DATABASE_URI, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + version_table_schema="__reana", + include_schemas=True, + include_object=_include_object, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + conf = { + "script_location": "reana_db/alembic", + "sqlalchemy.url": SQLALCHEMY_DATABASE_URI, + } + connectable = engine_from_config( + conf, prefix="sqlalchemy.", poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + version_table_schema="__reana", + include_schemas=True, + include_object=_include_object, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/reana_db/alembic/script.py.mako b/reana_db/alembic/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/reana_db/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/reana_db/alembic/versions/.gitkeep b/reana_db/alembic/versions/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/reana_db/cli.py b/reana_db/cli.py new file mode 100644 index 0000000..babbde3 --- /dev/null +++ b/reana_db/cli.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2020 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +"""REANA DB command line.""" + +import os + +from alembic import command +from alembic import config as alembic_config +import click + +from reana_db.database import init_db + + +@click.group() +def cli(): + """REANA database commands.""" + + +@cli.command() +def init(): + """Show REANA database migration recipes history.""" + init_db() + click.secho("Database initialised.", fg="green") + + +@cli.group("alembic") +@click.pass_context +def alembic_group(ctx): + """REANA database migration commands. + + Note that this command is just a light wrapper around alembic. + """ + reana_alembic_ini = os.path.join(os.path.dirname(__file__), "alembic.ini") + config = alembic_config.Config(reana_alembic_ini) + ctx.obj = config + + +@alembic_group.command() +@click.option( + "-m", "--message", default=None, help="Message string to use with 'revision'" +) +@click.option( + "--autogenerate/--no-autogenerate", + default=True, + help=( + "Populate revision script with candidate " + "migration operations, based on comparison of " + "database to model" + ), +) +@click.option( + "--sql", + default=False, + help=( + "Don't emit SQL to database - dump to standard output/file instead. See alembic docs on offline mode." + ), +) +@click.option( + "--head", + default="head", + help=("Specify head revision or @head to base new revision on."), +) +@click.option( + "--splice", + is_flag=True, + help=("Allow a non-head revision as the 'head' to splice onto."), +) +@click.option( + "--branch-label", + default=None, + help=("Specify a branch label to apply to the new revision"), +) +@click.option( + "--version-path", + default=None, + help=("Specify specific path from config for version file"), +) +@click.option( + "--rev-id", + default=None, + help=("Specify a hardcoded revision id instead of generating one"), +) +@click.option( + "--depends-on", + default=None, + help=( + "Specify one or more revision identifiers which this revision should depend on." + ), +) +@click.pass_obj +def revision( + config, + message, + autogenerate, + sql, + head, + splice, + branch_label, + version_path, + rev_id, + depends_on, +): + """Create a REANA database alembic revision.""" + command.revision( + config, + message=message, + autogenerate=autogenerate, + sql=sql, + head=head, + splice=splice, + branch_label=branch_label, + version_path=version_path, + rev_id=rev_id, + depends_on=depends_on, + ) + + +@alembic_group.command() +@click.argument("revision", default="head") +@click.option( + "--sql", + is_flag=True, + help=( + "Don't emit SQL to database - dump to standard output/file instead. See alembic docs on offline mode." + ), +) +@click.option( + "--tag", + default=None, + help=("Arbitrary 'tag' name - can be used by custom env.py scripts."), +) +@click.pass_obj +def upgrade(config, revision, sql, tag): + """Upgrade REANA database.""" + command.upgrade(config, revision, sql=sql, tag=tag) + + +@alembic_group.command() +@click.argument("revision", default="head") +@click.option( + "--sql", + is_flag=True, + help=( + "Don't emit SQL to database - dump to standard output/file instead. See alembic docs on offline mode." + ), +) +@click.option( + "--tag", + default=None, + help=("Arbitrary 'tag' name - can be used by custom env.py scripts."), +) +@click.pass_obj +def downgrade(config, revision, sql, tag): + """Downgrade REANA database.""" + command.downgrade(config, revision, sql=sql, tag=tag) + + +@alembic_group.command() +@click.option( + "-v", "--verbose", is_flag=True, help=("Use more verbose output."), +) +@click.pass_obj +def current(config, verbose): + """Show current database state.""" + command.current(config, verbose=verbose) + + +@alembic_group.command() +@click.option( + "-r", + "--rev-range", + default=None, + help=("Specify a revision range; format is [start]:[end]."), +) +@click.option( + "-v", "--verbose", is_flag=True, help=("Use more verbose output."), +) +@click.option( + "-i", "--indicate-current", is_flag=True, help=("Indicate the current revision."), +) +@click.pass_obj +def history(config, rev_range, verbose, indicate_current): + """Show REANA database migration recipes history.""" + command.history( + config, rev_range=rev_range, verbose=verbose, indicate_current=indicate_current + ) diff --git a/reana_db/database.py b/reana_db/database.py index 03f79a1..01a131b 100644 --- a/reana_db/database.py +++ b/reana_db/database.py @@ -10,8 +10,9 @@ from __future__ import absolute_import -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.schema import CreateSchema from sqlalchemy_utils import create_database, database_exists from .config import SQLALCHEMY_DATABASE_URI @@ -27,6 +28,7 @@ def init_db(): """Initialize the DB.""" import reana_db.models + event.listen(Base.metadata, "before_create", CreateSchema("__reana")) if not database_exists(engine.url): create_database(engine.url) Base.metadata.create_all(bind=engine) diff --git a/reana_db/models.py b/reana_db/models.py index 0f3bcbb..220fb17 100644 --- a/reana_db/models.py +++ b/reana_db/models.py @@ -46,6 +46,7 @@ class User(Base, Timestamp): """User table.""" __tablename__ = "user_" + __table_args__ = {"schema": "__reana"} id_ = Column(UUIDType, primary_key=True, unique=True, default=generate_uuid) email = Column(String(length=255), unique=True, primary_key=True) @@ -179,14 +180,15 @@ class UserToken(Base, Timestamp): """User tokens table.""" __tablename__ = "user_token" + __table_args__ = {"schema": "__reana"} - id_ = Column(UUIDType, primary_key=True, unique=True, default=generate_uuid) + id_ = Column(UUIDType, primary_key=True, default=generate_uuid) token = Column( EncryptedType(String(length=255), DB_SECRET_KEY, AesEngine, "pkcs5"), unique=True, ) status = Column(Enum(UserTokenStatus)) - user_id = Column(UUIDType, ForeignKey("user_.id_"), nullable=False) + user_id = Column(UUIDType, ForeignKey("__reana.user_.id_"), nullable=False) type_ = Column(Enum(UserTokenType), nullable=False) @@ -241,7 +243,7 @@ class Workflow(Base, Timestamp): id_ = Column(UUIDType, primary_key=True) name = Column(String(255)) status = Column(Enum(WorkflowStatus), default=WorkflowStatus.created) - owner_id = Column(UUIDType, ForeignKey("user_.id_")) + owner_id = Column(UUIDType, ForeignKey("__reana.user_.id_")) reana_specification = Column(JSONType) input_parameters = Column(JSONType) operational_options = Column(JSONType) @@ -271,6 +273,7 @@ class Workflow(Base, Timestamp): UniqueConstraint( "name", "owner_id", "run_number", name="_user_workflow_run_uc" ), + {"schema": "__reana"}, ) def __init__( @@ -420,8 +423,9 @@ class Job(Base, Timestamp): """Job table.""" __tablename__ = "job" + __table_args__ = {"schema": "__reana"} - id_ = Column(UUIDType, unique=True, primary_key=True, default=generate_uuid) + id_ = Column(UUIDType, primary_key=True, default=generate_uuid) backend_job_id = Column(String(256)) workflow_uuid = Column(UUIDType) status = Column(Enum(JobStatus), default=JobStatus.created) @@ -441,9 +445,10 @@ class JobCache(Base, Timestamp): """Job Cache table.""" __tablename__ = "job_cache" + __table_args__ = {"schema": "__reana"} - id_ = Column(UUIDType, unique=True, primary_key=True, default=generate_uuid) - job_id = Column(UUIDType, ForeignKey("job.id_"), primary_key=True) + id_ = Column(UUIDType, primary_key=True, default=generate_uuid) + job_id = Column(UUIDType, ForeignKey("__reana.job.id_"), primary_key=True) parameters = Column(String(1024)) result_path = Column(String(1024)) workspace_hash = Column(String(1024)) @@ -462,9 +467,10 @@ class AuditLog(Base, Timestamp): """Audit log table.""" __tablename__ = "audit_log" + __table_args__ = {"schema": "__reana"} - id_ = Column(UUIDType, unique=True, primary_key=True, default=generate_uuid) - user_id = Column(UUIDType, ForeignKey("user_.id_"), nullable=False) + id_ = Column(UUIDType, primary_key=True, default=generate_uuid) + user_id = Column(UUIDType, ForeignKey("__reana.user_.id_"), nullable=False) action = Column(Enum(AuditLogAction), nullable=False) details = Column(JSONType) diff --git a/run-tests.sh b/run-tests.sh index 3884688..526ad6b 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -1,14 +1,69 @@ -#!/bin/sh +#!/bin/bash # # This file is part of REANA. -# Copyright (C) 2018 CERN. +# Copyright (C) 2018, 2020 CERN. # # REANA is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. -pydocstyle reana_db && \ -black --check . && \ -check-manifest --ignore ".travis-*" && \ -sphinx-build -qnNW docs docs/_build/html && \ -REANA_SQLALCHEMY_DATABASE_URI=sqlite:// python setup.py test && \ +# Quit on errors +set -o errexit + +# Quit on unbound symbols +set -o nounset + +export REANA_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://postgres:mysecretpassword@localhost/postgres + +# Verify that db container is running before continuing +_check_ready() { + RETRIES=40 + while ! $2 + do + echo "==> [INFO] Waiting for $1, $((RETRIES--)) remaining attempts..." + sleep 2 + if [ $RETRIES -eq 0 ] + then + echo "==> [ERROR] Couldn't reach $1" + exit 1 + fi + done +} + +_db_check() { + docker exec --user postgres reana-postgres bash -c "pg_isready" &>/dev/null; +} + +clean_old_db_container() { + OLD="$(docker ps --all --quiet --filter=name=reana-postgres)" + if [ -n "$OLD" ]; then + echo '==> [INFO] Cleaning old DB container...' + docker stop reana-postgres + fi +} + +start_db_container() { + echo '==> [INFO] Starting DB container...' + docker run --rm --name reana-postgres -p 5432:5432 -e POSTGRES_PASSWORD=mysecretpassword -d postgres + _check_ready "Postgres" _db_check +} + +stop_db_container() { + echo '==> [INFO] Stopping DB container...' + docker stop reana-postgres +} + +check_black() { + echo '==> [INFO] Checking Black compliance...' + black --check . +} + +pydocstyle reana_db +check_black +check-manifest --ignore ".travis-*" +sphinx-build -qnNW docs docs/_build/html +clean_old_db_container +start_db_container +python setup.py test +stop_db_container sphinx-build -qnNW -b doctest docs docs/_build/doctest +echo '==> [INFO] All tests passed! ✅' diff --git a/setup.py b/setup.py index d5b2519..af9520d 100755 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ ] install_requires = [ + "alembic>=1.4.2", "psycopg2-binary>=2.6.1", "SQLAlchemy>=1.2.7", 'sqlalchemy-utils>=0.35.0 ; python_version>="3"', @@ -64,6 +65,7 @@ author_email="info@reana.io", url="https://github.com/reanahub/reana-db", packages=["reana_db",], + entry_points={"console_scripts": ["reana-db=reana_db.cli:cli"],}, zip_safe=False, install_requires=install_requires, extras_require=extras_require, diff --git a/tests/conftest.py b/tests/conftest.py index 6d71240..76948c6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,12 +12,8 @@ from uuid import uuid4 import pytest -from mock import patch -from sqlalchemy import create_engine -from sqlalchemy.orm import scoped_session, sessionmaker -from sqlalchemy_utils import create_database, database_exists, drop_database -from reana_db.models import Base, User +from reana_db.models import User @pytest.fixture(scope="module") diff --git a/tests/test_models.py b/tests/test_models.py index 7caeec4..7e68f91 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -11,14 +11,10 @@ from uuid import uuid4 import pytest -import sqlalchemy -from mock import patch from reana_db.models import ( ALLOWED_WORKFLOW_STATUS_TRANSITIONS, - AuditLog, AuditLogAction, - User, UserTokenStatus, UserTokenType, Workflow, @@ -26,14 +22,14 @@ ) -def test_workflow_run_number_assignment(db, session): +def test_workflow_run_number_assignment(db, session, new_user): """Test workflow run number assignment.""" workflow_name = "workflow" - owner_id = str(uuid4()) + first_workflow = Workflow( id_=str(uuid4()), name=workflow_name, - owner_id=owner_id, + owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", @@ -44,7 +40,7 @@ def test_workflow_run_number_assignment(db, session): second_workflow = Workflow( id_=str(uuid4()), name=workflow_name, - owner_id=owner_id, + owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", @@ -55,7 +51,7 @@ def test_workflow_run_number_assignment(db, session): first_workflow_restart = Workflow( id_=str(uuid4()), name=workflow_name, - owner_id=owner_id, + owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", @@ -68,7 +64,7 @@ def test_workflow_run_number_assignment(db, session): first_workflow_second_restart = Workflow( id_=str(uuid4()), name=workflow_name, - owner_id=owner_id, + owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", @@ -106,15 +102,14 @@ def test_workflow_run_number_assignment(db, session): + [tuple + (True,) for tuple in ALLOWED_WORKFLOW_STATUS_TRANSITIONS], ) def test_workflow_can_transition_to( - db, session, from_status, to_status, can_transition + db, session, from_status, to_status, can_transition, new_user ): """Test workflow run number assignment.""" workflow_name = "test-workflow" - owner_id = str(uuid4()) workflow = Workflow( id_=str(uuid4()), name=workflow_name, - owner_id=owner_id, + owner_id=new_user.id_, reana_specification=[], type_="serial", logs="", @@ -148,7 +143,7 @@ def _audit_action(): ) assert audited_action.details == details else: - with pytest.raises(sqlalchemy.exc.IntegrityError): + with pytest.raises(Exception): _audit_action()