Feature/workflows (#583)

Adds dispatch workflows

src/dispatch/__init__.py

@@ -2,7 +2,6 @@
 import os.path
 from subprocess import check_output
 
-
 try:
     VERSION = __import__("pkg_resources").get_distribution("dispatch").version
 except Exception:
@@ -36,6 +35,7 @@
     from dispatch.term.models import Term  # noqa lgtm[py/unused-import]
     from dispatch.ticket.models import Ticket  # noqa lgtm[py/unused-import]
     from dispatch.plugin.models import Plugin  # noqa lgtm[py/unused-import]
+    from dispatch.workflow.models import Workflow  # noqa lgtm[py/unused-import]
 except Exception:
     pass
 

src/dispatch/alembic/versions/189759c2b5bf_.py

@@ -0,0 +1,28 @@
+"""Adding a run reason.
+
+Revision ID: 189759c2b5bf
+Revises: 99f3dec93615
+Create Date: 2020-10-01 14:53:31.791427
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "189759c2b5bf"
+down_revision = "99f3dec93615"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("workflow_instance", sa.Column("run_reason", sa.String(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("workflow_instance", "run_reason")
+    # ### end Alembic commands ###

src/dispatch/alembic/versions/466823f3046e_.py

@@ -0,0 +1,41 @@
+"""Adds artifact association table.
+
+Revision ID: 466823f3046e
+Revises: 7351fa734e2a
+Create Date: 2020-09-23 12:42:38.268434
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "466823f3046e"
+down_revision = "7351fa734e2a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "workflow_instance_artifacts",
+        sa.Column("document_id", sa.Integer(), nullable=False),
+        sa.Column("workflow_instance_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id"],
+            ["document.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["workflow_instance_id"],
+            ["workflow_instance.id"],
+        ),
+        sa.PrimaryKeyConstraint("document_id", "workflow_instance_id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("workflow_instance_artifacts")
+    # ### end Alembic commands ###

src/dispatch/alembic/versions/72c7fa0ce0a3_.py

@@ -0,0 +1,32 @@
+"""Adds workflow plugins.
+
+Revision ID: 72c7fa0ce0a3
+Revises: 466823f3046e
+Create Date: 2020-09-24 10:00:34.338853
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '72c7fa0ce0a3'
+down_revision = '466823f3046e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('workflow', sa.Column('plugin_id', sa.Integer(), nullable=True))
+    op.create_foreign_key(None, 'workflow', 'plugin', ['plugin_id'], ['id'])
+    op.drop_column('workflow', 'plugin_slug')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('workflow', sa.Column('plugin_slug', sa.VARCHAR(), autoincrement=False, nullable=True))
+    op.drop_constraint(None, 'workflow', type_='foreignkey')
+    op.drop_column('workflow', 'plugin_id')
+    # ### end Alembic commands ###

src/dispatch/alembic/versions/7351fa734e2a_.py

@@ -0,0 +1,77 @@
+"""Adds workflow table.
+
+Revision ID: 7351fa734e2a
+Revises: ecdb4e7566f2
+Create Date: 2020-09-23 11:42:36.889418
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlalchemy_utils
+
+
+# revision identifiers, used by Alembic.
+revision = "7351fa734e2a"
+down_revision = "ecdb4e7566f2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "workflow",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column("description", sa.String(), nullable=True),
+        sa.Column("is_active", sa.Boolean(), nullable=True),
+        sa.Column("parameters", sa.JSON(), nullable=True),
+        sa.Column("resource_id", sa.String(), nullable=True),
+        sa.Column("plugin_slug", sa.String(), nullable=True),
+        sa.Column("search_vector", sqlalchemy_utils.types.ts_vector.TSVectorType(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.Column("updated_at", sa.DateTime(), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "ix_workflow_search_vector",
+        "workflow",
+        ["search_vector"],
+        unique=False,
+        postgresql_using="gin",
+    )
+    op.create_table(
+        "workflow_instance",
+        sa.Column("resource_type", sa.String(), nullable=True),
+        sa.Column("resource_id", sa.String(), nullable=True),
+        sa.Column("weblink", sa.String(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("workflow_id", sa.Integer(), nullable=True),
+        sa.Column("creator_id", sa.Integer(), nullable=True),
+        sa.Column("status", sa.String(), nullable=True),
+        sa.Column("incident_id", sa.Integer(), nullable=True),
+        sa.Column("created_at", sa.DateTime(), nullable=True),
+        sa.Column("updated_at", sa.DateTime(), nullable=True),
+        sa.ForeignKeyConstraint(
+            ["creator_id"],
+            ["participant.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["incident_id"],
+            ["incident.id"],
+        ),
+        sa.ForeignKeyConstraint(
+            ["workflow_id"],
+            ["workflow.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("workflow_instance")
+    op.drop_index("ix_workflow_search_vector", table_name="workflow")
+    op.drop_table("workflow")
+    # ### end Alembic commands ###

src/dispatch/alembic/versions/995a59897e01_.py

@@ -0,0 +1,32 @@
+"""Allows plugins to be disable.
+
+Revision ID: 995a59897e01
+Revises: 72c7fa0ce0a3
+Create Date: 2020-09-24 10:08:07.729599
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "995a59897e01"
+down_revision = "72c7fa0ce0a3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("workflow", sa.Column("enabled", sa.Boolean(), nullable=True))
+    op.drop_column("workflow", "is_active")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "workflow", sa.Column("is_active", sa.BOOLEAN(), autoincrement=False, nullable=True)
+    )
+    op.drop_column("workflow", "enabled")
+    # ### end Alembic commands ###

src/dispatch/alembic/versions/99f3dec93615_.py

@@ -0,0 +1,66 @@
+"""Adds priorities, terms, types, and artifact relationships and tables.
+
+Revision ID: 99f3dec93615
+Revises: 9f927a8e4679
+Create Date: 2020-09-28 15:47:08.865362
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '99f3dec93615'
+down_revision = '9f927a8e4679'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('workflow_incident_priority',
+    sa.Column('incident_priority_id', sa.Integer(), nullable=False),
+    sa.Column('workflow_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['incident_priority_id'], ['incident_priority.id'], ),
+    sa.ForeignKeyConstraint(['workflow_id'], ['workflow.id'], ),
+    sa.PrimaryKeyConstraint('incident_priority_id', 'workflow_id')
+    )
+    op.create_table('workflow_incident_type',
+    sa.Column('incident_type_id', sa.Integer(), nullable=False),
+    sa.Column('workflow_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['incident_type_id'], ['incident_type.id'], ),
+    sa.ForeignKeyConstraint(['workflow_id'], ['workflow.id'], ),
+    sa.PrimaryKeyConstraint('incident_type_id', 'workflow_id')
+    )
+    op.create_table('workflow_term',
+    sa.Column('term_id', sa.Integer(), nullable=False),
+    sa.Column('workflow_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['term_id'], ['term.id'], ),
+    sa.ForeignKeyConstraint(['workflow_id'], ['workflow.id'], ),
+    sa.PrimaryKeyConstraint('term_id', 'workflow_id')
+    )
+    op.create_table('workflow_instance_artifact',
+    sa.Column('document_id', sa.Integer(), nullable=False),
+    sa.Column('workflow_instance_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['document_id'], ['document.id'], ),
+    sa.ForeignKeyConstraint(['workflow_instance_id'], ['workflow_instance.id'], ),
+    sa.PrimaryKeyConstraint('document_id', 'workflow_instance_id')
+    )
+    op.drop_table('workflow_instance_artifacts')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('workflow_instance_artifacts',
+    sa.Column('document_id', sa.INTEGER(), autoincrement=False, nullable=False),
+    sa.Column('workflow_instance_id', sa.INTEGER(), autoincrement=False, nullable=False),
+    sa.ForeignKeyConstraint(['document_id'], ['document.id'], name='workflow_instance_artifacts_document_id_fkey'),
+    sa.ForeignKeyConstraint(['workflow_instance_id'], ['workflow_instance.id'], name='workflow_instance_artifacts_workflow_instance_id_fkey'),
+    sa.PrimaryKeyConstraint('document_id', 'workflow_instance_id', name='workflow_instance_artifacts_pkey')
+    )
+    op.drop_table('workflow_instance_artifact')
+    op.drop_table('workflow_term')
+    op.drop_table('workflow_incident_type')
+    op.drop_table('workflow_incident_priority')
+    # ### end Alembic commands ###

src/dispatch/alembic/versions/9f927a8e4679_.py

@@ -0,0 +1,28 @@
+"""Adds instance parameters.
+
+Revision ID: 9f927a8e4679
+Revises: 995a59897e01
+Create Date: 2020-09-25 12:48:21.180604
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9f927a8e4679'
+down_revision = '995a59897e01'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('workflow_instance', sa.Column('parameters', sa.JSON(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('workflow_instance', 'parameters')
+    # ### end Alembic commands ###

src/dispatch/api.py

@@ -21,6 +21,7 @@
 from dispatch.task.views import router as task_router
 from dispatch.plugin.views import router as plugin_router
 from dispatch.auth.views import user_router, auth_router
+from dispatch.workflow.views import router as workflow_router
 
 from .config import DISPATCH_AUTHENTICATION_PROVIDER_SLUG
 
@@ -57,6 +58,7 @@
 authenticated_api_router.include_router(
     incident_priority_router, prefix="/incident_priorities", tags=["incident_priorities"]
 )
+authenticated_api_router.include_router(workflow_router, prefix="/workflows", tags=["workflows"])
 authenticated_api_router.include_router(plugin_router, prefix="/plugins", tags=["plugins"])
 
 doc_router = APIRouter()

src/dispatch/cli.py

@@ -125,6 +125,7 @@ def sync_triggers():
     sync_trigger(engine, "team_contact", "search_vector", ["name", "company", "notes"])
     sync_trigger(engine, "term", "search_vector", ["text"])
     sync_trigger(engine, "dispatch_user", "search_vector", ["email"])
+    sync_trigger(engine, "workflow", "search_vector", ["name", "description"])
 
 
 @dispatch_cli.group("database")
@@ -418,6 +419,7 @@ def dispatch_scheduler():
     from .tag.scheduled import sync_tags  # noqa
     from .task.scheduled import sync_tasks, create_task_reminders  # noqa
     from .term.scheduled import sync_terms  # noqa
+    from .workflow.scheduled import sync_workflows, sync_active_stable_workflows  # noqa
 
 
 @dispatch_scheduler.command("list")

src/dispatch/common/utils/cli.py

@@ -37,6 +37,8 @@ def install_plugins():
             logger.error(
                 "Something went wrong with creating plugin rows, is the database setup correctly?"
             )
+        except KeyError as e:
+            logger.warning(f"failed to load plugin {ep.name}. Reason: {e}")
         except Exception:
             logger.error(f"Failed to load plugin {ep.name}:{traceback.format_exc()}")
         else:

src/dispatch/document/service.py

@@ -107,6 +107,20 @@ def create(*, db_session, document_in: DocumentCreate) -> Document:
     return document
 
 
+def get_or_create(*, db_session, document_in) -> Document:
+    """Gets a document by it's resource_id or creates a new document."""
+    if hasattr(document_in, "resource_id"):
+        q = db_session.query(Document).filter(Document.resource_id == document_in.resource_id)
+    else:
+        q = db_session.query(Document).filter_by(**document_in.dict())
+
+    instance = q.first()
+    if instance:
+        return instance
+
+    return create(db_session=db_session, document_in=document_in)
+
+
 def update(*, db_session, document: Document, document_in: DocumentUpdate) -> Document:
     """Updates a document."""
     document_data = jsonable_encoder(document)