diff --git a/docs/restapi.rst b/docs/restapi.rst index ef5a2012..e0960153 100644 --- a/docs/restapi.rst +++ b/docs/restapi.rst @@ -1,15 +1,15 @@ REST API ======== -GET /workflows --------------- +GET /api/workflows +------------------ -.. autofunction:: reana_workflow_controller.app.get_workflows +.. autofunction:: reana_workflow_controller.api.get_workflows -POST /yadage ------------- +POST /api/yadage +---------------- -.. autofunction:: reana_workflow_controller.app.yadage_endpoint +.. autofunction:: reana_workflow_controller.api.yadage_endpoint diff --git a/reana_workflow_controller/api.py b/reana_workflow_controller/api.py new file mode 100644 index 00000000..847fae4f --- /dev/null +++ b/reana_workflow_controller/api.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2017 CERN. +# +# REANA is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# REANA is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# REANA; if not, write to the Free Software Foundation, Inc., 59 Temple Place, +# Suite 330, Boston, MA 02111-1307, USA. +# +# In applying this license, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +"""REANA Workflow Controller Rest API.""" + +import os +import traceback + +from flask import Blueprint, abort, jsonify, redirect, request + +from .app import db +from .fsdb import get_all_workflows +from .models import Tenant +from .tasks import run_yadage_workflow + +experiment_to_queue = { + 'alice': 'alice-queue', + 'atlas': 'atlas-queue', + 'lhcb': 'lhcb-queue', + 'cms': 'cms-queue', + 'recast': 'recast-queue' +} + +api = Blueprint('api', __name__) + + +@api.before_request +def before_request(): + """Retrieve organization from request.""" + org = request.args.get('organization') + if org: + db.choose_organization(org) + + +@api.route('/workflows', methods=['GET']) +def get_workflows(): + """Get all workflows. + + .. http:get:: /api/workflows + + Returns a JSON list with all the workflows. + + **Request**: + + .. sourcecode:: http + + GET /api/workflows HTTP/1.1 + Content-Type: apilication/json + Host: localhost:5000 + + :reqheader Content-Type: apilication/json + :query organization: organization name. It finds workflows + inside a given organization. + :query tenant: tenant uuid. It finds workflows inside a given + organization owned by tenant. + + **Responses**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Length: 22 + Content-Type: apilication/json + + { + "workflows": [ + { + "id": "256b25f4-4cfb-4684-b7a8-73872ef455a1", + "organization": "default_org", + "status": "running", + "tenant": "default_tenant" + }, + { + "id": "3c9b117c-d40a-49e3-a6de-5f89fcada5a3", + "organization": "default_org", + "status": "finished", + "tenant": "default_tenant" + }, + { + "id": "72e3ee4f-9cd3-4dc7-906c-24511d9f5ee3", + "organization": "default_org", + "status": "waiting", + "tenant": "default_tenant" + }, + { + "id": "c4c0a1a6-beef-46c7-be04-bf4b3beca5a1", + "organization": "default_org", + "status": "waiting", + "tenant": "default_tenant" + } + ] + } + + :resheader Content-Type: apilication/json + :statuscode 200: no error - the list has been returned. + + .. sourcecode:: http + + HTTP/1.1 500 Internal Error + Content-Length: 22 + Content-Type: apilication/json + + { + "msg": "Either organization or tenant doesn't exist." + } + + :resheader Content-Type: apilication/json + :statuscode 500: error - the list couldn't be returned. + """ + org = request.args.get('organization', 'default') + tenant = request.args['tenant'] + try: + if Tenant.query.filter(Tenant.id_ == tenant).count() < 1: + return jsonify({'msg': 'Tenant {} does not exist'.format(tenant)}) + + return jsonify({"workflows": get_all_workflows(org, tenant)}), 200 + except Exception as e: + return jsonify({"msg": str(e)}), 500 + + +@api.route('/yadage', methods=['POST']) +def yadage_endpoint(): + """Create a new job. + + .. http:post:: /api/yadage + + This resource is expecting JSON data with all the necessary + information to run a yadage workflow. + + **Request**: + + .. sourcecode:: http + + POST /api/yadage HTTP/1.1 + Content-Type: apilication/json + Host: localhost:5000 + + { + "experiment": "atlas", + "toplevel": "from-github/testing/scriptflow", + "workflow": "workflow.yml", + "nparallel": "100", + "preset_pars": {} + } + + :reqheader Content-Type: apilication/json + :json body: JSON with the information of the yadage workflow. + + **Responses**: + + .. sourcecode:: http + + HTTP/1.0 200 OK + Content-Length: 80 + Content-Type: apilication/json + + { + "msg", "Workflow successfully launched", + "workflow_id": "cdcf48b1-c2f3-4693-8230-b066e088c6ac" + } + + :resheader Content-Type: apilication/json + :statuscode 200: no error - the workflow was created + :statuscode 400: invalid request - problably a malformed JSON + """ + if request.method == 'POST': + try: + if request.json: + queue = experiment_to_queue[request.json['experiment']] + resultobject = run_yadage_workflow.apily_async( + args=[request.json], + queue='yadage-{}'.format(queue) + ) + if 'redirect' in request.args: + return redirect('{}/{}'.format( + os.environ['YADAGE_MONITOR_URL']), + resultobject.id) + return jsonify({'msg': 'Workflow successfully launched', + 'workflow_id': resultobject.id}) + + except (KeyError, ValueError): + traceback.print_exc() + abort(400) diff --git a/reana_workflow_controller/app.py b/reana_workflow_controller/app.py index fccf6988..86516b7a 100644 --- a/reana_workflow_controller/app.py +++ b/reana_workflow_controller/app.py @@ -24,195 +24,28 @@ from __future__ import absolute_import -import os -import traceback +from flask import Flask -from flask import Flask, abort, jsonify, redirect, request - -from .fsdb import get_all_workflows from .multiorganization import MultiOrganizationSQLAlchemy -from .tasks import run_yadage_workflow - -app = Flask(__name__) -app.config.from_object('reana_workflow_controller.config') -app.secret_key = "super secret key" # Initialize DB -db = MultiOrganizationSQLAlchemy(app) - -# Import models so that they are registered with SQLAlchemy -from .models import Tenant # isort:skip # noqa - -db.initialize_dbs() - -experiment_to_queue = { - 'alice': 'alice-queue', - 'atlas': 'atlas-queue', - 'lhcb': 'lhcb-queue', - 'cms': 'cms-queue', - 'recast': 'recast-queue' -} - - -@app.before_request -def before_request(): - """Retrieve organization from request.""" - org = request.args.get('organization') - if org: - db.choose_organization(org) - - -@app.route('/workflows', methods=['GET']) -def get_workflows(): - """Get all workflows. - - .. http:get:: /workflows - - Returns a JSON list with all the workflows. - - **Request**: - - .. sourcecode:: http - - GET /workflows HTTP/1.1 - Content-Type: application/json - Host: localhost:5000 - - :reqheader Content-Type: application/json - :query organization: organization name. It finds workflows - inside a given organization. - :query tenant: tenant uuid. It finds workflows inside a given - organization owned by tenant. - - **Responses**: - - .. sourcecode:: http - - HTTP/1.1 200 OK - Content-Length: 22 - Content-Type: application/json - - { - "workflows": [ - { - "id": "256b25f4-4cfb-4684-b7a8-73872ef455a1", - "organization": "default_org", - "status": "running", - "tenant": "default_tenant" - }, - { - "id": "3c9b117c-d40a-49e3-a6de-5f89fcada5a3", - "organization": "default_org", - "status": "finished", - "tenant": "default_tenant" - }, - { - "id": "72e3ee4f-9cd3-4dc7-906c-24511d9f5ee3", - "organization": "default_org", - "status": "waiting", - "tenant": "default_tenant" - }, - { - "id": "c4c0a1a6-beef-46c7-be04-bf4b3beca5a1", - "organization": "default_org", - "status": "waiting", - "tenant": "default_tenant" - } - ] - } - - :resheader Content-Type: application/json - :statuscode 200: no error - the list has been returned. - - .. sourcecode:: http - - HTTP/1.1 500 Internal Error - Content-Length: 22 - Content-Type: application/json - - { - "msg": "Either organization or tenant doesn't exist." - } - - :resheader Content-Type: application/json - :statuscode 500: error - the list couldn't be returned. - """ - org = request.args.get('organization', 'default') - tenant = request.args['tenant'] - try: - if Tenant.query.filter(Tenant.id_ == tenant).count() < 1: - return jsonify({'msg': 'Tenant {} does not exist'.format(tenant)}) - - return jsonify({"workflows": get_all_workflows(org, tenant)}), 200 - except Exception as e: - return jsonify({"msg": str(e)}), 500 - - -@app.route('/yadage', methods=['POST']) -def yadage_endpoint(): - """Create a new job. - - .. http:post:: /yadage - - This resource is expecting JSON data with all the necessary - information to run a yadage workflow. - - **Request**: - - .. sourcecode:: http - - POST /yadage HTTP/1.1 - Content-Type: application/json - Host: localhost:5000 - - { - "experiment": "atlas", - "toplevel": "from-github/testing/scriptflow", - "workflow": "workflow.yml", - "nparallel": "100", - "preset_pars": {} - } - - :reqheader Content-Type: application/json - :json body: JSON with the information of the yadage workflow. - - **Responses**: - - .. sourcecode:: http - - HTTP/1.0 200 OK - Content-Length: 80 - Content-Type: application/json +db = MultiOrganizationSQLAlchemy() - { - "msg", "Workflow successfully launched", - "workflow_id": "cdcf48b1-c2f3-4693-8230-b066e088c6ac" - } +from .models import Tenant # isort:skip # noqa - :resheader Content-Type: application/json - :statuscode 200: no error - the workflow was created - :statuscode 400: invalid request - problably a malformed JSON - """ - if request.method == 'POST': - try: - if request.json: - queue = experiment_to_queue[request.json['experiment']] - resultobject = run_yadage_workflow.apply_async( - args=[request.json], - queue='yadage-{}'.format(queue) - ) - if 'redirect' in request.args: - return redirect('{}/{}'.format( - os.environ['YADAGE_MONITOR_URL']), - resultobject.id) - return jsonify({'msg': 'Workflow successfully launched', - 'workflow_id': resultobject.id}) - except (KeyError, ValueError): - traceback.print_exc() - abort(400) +def create_app(): + """REANA Workflow Controller application factory.""" + app = Flask(__name__) + app.config.from_object('reana_workflow_controller.config') + app.secret_key = "super secret key" + # Initialize flask extensions + db.init_app(app) + # Register API routes + from .api import api as api_blueprint # noqa + app.register_blueprint(api_blueprint, url_prefix='/api') + with app.app_context(): + db.initialize_dbs() -if __name__ == '__main__': - app.run(debug=True, port=5000, - host='0.0.0.0') + return app diff --git a/reana_workflow_controller/config.py b/reana_workflow_controller/config.py index 2fc727a2..4d6eca52 100644 --- a/reana_workflow_controller/config.py +++ b/reana_workflow_controller/config.py @@ -27,20 +27,13 @@ SHARED_VOLUME_PATH = os.getenv('SHARED_VOLUME_PATH', '/reana') """Path to the mounted REANA shared volume.""" -SQLALCHEMY_DATABASE_URI = 'sqlite:////reana/default/default.db' +SQLALCHEMY_DATABASE_URI = 'sqlite:///{path}'.format( + path=os.path.join(SHARED_VOLUME_PATH, 'default/reana.db')) """SQLAlchemy database location""" -SQLALCHEMY_BINDS = { - 'alice': 'sqlite:///{path}'.format( - path=os.path.join(SHARED_VOLUME_PATH, 'alice/alice.db')), - 'atlas': 'sqlite:///{path}'.format( - path=os.path.join(SHARED_VOLUME_PATH, 'atlas/atlas.db')), - 'cms': 'sqlite:///{path}'.format( - path=os.path.join(SHARED_VOLUME_PATH, 'cms/cms.db')), - 'lhcb': 'sqlite:///{path}'.format( - path=os.path.join(SHARED_VOLUME_PATH, 'lhcb/lhcb.db')), -} -"""Organization databases""" +ORGANIZATIONS = os.getenv('ORGANIZATIONS').split(',') \ + if os.getenv('ORGANIZATIONS') else [] +"""Organizations.""" SQLALCHEMY_TRACK_MODIFICATIONS = False """Track modifications flag.""" diff --git a/reana_workflow_controller/instance.py b/reana_workflow_controller/instance.py new file mode 100644 index 00000000..da017d1f --- /dev/null +++ b/reana_workflow_controller/instance.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2017 CERN. +# +# REANA is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# REANA is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# REANA; if not, write to the Free Software Foundation, Inc., 59 Temple Place, +# Suite 330, Boston, MA 02111-1307, USA. +# +# In applying this license, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +"""REANA Workflow Controller Instance.""" + +from .app import create_app + +app = create_app() diff --git a/reana_workflow_controller/multiorganization.py b/reana_workflow_controller/multiorganization.py index 9b371f06..aea4bf83 100644 --- a/reana_workflow_controller/multiorganization.py +++ b/reana_workflow_controller/multiorganization.py @@ -24,19 +24,29 @@ from __future__ import absolute_import -from flask import g +from flask import current_app, g from flask_sqlalchemy import SQLAlchemy class MultiOrganizationSQLAlchemy(SQLAlchemy): """Multiorganization support for SQLAlchemy.""" + def _initialize_binds(self): + """Initialize binds from configuration if necessary.""" + current_app.config['SQLALCHEMY_BINDS'] = {} + for org in current_app.config.get('ORGANIZATIONS'): + current_app.config['SQLALCHEMY_BINDS'][org] = current_app\ + .config['SQLALCHEMY_DATABASE_URI'].replace( + 'default/reana.db', + '{organization}/reana.db'.format(organization=org)) + def initialize_dbs(self): """Initialize all organizations dbs.""" - with self.app.app_context(): + with current_app.app_context(): # Default organization DB self.create_all() - for bind in self.app.config['SQLALCHEMY_BINDS'].keys(): + self._initialize_binds() + for bind in current_app.config.get('SQLALCHEMY_BINDS').keys(): self.choose_organization(bind) self.create_all() diff --git a/tests/conftest.py b/tests/conftest.py index 79996f91..c5ea348b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,13 +24,13 @@ from __future__ import absolute_import, print_function +import os import shutil import tempfile -from os.path import dirname, join import pytest -from reana_workflow_controller.app import app as reana_workflow_controller_app +from reana_workflow_controller.app import create_app from reana_workflow_controller.models import Tenant from reana_workflow_controller.multiorganization import \ MultiOrganizationSQLAlchemy @@ -38,29 +38,31 @@ @pytest.fixture def tmp_fsdb_path(request): - """Fixture data for XrootDPyFS.""" + """Fixture temporary file system database.""" path = tempfile.mkdtemp() - shutil.copytree(join(dirname(__file__), "data"), join(path, "reana")) + shutil.copytree(os.path.join(os.path.dirname(__file__), "data"), + os.path.join(path, "reana")) def cleanup(): shutil.rmtree(path) request.addfinalizer(cleanup) - return join(path, "reana") + return os.path.join(path, "reana") -@pytest.fixture() +@pytest.yield_fixture() def base_app(tmp_fsdb_path): """Flask application fixture.""" - app_ = reana_workflow_controller_app + os.environ['SHARED_VOLUME_PATH'] = tmp_fsdb_path + app_ = create_app() app_.config.from_object('reana_workflow_controller.config') - app_.config['SHARED_VOLUME_PATH'] = tmp_fsdb_path app_.config.update( SERVER_NAME='localhost:5000', SECRET_KEY='SECRET_KEY', TESTING=True, ) - return app_ + yield app_ + del os.environ['SHARED_VOLUME_PATH'] @pytest.yield_fixture() diff --git a/tests/test_views.py b/tests/test_views.py index a3bf2c78..ec70c45c 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -31,7 +31,7 @@ def test_get_workflows(app, default_tenant): """Test listing all workflows.""" with app.test_client() as client: - res = client.get(url_for('get_workflows'), + res = client.get(url_for('api.get_workflows'), query_string={"tenant": default_tenant.id_}) assert res.status_code == 200