diff --git a/MANIFEST.in b/MANIFEST.in index e42fd44a..1e830c96 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -23,6 +23,7 @@ include COPYING include *.rst include *.sh include pytest.ini +include docs/openapi.json recursive-include reana_job_controller *.json recursive-include docs *.py recursive-include docs *.png diff --git a/docs/conf.py b/docs/conf.py index 30a1f98c..55ba8161 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,6 +51,7 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode', 'sphinxcontrib.httpdomain', + 'sphinxcontrib.openapi' ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index ab775d61..7dd576ce 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -10,3 +10,12 @@ Complex example --------------- FIXME + +CLI +--- + +The REANA Job Controller package offers the possibility to create its OpenAPI specification file using the flask CLI. + +.. code-block:: console + + $ flask openapi create output_file diff --git a/docs/restapi.rst b/docs/restapi.rst index 303704ee..e3ba5f4d 100644 --- a/docs/restapi.rst +++ b/docs/restapi.rst @@ -1,17 +1,4 @@ REST API ======== -GET /jobs ---------- - -.. autofunction:: reana_job_controller.app.get_jobs - -POST /jobs ----------- - -.. autofunction:: reana_job_controller.app.create_job - -GET /jobs/ ------------------- - -.. autofunction:: reana_job_controller.app.get_job +.. openapi:: openapi.json diff --git a/reana_job_controller/app.py b/reana_job_controller/app.py index 2563e0d6..bbffaea0 100644 --- a/reana_job_controller/app.py +++ b/reana_job_controller/app.py @@ -28,14 +28,18 @@ import uuid from flask import Flask, abort, jsonify, request - from reana_job_controller.k8s import (create_api_client, instantiate_job, watch_jobs, watch_pods) +from reana_job_controller.schemas import Job, JobRequest +from reana_job_controller.spec import build_openapi_spec app = Flask(__name__) app.secret_key = "mega secret key" JOB_DB = {} +job_request_schema = JobRequest() +job_schema = Job() + def filter_jobs(job_db): """Filter unsolicited job_db fields. @@ -55,212 +59,195 @@ def filter_jobs(job_db): @app.route('/jobs', methods=['GET']) -def get_jobs(): +def get_jobs(): # noqa """Get all jobs. - .. http:get:: /jobs - - Returns a JSON list with all the jobs. - - **Request**: - - .. sourcecode:: http - - GET /jobs HTTP/1.1 - Content-Type: application/json - Host: localhost:5000 - - :reqheader Content-Type: application/json - - **Responses**: - - .. sourcecode:: http - - HTTP/1.0 200 OK - Content-Length: 80 - Content-Type: application/json - - { - "jobs": { - "1612a779-f3fa-4344-8819-3d12fa9b9d90": { - "cmd": "sleep 1000", - "cvmfs_mounts": [ - "atlas-condb", - "atlas" - ], - "docker-img": "busybox", - "experiment": "atlas", - "job-id": "1612a779-f3fa-4344-8819-3d12fa9b9d90", - "max_restart_count": 3, - "restart_count": 0, - "status": "succeeded" - }, - "2e4bbc1d-db5e-4ee0-9701-6e2b1ba55c20": { - "cmd": "sleep 1000", - "cvmfs_mounts": [ - "atlas-condb", - "atlas" - ], - "docker-img": "busybox", - "experiment": "atlas", - "job-id": "2e4bbc1d-db5e-4ee0-9701-6e2b1ba55c20", - "max_restart_count": 3, - "restart_count": 0, - "status": "started" + --- + get: + description: Get all Jobs + produces: + - application/json + responses: + 200: + description: Job list. + schema: + type: array + items: + $ref: '#/definitions/Job' + examples: + application/json: + { + "jobs": { + "1612a779-f3fa-4344-8819-3d12fa9b9d90": { + "cmd": "sleep 1000", + "cvmfs_mounts": [ + "atlas-condb", + "atlas" + ], + "docker_img": "busybox", + "experiment": "atlas", + "job_id": "1612a779-f3fa-4344-8819-3d12fa9b9d90", + "max_restart_count": 3, + "restart_count": 0, + "status": "succeeded" + }, + "2e4bbc1d-db5e-4ee0-9701-6e2b1ba55c20": { + "cmd": "sleep 1000", + "cvmfs_mounts": [ + "atlas-condb", + "atlas" + ], + "docker_img": "busybox", + "experiment": "atlas", + "job_id": "2e4bbc1d-db5e-4ee0-9701-6e2b1ba55c20", + "max_restart_count": 3, + "restart_count": 0, + "status": "started" + } } } - } - - :resheader Content-Type: application/json - :statuscode 200: no error - the list has been returned. """ + # FIXME do Marshmallow validation after fixing the structure + # of job list. Now it has the ID as key, it should be a plain + # list of jobs so it can be validated with Marshmallow. + return jsonify({"jobs": filter_jobs(JOB_DB)}), 200 @app.route('/jobs', methods=['POST']) -def create_job(): +def create_job(): # noqa """Create a new job. - .. http:post:: /jobs - + --- + post: + summary: |- This resource is expecting JSON data with all the necessary information of a new job. + description: Create a new Job. + operationId: create_job + consumes: + - application/json + produces: + - application/json + parameters: + - name: job + in: body + description: Information needed to instantiate a Job + required: true + schema: + $ref: '#/definitions/JobRequest' + responses: + 201: + description: The Job has been created. + schema: + type: object + properties: + job_id: + type: string + examples: + application/json: + { + "job_id": "cdcf48b1-c2f3-4693-8230-b066e088c6ac" + } + 400: + description: Invalid request - probably malformed JSON + 500: + description: Internal error - probably the job could not be allocated + """ - **Request**: - - .. sourcecode:: http - - POST /jobs HTTP/1.1 - Content-Type: application/json - Host: localhost:5000 - - { - "docker-img": "busybox", - "cmd": "sleep 1000", - "cvmfs_mounts": ["atlas-condb", "atlas"], - "env-vars": {"DATA": "/data"}, - "experiment": "atlas" - } - - :reqheader Content-Type: application/json - :json body: JSON with the information of the job. - - **Responses**: - - .. sourcecode:: http - - HTTP/1.0 200 OK - Content-Length: 80 - Content-Type: application/json - - { - "job-id": "cdcf48b1-c2f3-4693-8230-b066e088c6ac" - } + json_data = request.get_json() + if not json_data: + return jsonify({'message': 'Empty request'}), 400 - :resheader Content-Type: application/json - :statuscode 201: no error - the job was created - :statuscode 400: invalid request - problably a malformed JSON - :statuscode 500: internal error - probably the job could not be - created - """ - if not request.json \ - or not ('experiment') in request.json\ - or not ('docker-img' in request.json): - print(request.json) - abort(400) - - cmd = request.json['cmd'] if 'cmd' in request.json else None - env_vars = (request.json['env-vars'] - if 'env-vars' in request.json else {}) - - if request.json.get('cvmfs_mounts'): - cvmfs_repos = request.json.get('cvmfs_mounts') - else: - cvmfs_repos = [] + # Validate and deserialize input + job_request, errors = job_request_schema.load(json_data) - job_id = str(uuid.uuid4()) + if errors: + return jsonify(errors), 400 - job_obj = instantiate_job(job_id, - request.json['docker-img'], - cmd, - cvmfs_repos, - env_vars, - request.json['experiment'], - shared_file_system=True) + job_obj = instantiate_job(job_request['job_id'], + job_request['docker_img'], + job_request['cmd'], + job_request['cvmfs_mounts'], + job_request['env_vars'], + job_request['experiment'], + job_request['shared_file_system']) if job_obj: - job = copy.deepcopy(request.json) - job['job-id'] = job_id + job = copy.deepcopy(job_request) job['status'] = 'started' job['restart_count'] = 0 job['max_restart_count'] = 3 job['obj'] = job_obj job['deleted'] = False - JOB_DB[job_id] = job - return jsonify({'job-id': job_id}), 201 + JOB_DB[job['job_id']] = job + return jsonify({'job_id': job['job_id']}), 201 else: return jsonify({'job': 'Could not be allocated'}), 500 @app.route('/jobs/', methods=['GET']) -def get_job(job_id): - """Get a job. - - FIXME --> probably this endpoint should be merged with `get_jobs()` - - .. http:get:: /jobs/ - - Returns a JSON list with all the jobs. - - **Request**: - - .. sourcecode:: http - - GET /jobs/cdcf48b1-c2f3-4693-8230-b066e088c6ac HTTP/1.1 - Content-Type: application/json - Host: localhost:5000 - - :reqheader Content-Type: application/json - - **Responses**: - - .. sourcecode:: http - - HTTP/1.0 200 OK - Content-Length: 80 - Content-Type: application/json - - { +def get_job(job_id): # noqa + """Get a Job. + + --- + get: + description: Get a Job by its id + produces: + - application/json + parameters: + - name: job_id + in: path + description: ID of the Job + required: true + type: string + responses: + 200: + description: The Job. + schema: + $ref: '#/definitions/Job' + examples: + application/json: "job": { "cmd": "sleep 1000", "cvmfs_mounts": [ "atlas-condb", "atlas" ], - "docker-img": "busybox", + "docker_img": "busybox", "experiment": "atlas", - "job-id": "cdcf48b1-c2f3-4693-8230-b066e088c6ac", + "job_id": "cdcf48b1-c2f3-4693-8230-b066e088c6ac", "max_restart_count": 3, "restart_count": 0, "status": "started" } - } - - :resheader Content-Type: application/json - :statuscode 200: no error - the list has been returned. - :statuscode 404: error - the specified job doesn't exist. + 404: + description: The Job does not exist. """ + if job_id in JOB_DB: job_copy = copy.deepcopy(JOB_DB[job_id]) del(job_copy['obj']) del(job_copy['deleted']) if job_copy.get('pod'): del(job_copy['pod']) + + # FIXME job_schema.dump(job_copy) no time now to test + # since it needs to be run inside the cluster return jsonify({'job': job_copy}), 200 else: - abort(404) + return jsonify({'message': 'The job {} doesn\'t exist'. + format(job_id)}), 400 +@app.route('/apispec', methods=['GET']) +def get_openapi_spec(): + """Get OpenAPI Spec. + + FIXME add openapi spec + """ + return jsonify(app.config['OPENAPI_SPEC']) + if __name__ == '__main__': logging.basicConfig( level=logging.DEBUG, @@ -271,11 +258,17 @@ def get_job(job_id): job_event_reader_thread = threading.Thread(target=watch_jobs, args=(JOB_DB, app.config['PYKUBE_API'])) + job_event_reader_thread.start() pod_event_reader_thread = threading.Thread(target=watch_pods, args=(JOB_DB, app.config['PYKUBE_API'])) - app.config['PYKUBE_CLIENT'] = create_api_client(app.config['PYKUBE_API']) + with app.app_context(): + app.config['OPENAPI_SPEC'] = build_openapi_spec() + app.config['PYKUBE_CLIENT'] = create_api_client( + app.config['PYKUBE_API']) + pod_event_reader_thread.start() + app.run(debug=True, port=5000, host='0.0.0.0') diff --git a/reana_job_controller/cli.py b/reana_job_controller/cli.py new file mode 100644 index 00000000..5eefb168 --- /dev/null +++ b/reana_job_controller/cli.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2017 CERN. +# +# REANA is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# REANA is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# REANA; if not, write to the Free Software Foundation, Inc., 59 Temple Place, +# Suite 330, Boston, MA 02111-1307, USA. +# +# In applying this license, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +"""Click command-line interface for REANA Job Controller.""" + +import io +import json + +import click +from flask import current_app +from flask.cli import with_appcontext + +from .spec import build_openapi_spec + + +@click.group() +def openapi(): + """Openapi management commands.""" + + +@openapi.command() +@click.argument('output', type=click.File('w')) +@with_appcontext +def create(output): + """Generate OpenAPI file.""" + spec = build_openapi_spec() + output.write(json.dumps(spec, indent=2)) + if not isinstance(output, io.TextIOWrapper): + click.echo( + click.style('OpenAPI specification written to {}'.format( + output.name), fg='green')) diff --git a/reana_job_controller/schemas.py b/reana_job_controller/schemas.py new file mode 100644 index 00000000..89a710e2 --- /dev/null +++ b/reana_job_controller/schemas.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2017 CERN. +# +# REANA is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# REANA is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# REANA; if not, write to the Free Software Foundation, Inc., 59 Temple Place, +# Suite 330, Boston, MA 02111-1307, USA. +# +# In applying this license, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +"""REANA Job Controller models.""" + +import uuid + +from marshmallow import Schema, fields, pre_load + + +class Job(Schema): + """Job model.""" + + cmd = fields.Str(required=True) + docker_img = fields.Str(required=True) + experiment = fields.Str(required=True) + job_id = fields.Str(required=True) + max_restart_count = fields.Int(required=True) + restart_count = fields.Int(required=True) + status = fields.Str(required=True) + cvmfs_mounts = fields.List(fields.String(), required=True) + + +class JobRequest(Schema): + """Job request model.""" + + job_id = fields.UUID() + cmd = fields.Str(missing='') + docker_img = fields.Str(required=True) + experiment = fields.Str(required=True) + cvmfs_mounts = fields.List(fields.String(), missing=[]) + env_vars = fields.Dict(missing={}) + shared_file_system = fields.Bool(missing=True) + + @pre_load + def make_id(self, data): + """Generate UUID for new Jobs.""" + data['job_id'] = uuid.uuid4() + return data diff --git a/reana_job_controller/spec.py b/reana_job_controller/spec.py new file mode 100644 index 00000000..3773e2d4 --- /dev/null +++ b/reana_job_controller/spec.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2017 CERN. +# +# REANA is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# REANA is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# REANA; if not, write to the Free Software Foundation, Inc., 59 Temple Place, +# Suite 330, Boston, MA 02111-1307, USA. +# +# In applying this license, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +"""OpenAPI generator.""" + +from apispec import APISpec +from flask import current_app + +from reana_job_controller.schemas import Job, JobRequest + + +def build_openapi_spec(): + """Create OpenAPI definition.""" + # Create OpenAPI specification object + # FIXME set `title`, `version` ... as parameters + spec = APISpec( + title='reana-job-controller', + version='0.0.1', + info=dict( + description='REANA Job Controller API' + ), + plugins=[ + 'apispec.ext.flask', + 'apispec.ext.marshmallow', + ] + ) + + # Add marshmallow models to specification + spec.definition('Job', schema=Job) + spec.definition('JobRequest', schema=JobRequest) + + # Collect OpenAPI docstrings from Flask endpoints + for key in current_app.view_functions: + if key != 'static' and key != 'get_openapi_spec': + spec.add_path(view=current_app.view_functions[key]) + + return spec.to_dict() diff --git a/run-tests.sh b/run-tests.sh index ee00e837..10c5b78c 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -23,7 +23,9 @@ pydocstyle reana_job_controller && \ isort -rc -c -df **/*.py && \ check-manifest --ignore ".travis-*" && \ -sphinx-build -qnNW docs docs/_build/html && \ +FLASK_APP=reana_job_controller/app.py flask openapi create docs/openapi.json && \ +sphinx-build -qnN docs docs/_build/ && \ python setup.py test && \ -sphinx-build -qnNW -b doctest docs docs/_build/doctest && \ +sphinx-build -qnN -b doctest docs docs/_build/doctest && \ +rm -rf docs/openapi.json && \ docker build -t reanahub/reana-job-controller . diff --git a/setup.py b/setup.py index a0f036b7..0a88276b 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ 'pytest-cov>=1.8.0', 'pytest-pep8>=1.0.6', 'pytest>=2.8.0', + 'swagger_spec_validator>=2.1.0', ] extras_require = { @@ -48,6 +49,7 @@ 'Sphinx>=1.4.4', 'sphinx-rtd-theme>=0.1.9', 'sphinxcontrib-httpdomain>=1.5.0', + 'sphinxcontrib-openapi>=0.3.0' ], 'tests': tests_require, } @@ -63,8 +65,10 @@ ] install_requires = [ - 'Flask==0.10.1', + 'Flask>=0.11', 'pykube>=0.14.0', + 'apispec>=0.21.0', + 'marshmallow>=2.13', ] packages = find_packages() @@ -87,6 +91,11 @@ url='https://github.com/reanahub/reana-job-controller', packages=['reana_job_controller', ], zip_safe=False, + entry_points={ + 'flask.commands': [ + 'openapi = reana_job_controller.cli:openapi', + ], + }, extras_require=extras_require, install_requires=install_requires, setup_requires=setup_requires, diff --git a/tests/test_openapi_spec.py b/tests/test_openapi_spec.py new file mode 100644 index 00000000..a8fce09b --- /dev/null +++ b/tests/test_openapi_spec.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# This file is part of REANA. +# Copyright (C) 2017 CERN. +# +# REANA is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# REANA is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# REANA; if not, write to the Free Software Foundation, Inc., 59 Temple Place, +# Suite 330, Boston, MA 02111-1307, USA. +# +# In applying this license, CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization or +# submit itself to any jurisdiction. + +"""REANA-Job-Controller tests.""" + +from __future__ import absolute_import, print_function + +import json +import os + +from swagger_spec_validator.validator20 import validate_json + + +def test_openapi_spec(): + """Test OpenAPI spec validation.""" + + current_dir = os.path.abspath(os.path.dirname(__file__)) + with open(os.path.join(current_dir, '../docs/openapi.json')) as f: + reana_job_controller_spec = json.load(f) + + validate_json(reana_job_controller_spec, 'schemas/v2.0/schema.json')