From 42f547076deedff2524b9ad48afd904ab2547b2d Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Tue, 23 Jun 2020 19:32:40 +0100 Subject: [PATCH 01/26] init readthedocs --- .gitignore | 1 + .readthedocs.yml | 2 ++ docs/conf.py | 53 ++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 44 ++++++++++++++++++++++++++++++++++++ docs/ipython2cwl.rst | 14 ++++++++++++ 5 files changed, 114 insertions(+) create mode 100644 .readthedocs.yml create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/ipython2cwl.rst diff --git a/.gitignore b/.gitignore index 4af9dd9..2a5960e 100644 --- a/.gitignore +++ b/.gitignore @@ -243,3 +243,4 @@ cython_debug/ /external_examples/ /tests/jn/output/ tmp.py +/html/ diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..7eb135b --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,2 @@ +python: + setup_py_install: true diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..3ce62ae --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,53 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +# -- Project information ----------------------------------------------------- + +project = 'ipython2cwl' +copyright = '2020, Yannis Doukas' +author = 'Yannis Doukas' + +# The full version, including alpha/beta/rc tags +release = "0.1" + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +master_doc = 'index' \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..2e7d704 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,44 @@ +IPython2CWL: Convert Jupyter Notebook to CWL +================================================================================ + +.. image:: https://travis-ci.com/giannisdoukas/ipython2cwl.svg?branch=master + :target: https://travis-ci.com/giannisdoukas/ipython2cwl +.. image:: https://coveralls.io/repos/github/giannisdoukas/ipython2cwl/badge.svg?branch=master + :target: https://coveralls.io/github/giannisdoukas/ipython2cwl?branch=master + + +------------------------------------------------------------------------------------------ + +IPython2CWL is a tool for converting `IPython `_ Jupyter Notebooks to +`CWL `_ Command Line Tools by simply providing typing annotation. + +.. code-block:: python + + from ipython2cwl.iotypes import CWLFilePathInput, CWLFilePathOutput + import csv + input_filename: CWLFilePathInput = 'data.csv' + with open(input_filename) as f: + csv_reader = csv.reader(f) + data = [line for line in csv_reader] + number_of_lines = len(data) + result_file: CWLFilePathOutput = 'number_of_lines.txt' + with open(result_file, 'w') as f: + f.write(str(number_of_lines)) + + +------------------------------------------------------------------------------------------ + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + ipython2cwl + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/ipython2cwl.rst b/docs/ipython2cwl.rst new file mode 100644 index 0000000..cbd0496 --- /dev/null +++ b/docs/ipython2cwl.rst @@ -0,0 +1,14 @@ +IPython2CWL Module +=================== + +IPython2CWL +=============== +.. automodule:: ipython2cwl.ipython2cwl + :members: + :undoc-members: + +CWLTool +=============== +.. automodule:: ipython2cwl.cwltool + :members: + :undoc-members: From c3bc5923c859962d78912026419edd0a742b9484 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Tue, 23 Jun 2020 19:35:49 +0100 Subject: [PATCH 02/26] add readthedocs badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b7c4ad1..d809e95 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,5 @@ [![Build Status](https://travis-ci.com/giannisdoukas/ipython2cwl.svg)](https://travis-ci.com/giannisdoukas/ipython2cwl) [![Coverage Status](https://coveralls.io/repos/github/giannisdoukas/ipython2cwl/badge.svg?branch=dev)](https://coveralls.io/github/giannisdoukas/ipython2cwl?branch=dev) +[![Documentation Status](https://readthedocs.org/projects/ipython2cwl/badge/?version=latest)](https://ipython2cwl.readthedocs.io/en/latest/?badge=latest) From b364ad0582e0d5f08f0141c2a809045318047ac2 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Fri, 26 Jun 2020 23:30:03 +0100 Subject: [PATCH 03/26] init repo2cwl --- ipython2cwl/ipython2cwl.py | 13 +++-- ipython2cwl/repo2cwl.py | 82 +++++++++++++++++++++++++++++ setup.py | 4 +- test-requirements.txt | 4 +- tests/simple.ipynb | 73 +++++++++++++++++++++++++ tests/test_cwltool.py | 18 +++++++ tests/test_ipython2cwl_from_repo.py | 67 +++++++++++++++++++++++ 7 files changed, 254 insertions(+), 7 deletions(-) create mode 100644 ipython2cwl/repo2cwl.py create mode 100644 tests/simple.ipynb create mode 100644 tests/test_ipython2cwl_from_repo.py diff --git a/ipython2cwl/ipython2cwl.py b/ipython2cwl/ipython2cwl.py index c7bad11..7e311d5 100644 --- a/ipython2cwl/ipython2cwl.py +++ b/ipython2cwl/ipython2cwl.py @@ -3,9 +3,17 @@ from typing import List, Optional import nbformat + from .cwltool import AnnotatedIPython2CWLToolConverter +def jn2code(notebook): + return '\n'.join( + [f"\n\n# --------- cell - {i} ---------\n\n{cell.source}" for i, cell in + enumerate(filter(lambda c: c.cell_type == 'code', notebook.cells), start=1)] + ) + + def main(argv: Optional[List[str]] = None): if argv is None: import sys @@ -17,10 +25,7 @@ def main(argv: Optional[List[str]] = None): notebook = nbformat.read(args.jn[0], as_version=4) output: Path = args.output args.jn[0].close() - script_code = '\n'.join( - [f"\n\n# --------- cell - {i} ---------\n\n{cell.source}" for i, cell in - enumerate(filter(lambda c: c.cell_type == 'code', notebook.cells), start=1)] - ) + script_code = jn2code(notebook) converter = AnnotatedIPython2CWLToolConverter(script_code) converter.compile(output) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py new file mode 100644 index 0000000..e8017bd --- /dev/null +++ b/ipython2cwl/repo2cwl.py @@ -0,0 +1,82 @@ +import argparse +import os +from pathlib import Path +from typing import List, Optional, Tuple, Dict +from urllib.parse import urlparse + +import nbformat +from git import Repo +from repo2docker import Repo2Docker + +from .cwltool import AnnotatedIPython2CWLToolConverter +from .ipython2cwl import jn2code + + +def main(argv: Optional[List[str]] = None): + if argv is None: + import sys + argv = sys.argv + parser = argparse.ArgumentParser() + parser.add_argument('repo', type=urlparse, nargs=1) + parser.add_argument('-o', '--output', type=Path, required=True) + args = parser.parse_args(argv[1:]) + + notebook = nbformat.read(args.jn[0], as_version=4) + output: Path = args.output + args.jn[0].close() + script_code = '\n'.join( + [f"\n\n# --------- cell - {i} ---------\n\n{cell.source}" for i, cell in + enumerate(filter(lambda c: c.cell_type == 'code', notebook.cells), start=1)] + ) + + converter = AnnotatedIPython2CWLToolConverter(script_code) + converter.compile(output) + + return 0 + + +def repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: + """ + Takes an original + :param git_directory_path: + :return: The generated build image id & the cwl description + """ + r2d = Repo2Docker() + r2d.target_repo_dir = os.path.join(os.path.sep, 'app') + r2d.repo = git_directory_path.tree().abspath + bin_path = os.path.join(r2d.repo, 'cwl', 'bin') + os.makedirs(bin_path, exist_ok=True) + notebooks_paths = [] + for path, subdirs, files in os.walk(r2d.repo): + for name in files: + if name.endswith('.ipynb'): + notebooks_paths.append(os.path.join(path, name)) + + tools = [] + for notebook in notebooks_paths: + with open(notebook) as fd: + code = jn2code(nbformat.read(fd, as_version=4)) + + converter = AnnotatedIPython2CWLToolConverter(code) + + new_script_path = os.path.join(bin_path, os.path.basename(notebook)[:-6]) + script = os.linesep.join([ + '#!/usr/bin/env python', + converter._wrap_script_to_method(converter._tree, converter._variables) + ]) + with open(new_script_path, 'w') as fd: + fd.write(script) + tool = converter.cwl_command_line_tool(r2d.output_image_spec) + tool['baseCommand'] = '/cwl/bin/simple' + tools.append(tool) + git_directory_path.index.commit("auto-commit") + + r2d.build() + # fix dockerImageId + for tool in tools: + tool['hints']['DockerRequirement']['dockerImageId'] = r2d.output_image_spec + return r2d.output_image_spec, tools + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 8ac2afa..c613a0e 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,6 @@ import os.path from setuptools import setup -from setuptools import find_packages name = 'ipython2cwl' @@ -56,7 +55,8 @@ def get_version(rel_path): install_requires=[ 'nbformat>=5.0.6', 'astor>=0.8.1', - 'PyYAML>=5.3.1' + 'PyYAML>=5.3.1', + 'gitpython>=3.1.3', ], test_suite='tests', ) diff --git a/test-requirements.txt b/test-requirements.txt index e4d5a20..15475e2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,6 @@ pycodestyle>=2.6.0 coverage>=5.1 coveralls>=2.0.0 -virtualenv>=3.1.0 \ No newline at end of file +virtualenv>=3.1.0 +gitpython>=3.1.3 +docker>=4.2.1 diff --git a/tests/simple.ipynb b/tests/simple.ipynb new file mode 100644 index 0000000..a9d7b61 --- /dev/null +++ b/tests/simple.ipynb @@ -0,0 +1,73 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import matplotlib\n", + "from ipython2cwl.iotypes import CWLFilePathInput, CWLFilePathOutput" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "dataset: CWLFilePathInput = 'example.csv'" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "data = pd.read_csv(dataset)\n", + "# original data\n", + "fig = data.plot()\n", + "\n", + "original_image: CWLFilePathOutput = 'original_data.png'\n", + "fig.figure.savefig(original_image)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# transform data\n", + "data.sort_values(by='Random B', ascending=False, inplace=True, ignore_index=True)\n", + "fig = data.plot()\n", + "\n", + "after_transform_data: CWLFilePathOutput = 'new_data.png'\n", + "fig.figure.savefig(after_transform_data)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/tests/test_cwltool.py b/tests/test_cwltool.py index c4e9bd1..5132fe4 100644 --- a/tests/test_cwltool.py +++ b/tests/test_cwltool.py @@ -256,3 +256,21 @@ def test_AnnotatedIPython2CWLToolConverter_output_file_annotation(self): }, tool ) + + def test_AnnotatedIPython2CWLToolConverter_exclamation_mark_command(self): + printed_message = '' + annotated_python_script = os.linesep.join([ + '!ls -la', + 'global printed_message', + f"msg: {CWLStringInput.__name__} = 'original'", + "print('message:', msg)", + "printed_message = msg" + ]) + exec(annotated_python_script) + self.assertEqual('original', globals()['printed_message']) + converter = AnnotatedIPython2CWLToolConverter(annotated_python_script) + new_script = converter._wrap_script_to_method(converter._tree, converter._variables) + print('\n' + new_script, '\n') + exec(new_script) + locals()['main']('new message') + self.assertEqual('new message', globals()['printed_message']) diff --git a/tests/test_ipython2cwl_from_repo.py b/tests/test_ipython2cwl_from_repo.py new file mode 100644 index 0000000..8bb042e --- /dev/null +++ b/tests/test_ipython2cwl_from_repo.py @@ -0,0 +1,67 @@ +import os +import shutil +import tempfile +from unittest import TestCase + +import docker +from git import Repo + +from ipython2cwl.repo2cwl import repo2cwl + + +class Test2CWLFromRepo(TestCase): + maxDiff = None + here = os.path.abspath(os.path.dirname(__file__)) + + def test_docker_build(self): + # TODO: test with jn with same name + # setup a simple git repo + git_dir = tempfile.mkdtemp() + jn_repo = Repo.init(git_dir) + shutil.copy( + os.path.join(self.here, 'simple.ipynb'), + os.path.join(git_dir, 'simple.ipynb'), + ) + jn_repo.index.add('simple.ipynb') + jn_repo.index.commit("initial commit") + + print(git_dir) + + dockerfile_image_id, cwl_tool = repo2cwl(jn_repo) + self.assertEqual(1, len(cwl_tool)) + docker_client = docker.from_env() + script = docker_client.containers.run(dockerfile_image_id, 'cat /app/cwl/bin/simple') + self.assertIn('fig.figure.savefig(after_transform_data)', script.decode()) + self.assertDictEqual( + { + 'cwlVersion': "v1.1", + 'class': 'CommandLineTool', + 'baseCommand': '/cwl/bin/simple', + 'hints': { + 'DockerRequirement': {'dockerImageId': dockerfile_image_id} + }, + 'inputs': { + 'dataset': { + 'type': 'File', + 'inputBinding': { + 'prefix': '--dataset' + } + } + }, + 'outputs': { + 'original_image': { + 'type': 'File', + 'outputBinding': { + 'glob': 'original_data.png' + } + }, + 'after_transform_data': { + 'type': 'File', + 'outputBinding': { + 'glob': 'new_data.png' + } + } + }, + }, + cwl_tool[0] + ) From c8f338b8ad2a2cd00d99dc5fdb1aa0f73e5a51f6 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Fri, 26 Jun 2020 23:53:01 +0100 Subject: [PATCH 04/26] add requirement --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index c613a0e..b6bb937 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ def get_version(rel_path): 'astor>=0.8.1', 'PyYAML>=5.3.1', 'gitpython>=3.1.3', + 'jupyter-repo2docker>=0.11.0' ], test_suite='tests', ) From 16059737c41b4f7882d365002e4efd1f25d66554 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 00:45:03 +0100 Subject: [PATCH 05/26] rm test from that branch --- tests/test_cwltool.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_cwltool.py b/tests/test_cwltool.py index 5132fe4..4356913 100644 --- a/tests/test_cwltool.py +++ b/tests/test_cwltool.py @@ -257,20 +257,20 @@ def test_AnnotatedIPython2CWLToolConverter_output_file_annotation(self): tool ) - def test_AnnotatedIPython2CWLToolConverter_exclamation_mark_command(self): - printed_message = '' - annotated_python_script = os.linesep.join([ - '!ls -la', - 'global printed_message', - f"msg: {CWLStringInput.__name__} = 'original'", - "print('message:', msg)", - "printed_message = msg" - ]) - exec(annotated_python_script) - self.assertEqual('original', globals()['printed_message']) - converter = AnnotatedIPython2CWLToolConverter(annotated_python_script) - new_script = converter._wrap_script_to_method(converter._tree, converter._variables) - print('\n' + new_script, '\n') - exec(new_script) - locals()['main']('new message') - self.assertEqual('new message', globals()['printed_message']) + # def test_AnnotatedIPython2CWLToolConverter_exclamation_mark_command(self): + # printed_message = '' + # annotated_python_script = os.linesep.join([ + # '!ls -la', + # 'global printed_message', + # f"msg: {CWLStringInput.__name__} = 'original'", + # "print('message:', msg)", + # "printed_message = msg" + # ]) + # exec(annotated_python_script) + # self.assertEqual('original', globals()['printed_message']) + # converter = AnnotatedIPython2CWLToolConverter(annotated_python_script) + # new_script = converter._wrap_script_to_method(converter._tree, converter._variables) + # print('\n' + new_script, '\n') + # exec(new_script) + # locals()['main']('new message') + # self.assertEqual('new message', globals()['printed_message']) From 6ca19b49cf22e4ece6b5ee9602e974963712b3d7 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 01:06:23 +0100 Subject: [PATCH 06/26] fix permissions issues --- ipython2cwl/repo2cwl.py | 10 ++++++++-- tests/test_ipython2cwl_from_repo.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index e8017bd..f0b80bf 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -1,5 +1,6 @@ import argparse import os +import stat from pathlib import Path from typing import List, Optional, Tuple, Dict from urllib.parse import urlparse @@ -59,7 +60,8 @@ def repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: converter = AnnotatedIPython2CWLToolConverter(code) - new_script_path = os.path.join(bin_path, os.path.basename(notebook)[:-6]) + script_name = os.path.basename(notebook)[:-6] + new_script_path = os.path.join(bin_path, script_name) script = os.linesep.join([ '#!/usr/bin/env python', converter._wrap_script_to_method(converter._tree, converter._variables) @@ -67,7 +69,11 @@ def repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: with open(new_script_path, 'w') as fd: fd.write(script) tool = converter.cwl_command_line_tool(r2d.output_image_spec) - tool['baseCommand'] = '/cwl/bin/simple' + in_git_dir_script_file = os.path.join(bin_path, script_name) + tool_st = os.stat(in_git_dir_script_file) + os.chmod(in_git_dir_script_file, tool_st.st_mode | stat.S_IEXEC) + + tool['baseCommand'] = os.path.join('/app', 'cwl', 'bin', script_name) tools.append(tool) git_directory_path.index.commit("auto-commit") diff --git a/tests/test_ipython2cwl_from_repo.py b/tests/test_ipython2cwl_from_repo.py index 8bb042e..fc04b3e 100644 --- a/tests/test_ipython2cwl_from_repo.py +++ b/tests/test_ipython2cwl_from_repo.py @@ -1,9 +1,11 @@ import os import shutil import tempfile +from io import StringIO from unittest import TestCase import docker +import yaml from git import Repo from ipython2cwl.repo2cwl import repo2cwl @@ -15,6 +17,7 @@ class Test2CWLFromRepo(TestCase): def test_docker_build(self): # TODO: test with jn with same name + # TODO: test having notebooks without typing annotations # setup a simple git repo git_dir = tempfile.mkdtemp() jn_repo = Repo.init(git_dir) @@ -22,7 +25,11 @@ def test_docker_build(self): os.path.join(self.here, 'simple.ipynb'), os.path.join(git_dir, 'simple.ipynb'), ) + with open(os.path.join(git_dir, 'requirements.txt'), 'w') as f: + f.write('pandas\n') + f.write('matplotlib\n') jn_repo.index.add('simple.ipynb') + jn_repo.index.add('requirements.txt') jn_repo.index.commit("initial commit") print(git_dir) @@ -36,7 +43,7 @@ def test_docker_build(self): { 'cwlVersion': "v1.1", 'class': 'CommandLineTool', - 'baseCommand': '/cwl/bin/simple', + 'baseCommand': '/app/cwl/bin/simple', 'hints': { 'DockerRequirement': {'dockerImageId': dockerfile_image_id} }, @@ -65,3 +72,6 @@ def test_docker_build(self): }, cwl_tool[0] ) + cwl = StringIO() + yaml.safe_dump(cwl_tool[0], cwl) + print(cwl.getvalue()) From 59c876a29904101ba007556e69fb290a1f237a12 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 08:48:32 +0100 Subject: [PATCH 07/26] add docker service in travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 36a1576..a9b1c60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +services: + - docker language: python python: - "3.6" From c0cf051e5622326d7a4c5ef75ba4c62d158a28be Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 09:27:39 +0100 Subject: [PATCH 08/26] add sudo in travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a9b1c60..6f236fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,6 @@ services: - docker +sudo: required language: python python: - "3.6" From 267b4d9a15e87f031be3c1ffd81d81fdfc78875c Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 10:14:01 +0100 Subject: [PATCH 09/26] fix issue with entrypoint for travis --- tests/test_ipython2cwl_from_repo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_ipython2cwl_from_repo.py b/tests/test_ipython2cwl_from_repo.py index fc04b3e..5e45526 100644 --- a/tests/test_ipython2cwl_from_repo.py +++ b/tests/test_ipython2cwl_from_repo.py @@ -18,6 +18,7 @@ class Test2CWLFromRepo(TestCase): def test_docker_build(self): # TODO: test with jn with same name # TODO: test having notebooks without typing annotations + # TODO: should I execute cwltool?? # setup a simple git repo git_dir = tempfile.mkdtemp() jn_repo = Repo.init(git_dir) @@ -37,7 +38,7 @@ def test_docker_build(self): dockerfile_image_id, cwl_tool = repo2cwl(jn_repo) self.assertEqual(1, len(cwl_tool)) docker_client = docker.from_env() - script = docker_client.containers.run(dockerfile_image_id, 'cat /app/cwl/bin/simple') + script = docker_client.containers.run(dockerfile_image_id, '/app/cwl/bin/simple', entrypoint='/bin/cat') self.assertIn('fig.figure.savefig(after_transform_data)', script.decode()) self.assertDictEqual( { @@ -74,4 +75,5 @@ def test_docker_build(self): ) cwl = StringIO() yaml.safe_dump(cwl_tool[0], cwl) - print(cwl.getvalue()) + cwl_code = cwl.getvalue() + print(cwl_code) From 1c04268c869cb6b785d9d18961944bfd05b059a0 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 10:19:46 +0100 Subject: [PATCH 10/26] rm sudo from travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6f236fc..a9b1c60 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ services: - docker -sudo: required language: python python: - "3.6" From a93ea54f8321a425bf1b483f7885dae17883faa9 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 10:24:54 +0100 Subject: [PATCH 11/26] add docker for osx matrix --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a9b1c60..d7f6d54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,4 +29,6 @@ matrix: - virtualenv -p python3 venv - source venv/bin/activate - pip3 install -U -r test-requirements.txt - script: coverage run --source ipython2cwl -m unittest discover tests \ No newline at end of file + script: coverage run --source ipython2cwl -m unittest discover tests + services: + - docker \ No newline at end of file From b1fa5c84a652178c283cd97f1381df7db8b203b5 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 10:55:47 +0100 Subject: [PATCH 12/26] skip tests with docker in travis-osx --- .travis.yml | 3 +-- tests/test_ipython2cwl_from_repo.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d7f6d54..b626821 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,5 +30,4 @@ matrix: - source venv/bin/activate - pip3 install -U -r test-requirements.txt script: coverage run --source ipython2cwl -m unittest discover tests - services: - - docker \ No newline at end of file + env: TRAVIS_IGNORE_DOCKER=true \ No newline at end of file diff --git a/tests/test_ipython2cwl_from_repo.py b/tests/test_ipython2cwl_from_repo.py index 5e45526..eb5fb71 100644 --- a/tests/test_ipython2cwl_from_repo.py +++ b/tests/test_ipython2cwl_from_repo.py @@ -2,7 +2,7 @@ import shutil import tempfile from io import StringIO -from unittest import TestCase +from unittest import TestCase, skipIf import docker import yaml @@ -15,6 +15,8 @@ class Test2CWLFromRepo(TestCase): maxDiff = None here = os.path.abspath(os.path.dirname(__file__)) + @skipIf("TRAVIS_IGNORE_DOCKER" in os.environ and os.environ["TRAVIS_IGNORE_DOCKER"] == "true", + "Skipping this test on Travis CI.") def test_docker_build(self): # TODO: test with jn with same name # TODO: test having notebooks without typing annotations From 0bb0426c12f8485b4e55e787e71c79ed9da524c2 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 12:03:26 +0100 Subject: [PATCH 13/26] support non annotated jupyter notebook & notebooks with the same name in different directories --- ipython2cwl/repo2cwl.py | 84 +++++++++++++++++++---------- tests/non-annotated.ipynb | 72 +++++++++++++++++++++++++ tests/test_ipython2cwl_from_repo.py | 31 +++++++++-- 3 files changed, 157 insertions(+), 30 deletions(-) create mode 100644 tests/non-annotated.ipynb diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index f0b80bf..7be16b1 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -1,4 +1,5 @@ import argparse +import logging import os import stat from pathlib import Path @@ -12,6 +13,8 @@ from .cwltool import AnnotatedIPython2CWLToolConverter from .ipython2cwl import jn2code +logger = logging.getLogger() + def main(argv: Optional[List[str]] = None): if argv is None: @@ -36,6 +39,47 @@ def main(argv: Optional[List[str]] = None): return 0 +def _get_notebook_paths_from_dir(dir_path: str): + notebooks_paths = [] + for path, subdirs, files in os.walk(dir_path): + for name in files: + if name.endswith('.ipynb'): + notebooks_paths.append(os.path.join(path, name)) + return notebooks_paths + + +def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bin_absolute_path: str, image_id: str) \ + -> Tuple[Optional[Dict], Optional[str]]: + with open(notebook_path) as fd: + code = jn2code(nbformat.read(fd, as_version=4)) + + converter = AnnotatedIPython2CWLToolConverter(code) + if len(converter._variables) == 0: + logger.info(f"Notebook {notebook_path} does not contains typing annotations. skipping...") + return None, None + script_relative_path = os.path.relpath(notebook_path, git_directory_absolute_path)[:-6] + script_relative_parent_directories = script_relative_path.split(os.sep) + if len(script_relative_parent_directories) > 1: + script_absolute_name = os.path.join(bin_absolute_path, os.sep.join(script_relative_parent_directories[:-1])) + os.makedirs( + script_absolute_name, + exist_ok=True) + script_absolute_name = os.path.join(script_absolute_name, os.path.basename(script_relative_path)) + else: + script_absolute_name = os.path.join(bin_absolute_path, script_relative_path) + script = os.linesep.join([ + '#!/usr/bin/env python', + converter._wrap_script_to_method(converter._tree, converter._variables) + ]) + with open(script_absolute_name, 'w') as fd: + fd.write(script) + tool = converter.cwl_command_line_tool(image_id) + in_git_dir_script_file = os.path.join(bin_absolute_path, script_relative_path) + tool_st = os.stat(in_git_dir_script_file) + os.chmod(in_git_dir_script_file, tool_st.st_mode | stat.S_IEXEC) + return tool, script_relative_path + + def repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: """ Takes an original @@ -47,40 +91,26 @@ def repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: r2d.repo = git_directory_path.tree().abspath bin_path = os.path.join(r2d.repo, 'cwl', 'bin') os.makedirs(bin_path, exist_ok=True) - notebooks_paths = [] - for path, subdirs, files in os.walk(r2d.repo): - for name in files: - if name.endswith('.ipynb'): - notebooks_paths.append(os.path.join(path, name)) + notebooks_paths = _get_notebook_paths_from_dir(r2d.repo) tools = [] for notebook in notebooks_paths: - with open(notebook) as fd: - code = jn2code(nbformat.read(fd, as_version=4)) - - converter = AnnotatedIPython2CWLToolConverter(code) - - script_name = os.path.basename(notebook)[:-6] - new_script_path = os.path.join(bin_path, script_name) - script = os.linesep.join([ - '#!/usr/bin/env python', - converter._wrap_script_to_method(converter._tree, converter._variables) - ]) - with open(new_script_path, 'w') as fd: - fd.write(script) - tool = converter.cwl_command_line_tool(r2d.output_image_spec) - in_git_dir_script_file = os.path.join(bin_path, script_name) - tool_st = os.stat(in_git_dir_script_file) - os.chmod(in_git_dir_script_file, tool_st.st_mode | stat.S_IEXEC) - - tool['baseCommand'] = os.path.join('/app', 'cwl', 'bin', script_name) - tools.append(tool) + cwl_command_line_tool, script_name = _store_jn_as_script( + notebook, + git_directory_path.tree().abspath, + bin_path, + r2d.output_image_spec + ) + if cwl_command_line_tool is None: + continue + cwl_command_line_tool['baseCommand'] = os.path.join('/app', 'cwl', 'bin', script_name) + tools.append(cwl_command_line_tool) git_directory_path.index.commit("auto-commit") r2d.build() # fix dockerImageId - for tool in tools: - tool['hints']['DockerRequirement']['dockerImageId'] = r2d.output_image_spec + for cwl_command_line_tool in tools: + cwl_command_line_tool['hints']['DockerRequirement']['dockerImageId'] = r2d.output_image_spec return r2d.output_image_spec, tools diff --git a/tests/non-annotated.ipynb b/tests/non-annotated.ipynb new file mode 100644 index 0000000..cac765d --- /dev/null +++ b/tests/non-annotated.ipynb @@ -0,0 +1,72 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import matplotlib\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = 'example.csv'" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "data = pd.read_csv(dataset)\n", + "# original data\n", + "fig = data.plot()\n", + "\n", + "original_image = 'original_data.png'\n", + "fig.figure.savefig(original_image)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# transform data\n", + "data.sort_values(by='Random B', ascending=False, inplace=True, ignore_index=True)\n", + "fig = data.plot()\n", + "\n", + "after_transform_data = 'new_data.png'\n", + "fig.figure.savefig(after_transform_data)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/tests/test_ipython2cwl_from_repo.py b/tests/test_ipython2cwl_from_repo.py index eb5fb71..5ffcd15 100644 --- a/tests/test_ipython2cwl_from_repo.py +++ b/tests/test_ipython2cwl_from_repo.py @@ -18,9 +18,6 @@ class Test2CWLFromRepo(TestCase): @skipIf("TRAVIS_IGNORE_DOCKER" in os.environ and os.environ["TRAVIS_IGNORE_DOCKER"] == "true", "Skipping this test on Travis CI.") def test_docker_build(self): - # TODO: test with jn with same name - # TODO: test having notebooks without typing annotations - # TODO: should I execute cwltool?? # setup a simple git repo git_dir = tempfile.mkdtemp() jn_repo = Repo.init(git_dir) @@ -79,3 +76,31 @@ def test_docker_build(self): yaml.safe_dump(cwl_tool[0], cwl) cwl_code = cwl.getvalue() print(cwl_code) + + # test for non-annotated jn + shutil.copy( + os.path.join(self.here, 'non-annotated.ipynb'), + os.path.join(git_dir, 'non-annotated.ipynb'), + ) + jn_repo.index.add("non-annotated.ipynb") + jn_repo.index.commit("add non annotated notebook") + dockerfile_image_id, new_cwl_tool = repo2cwl(jn_repo) + self.assertEqual(1, len(new_cwl_tool)) + cwl_tool[0]['hints']['DockerRequirement'].pop('dockerImageId') + new_cwl_tool[0]['hints']['DockerRequirement'].pop('dockerImageId') + self.assertListEqual(cwl_tool, new_cwl_tool) + + # test with jn with same name in different directory + os.makedirs(os.path.join(git_dir, 'subdir'), exist_ok=True) + shutil.copy( + os.path.join(self.here, 'simple.ipynb'), + os.path.join(git_dir, 'subdir', 'simple.ipynb'), + ) + jn_repo.index.add("subdir/simple.ipynb") + jn_repo.index.commit("add second jn with the same name") + dockerfile_image_id, new_cwl_tool = repo2cwl(jn_repo) + base_commands = [tool['baseCommand'] for tool in new_cwl_tool] + base_commands.sort() + self.assertListEqual(base_commands, ['/app/cwl/bin/simple', '/app/cwl/bin/subdir/simple']) + script = docker_client.containers.run(dockerfile_image_id, '/app/cwl/bin/subdir/simple', entrypoint='/bin/cat') + self.assertIn('fig.figure.savefig(after_transform_data)', script.decode()) From c141c7feaea848bfb862fdb02c7ec23adaf054ea Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 15:53:00 +0100 Subject: [PATCH 14/26] add main function for repo2cwl --- .../{cwltool.py => cwltoolextractor.py} | 4 +- ipython2cwl/ipython2cwl.py | 2 +- ipython2cwl/repo2cwl.py | 115 ++++++++++++++---- tests/test_ipython2cwl_from_repo.py | 8 +- 4 files changed, 95 insertions(+), 34 deletions(-) rename ipython2cwl/{cwltool.py => cwltoolextractor.py} (98%) diff --git a/ipython2cwl/cwltool.py b/ipython2cwl/cwltoolextractor.py similarity index 98% rename from ipython2cwl/cwltool.py rename to ipython2cwl/cwltoolextractor.py index a9a65b4..6b7f472 100644 --- a/ipython2cwl/cwltool.py +++ b/ipython2cwl/cwltoolextractor.py @@ -11,8 +11,8 @@ import astor import yaml -from .iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput -from .requirements_manager import RequirementsManager +from iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput +from requirements_manager import RequirementsManager with open(os.sep.join([os.path.abspath(os.path.dirname(__file__)), 'templates', 'template.dockerfile'])) as f: DOCKERFILE_TEMPLATE = f.read() diff --git a/ipython2cwl/ipython2cwl.py b/ipython2cwl/ipython2cwl.py index 7e311d5..20fdd3e 100644 --- a/ipython2cwl/ipython2cwl.py +++ b/ipython2cwl/ipython2cwl.py @@ -4,7 +4,7 @@ import nbformat -from .cwltool import AnnotatedIPython2CWLToolConverter +from cwltoolextractor import AnnotatedIPython2CWLToolConverter def jn2code(notebook): diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 7be16b1..21d9d91 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -1,42 +1,47 @@ import argparse import logging import os +import shutil import stat +import sys +import tempfile from pathlib import Path from typing import List, Optional, Tuple, Dict -from urllib.parse import urlparse +from urllib.parse import urlparse, ParseResult +import git import nbformat +import yaml from git import Repo from repo2docker import Repo2Docker -from .cwltool import AnnotatedIPython2CWLToolConverter -from .ipython2cwl import jn2code +from cwltoolextractor import AnnotatedIPython2CWLToolConverter +from ipython2cwl import jn2code logger = logging.getLogger() -def main(argv: Optional[List[str]] = None): - if argv is None: - import sys - argv = sys.argv - parser = argparse.ArgumentParser() - parser.add_argument('repo', type=urlparse, nargs=1) - parser.add_argument('-o', '--output', type=Path, required=True) - args = parser.parse_args(argv[1:]) - - notebook = nbformat.read(args.jn[0], as_version=4) - output: Path = args.output - args.jn[0].close() - script_code = '\n'.join( - [f"\n\n# --------- cell - {i} ---------\n\n{cell.source}" for i, cell in - enumerate(filter(lambda c: c.cell_type == 'code', notebook.cells), start=1)] - ) - - converter = AnnotatedIPython2CWLToolConverter(script_code) - converter.compile(output) - - return 0 +# def main(argv: Optional[List[str]] = None): +# if argv is None: +# import sys +# argv = sys.argv +# parser = argparse.ArgumentParser() +# parser.add_argument('repo', type=urlparse, nargs=1) +# parser.add_argument('-o', '--output', type=Path, required=True) +# args = parser.parse_args(argv[1:]) +# +# notebook = nbformat.read(args.jn[0], as_version=4) +# output: Path = args.output +# args.jn[0].close() +# script_code = '\n'.join( +# [f"\n\n# --------- cell - {i} ---------\n\n{cell.source}" for i, cell in +# enumerate(filter(lambda c: c.cell_type == 'code', notebook.cells), start=1)] +# ) +# +# converter = AnnotatedIPython2CWLToolConverter(script_code) +# converter.compile(output) +# +# return 0 def _get_notebook_paths_from_dir(dir_path: str): @@ -80,9 +85,58 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi return tool, script_relative_path -def repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: +def existing_path(path: str): + # TODO: validate that the path exists and it is a directory + path = Path(path) + if not path.is_dir(): + raise ValueError('Directory does not exists') + return path + + +def parser_arguments(argv: List[str]): + parser = argparse.ArgumentParser() + parser.add_argument('repo', type=lambda uri: urlparse(uri, scheme='file'), nargs=1) + parser.add_argument('-o', '--output', help='Output directory to store the generated cwl files', + type=existing_path, + required=True) + return parser.parse_args(argv[1:]) + + +def repo2cwl(argv: Optional[List[str]] = None): + argv = sys.argv if argv is None else argv + args = parser_arguments(argv) + uri: ParseResult = args.repo[0] + output_directory: Path = args.output + supported_schemes = {'file', 'http', 'https', 'ssh'} + if uri.scheme not in supported_schemes: + raise ValueError(f'Supported schema uris: {supported_schemes}') + local_git_directory = os.path.join(tempfile.mkdtemp(prefix='repo2cwl'), 'repo') + if uri.scheme == 'file': + if not os.path.isdir(uri.path): + raise ValueError(f'Directory does not exists') + logger.info(f'copy repo to temp directory: {local_git_directory}') + shutil.copytree(uri.path, local_git_directory) + else: + logger.info(f'cloning repo to temp directory: {local_git_directory}') + git.Git(local_git_directory).clone(uri.geturl()) + local_git = git.Repo(local_git_directory) + image_id, cwl_tools = _repo2cwl(local_git) + logger.info(f'Generated image id: {image_id}') + for tool in cwl_tools: + base_command_script_name = f'{tool["baseCommand"][len("/app/cwl/bin/"):].replace("/", "_")}.cwl' + tool_filename = str(output_directory.joinpath(base_command_script_name)) + with open(tool_filename, 'w') as f: + logger.info(f'Creating CWL command line tool: {tool_filename}') + yaml.safe_dump(tool, f) + + logger.info(f'Cleaning local temporary directory {local_git_directory}...') + shutil.rmtree(local_git_directory) + + +def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: """ - Takes an original + Takes a Repo mounted to a local directory. That function will create new files and it will commit the changes. + Do not use that function for Repositories you do not want to change them. :param git_directory_path: :return: The generated build image id & the cwl description """ @@ -115,4 +169,11 @@ def repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: if __name__ == '__main__': - main() + logger.setLevel(logging.DEBUG) + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + repo2cwl() diff --git a/tests/test_ipython2cwl_from_repo.py b/tests/test_ipython2cwl_from_repo.py index 5ffcd15..a0cef3b 100644 --- a/tests/test_ipython2cwl_from_repo.py +++ b/tests/test_ipython2cwl_from_repo.py @@ -8,7 +8,7 @@ import yaml from git import Repo -from ipython2cwl.repo2cwl import repo2cwl +from ipython2cwl.repo2cwl import _repo2cwl class Test2CWLFromRepo(TestCase): @@ -34,7 +34,7 @@ def test_docker_build(self): print(git_dir) - dockerfile_image_id, cwl_tool = repo2cwl(jn_repo) + dockerfile_image_id, cwl_tool = _repo2cwl(jn_repo) self.assertEqual(1, len(cwl_tool)) docker_client = docker.from_env() script = docker_client.containers.run(dockerfile_image_id, '/app/cwl/bin/simple', entrypoint='/bin/cat') @@ -84,7 +84,7 @@ def test_docker_build(self): ) jn_repo.index.add("non-annotated.ipynb") jn_repo.index.commit("add non annotated notebook") - dockerfile_image_id, new_cwl_tool = repo2cwl(jn_repo) + dockerfile_image_id, new_cwl_tool = _repo2cwl(jn_repo) self.assertEqual(1, len(new_cwl_tool)) cwl_tool[0]['hints']['DockerRequirement'].pop('dockerImageId') new_cwl_tool[0]['hints']['DockerRequirement'].pop('dockerImageId') @@ -98,7 +98,7 @@ def test_docker_build(self): ) jn_repo.index.add("subdir/simple.ipynb") jn_repo.index.commit("add second jn with the same name") - dockerfile_image_id, new_cwl_tool = repo2cwl(jn_repo) + dockerfile_image_id, new_cwl_tool = _repo2cwl(jn_repo) base_commands = [tool['baseCommand'] for tool in new_cwl_tool] base_commands.sort() self.assertListEqual(base_commands, ['/app/cwl/bin/simple', '/app/cwl/bin/subdir/simple']) From 2b87df7415f573a79fe2df87185cce26462cfc8c Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 16:24:33 +0100 Subject: [PATCH 15/26] change imports --- ipython2cwl/cwltoolextractor.py | 4 ++-- ipython2cwl/ipython2cwl.py | 2 +- ipython2cwl/repo2cwl.py | 4 ++-- tests/test_cwltool.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ipython2cwl/cwltoolextractor.py b/ipython2cwl/cwltoolextractor.py index 6b7f472..a9a65b4 100644 --- a/ipython2cwl/cwltoolextractor.py +++ b/ipython2cwl/cwltoolextractor.py @@ -11,8 +11,8 @@ import astor import yaml -from iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput -from requirements_manager import RequirementsManager +from .iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput +from .requirements_manager import RequirementsManager with open(os.sep.join([os.path.abspath(os.path.dirname(__file__)), 'templates', 'template.dockerfile'])) as f: DOCKERFILE_TEMPLATE = f.read() diff --git a/ipython2cwl/ipython2cwl.py b/ipython2cwl/ipython2cwl.py index 20fdd3e..8f9dfca 100644 --- a/ipython2cwl/ipython2cwl.py +++ b/ipython2cwl/ipython2cwl.py @@ -4,7 +4,7 @@ import nbformat -from cwltoolextractor import AnnotatedIPython2CWLToolConverter +from .cwltoolextractor import AnnotatedIPython2CWLToolConverter def jn2code(notebook): diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 21d9d91..f940a31 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -15,8 +15,8 @@ from git import Repo from repo2docker import Repo2Docker -from cwltoolextractor import AnnotatedIPython2CWLToolConverter -from ipython2cwl import jn2code +from .cwltoolextractor import AnnotatedIPython2CWLToolConverter +from .ipython2cwl import jn2code logger = logging.getLogger() diff --git a/tests/test_cwltool.py b/tests/test_cwltool.py index 4356913..9c3f5d8 100644 --- a/tests/test_cwltool.py +++ b/tests/test_cwltool.py @@ -4,7 +4,7 @@ from pathlib import Path from unittest import TestCase -from ipython2cwl.cwltool import AnnotatedIPython2CWLToolConverter +from ipython2cwl.cwltoolextractor import AnnotatedIPython2CWLToolConverter from ipython2cwl.iotypes import CWLStringInput, CWLFilePathOutput From b79de81dbadd6d617b790ee5ec451bca2ef518d3 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 16:50:10 +0100 Subject: [PATCH 16/26] support typing hints as strings --- ipython2cwl/cwltoolextractor.py | 6 ++++-- ipython2cwl/repo2cwl.py | 21 +++++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/ipython2cwl/cwltoolextractor.py b/ipython2cwl/cwltoolextractor.py index a9a65b4..7b03c88 100644 --- a/ipython2cwl/cwltoolextractor.py +++ b/ipython2cwl/cwltoolextractor.py @@ -51,7 +51,8 @@ def __init__(self, *args, **kwargs): def visit_AnnAssign(self, node): try: - if isinstance(node.annotation, ast.Name) and node.annotation.id in self.input_type_mapper: + if (isinstance(node.annotation, ast.Name) and node.annotation.id in self.input_type_mapper) or \ + (isinstance(node.annotation, ast.Str) and node.annotation.s in self.input_type_mapper): mapper = self.input_type_mapper[node.annotation.id] self.extracted_nodes.append( (node, mapper[0], mapper[1], True, True, False) @@ -72,7 +73,8 @@ def visit_AnnAssign(self, node): (node, mapper[0] + '[]', mapper[1], True, True, False) ) return None - elif isinstance(node.annotation, ast.Name) and node.annotation.id in self.output_type_mapper: + elif (isinstance(node.annotation, ast.Name) and node.annotation.id in self.output_type_mapper) or \ + (isinstance(node.annotation, ast.Str) and node.annotation.s in self.output_type_mapper): self.extracted_nodes.append( (node, None, None, None, False, True) ) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index f940a31..69d28f9 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -18,7 +18,7 @@ from .cwltoolextractor import AnnotatedIPython2CWLToolConverter from .ipython2cwl import jn2code -logger = logging.getLogger() +logger = logging.getLogger('repo2cwl') # def main(argv: Optional[List[str]] = None): @@ -59,6 +59,7 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi code = jn2code(nbformat.read(fd, as_version=4)) converter = AnnotatedIPython2CWLToolConverter(code) + if len(converter._variables) == 0: logger.info(f"Notebook {notebook_path} does not contains typing annotations. skipping...") return None, None @@ -102,7 +103,16 @@ def parser_arguments(argv: List[str]): return parser.parse_args(argv[1:]) +def setup_logger(): + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + def repo2cwl(argv: Optional[List[str]] = None): + setup_logger() argv = sys.argv if argv is None else argv args = parser_arguments(argv) uri: ParseResult = args.repo[0] @@ -147,6 +157,8 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: os.makedirs(bin_path, exist_ok=True) notebooks_paths = _get_notebook_paths_from_dir(r2d.repo) + print(5 * '-', 'NOTEBOOKS PATHS', notebooks_paths) + tools = [] for notebook in notebooks_paths: cwl_command_line_tool, script_name = _store_jn_as_script( @@ -169,11 +181,4 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: if __name__ == '__main__': - logger.setLevel(logging.DEBUG) - handler = logging.StreamHandler(sys.stdout) - handler.setLevel(logging.INFO) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - handler.setFormatter(formatter) - logger.addHandler(handler) - repo2cwl() From 8c81ffa14bd27516e54cb1d37c796e4ca2148ba9 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 16:50:29 +0100 Subject: [PATCH 17/26] add console script --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b6bb937..880bdd8 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ def get_version(rel_path): entry_points={ 'console_scripts': [ 'jupyter-jn2cwl=ipython2cwl.ipython2cwl:main', + 'jupyter-jnrepo2cwl=ipython2cwl.repo2cwl:repo2cwl', ], }, install_requires=[ From 47289cbad6aba7bc6a112ea97213c5522b399090 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 17:32:11 +0100 Subject: [PATCH 18/26] update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2a5960e..de7ce12 100644 --- a/.gitignore +++ b/.gitignore @@ -244,3 +244,4 @@ cython_debug/ /tests/jn/output/ tmp.py /html/ +cwlbuild \ No newline at end of file From c1b45f82a499db6bb0b047f80651277fb25adbac Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 17:49:42 +0100 Subject: [PATCH 19/26] rm forgotten print --- ipython2cwl/repo2cwl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 69d28f9..75d38bb 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -157,8 +157,6 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: os.makedirs(bin_path, exist_ok=True) notebooks_paths = _get_notebook_paths_from_dir(r2d.repo) - print(5 * '-', 'NOTEBOOKS PATHS', notebooks_paths) - tools = [] for notebook in notebooks_paths: cwl_command_line_tool, script_name = _store_jn_as_script( From 52c44fcb36a0f60f3ada685131b86798b02ec771 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 19:01:42 +0100 Subject: [PATCH 20/26] support magic and system commands --- ipython2cwl/cwltoolextractor.py | 11 +++++ ipython2cwl/ipython2cwl.py | 10 ++-- ipython2cwl/repo2cwl.py | 35 ++++---------- setup.py | 3 +- tests/test_cwltool.py | 84 ++++++++++++++++++++++++++------- 5 files changed, 94 insertions(+), 49 deletions(-) diff --git a/ipython2cwl/cwltoolextractor.py b/ipython2cwl/cwltoolextractor.py index 7b03c88..b782ff2 100644 --- a/ipython2cwl/cwltoolextractor.py +++ b/ipython2cwl/cwltoolextractor.py @@ -9,7 +9,9 @@ from typing import Dict, Any import astor +import nbconvert import yaml +from nbformat.notebooknode import NotebookNode from .iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput from .requirements_manager import RequirementsManager @@ -123,6 +125,9 @@ class AnnotatedIPython2CWLToolConverter: """The annotated python code to convert.""" def __init__(self, annotated_ipython_code: str): + """Creates an AnnotatedIPython2CWLToolConverter. If the annotated_ipython_code contains magic commands use the + from_jupyter_notebook_node method""" + self._code = annotated_ipython_code extractor = AnnotatedVariablesExtractor() self._tree = ast.fix_missing_locations(extractor.visit(ast.parse(self._code))) @@ -139,6 +144,12 @@ def __init__(self, annotated_ipython_code: str): node.value.s) ) + @classmethod + def from_jupyter_notebook_node(cls, node: NotebookNode) -> 'AnnotatedIPython2CWLToolConverter': + python_exporter = nbconvert.PythonExporter() + code = python_exporter.from_notebook_node(node)[0] + return cls(code) + @classmethod def _wrap_script_to_method(cls, tree, variables) -> str: main_template_code = os.linesep.join([ diff --git a/ipython2cwl/ipython2cwl.py b/ipython2cwl/ipython2cwl.py index 8f9dfca..0d1d8ff 100644 --- a/ipython2cwl/ipython2cwl.py +++ b/ipython2cwl/ipython2cwl.py @@ -1,17 +1,19 @@ import argparse +import json +from io import StringIO from pathlib import Path from typing import List, Optional +import nbconvert import nbformat from .cwltoolextractor import AnnotatedIPython2CWLToolConverter def jn2code(notebook): - return '\n'.join( - [f"\n\n# --------- cell - {i} ---------\n\n{cell.source}" for i, cell in - enumerate(filter(lambda c: c.cell_type == 'code', notebook.cells), start=1)] - ) + exporter = nbconvert.PythonExporter() + script = exporter.from_file(StringIO(json.dumps(notebook))) + return script def main(argv: Optional[List[str]] = None): diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 75d38bb..e2e43e7 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -16,34 +16,10 @@ from repo2docker import Repo2Docker from .cwltoolextractor import AnnotatedIPython2CWLToolConverter -from .ipython2cwl import jn2code logger = logging.getLogger('repo2cwl') -# def main(argv: Optional[List[str]] = None): -# if argv is None: -# import sys -# argv = sys.argv -# parser = argparse.ArgumentParser() -# parser.add_argument('repo', type=urlparse, nargs=1) -# parser.add_argument('-o', '--output', type=Path, required=True) -# args = parser.parse_args(argv[1:]) -# -# notebook = nbformat.read(args.jn[0], as_version=4) -# output: Path = args.output -# args.jn[0].close() -# script_code = '\n'.join( -# [f"\n\n# --------- cell - {i} ---------\n\n{cell.source}" for i, cell in -# enumerate(filter(lambda c: c.cell_type == 'code', notebook.cells), start=1)] -# ) -# -# converter = AnnotatedIPython2CWLToolConverter(script_code) -# converter.compile(output) -# -# return 0 - - def _get_notebook_paths_from_dir(dir_path: str): notebooks_paths = [] for path, subdirs, files in os.walk(dir_path): @@ -56,9 +32,9 @@ def _get_notebook_paths_from_dir(dir_path: str): def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bin_absolute_path: str, image_id: str) \ -> Tuple[Optional[Dict], Optional[str]]: with open(notebook_path) as fd: - code = jn2code(nbformat.read(fd, as_version=4)) + notebook = nbformat.read(fd, as_version=4) - converter = AnnotatedIPython2CWLToolConverter(code) + converter = AnnotatedIPython2CWLToolConverter.from_jupyter_notebook_node(notebook) if len(converter._variables) == 0: logger.info(f"Notebook {notebook_path} does not contains typing annotations. skipping...") @@ -74,7 +50,12 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi else: script_absolute_name = os.path.join(bin_absolute_path, script_relative_path) script = os.linesep.join([ - '#!/usr/bin/env python', + '#!/usr/bin/env ipython', + '"""', + 'DO NOT EDIT THIS FILE', + 'THIS FILE IS AUTO-GENERATED BY THE ipython2cwl.', + 'FOR MORE INFORMATION CHECK https://github.com/giannisdoukas/ipython2cwl', + '"""', converter._wrap_script_to_method(converter._tree, converter._variables) ]) with open(script_absolute_name, 'w') as fd: diff --git a/setup.py b/setup.py index 880bdd8..9c786b3 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,8 @@ def get_version(rel_path): 'astor>=0.8.1', 'PyYAML>=5.3.1', 'gitpython>=3.1.3', - 'jupyter-repo2docker>=0.11.0' + 'jupyter-repo2docker>=0.11.0', + 'nbconvert>=5.6.1' ], test_suite='tests', ) diff --git a/tests/test_cwltool.py b/tests/test_cwltool.py index 9c3f5d8..9c71751 100644 --- a/tests/test_cwltool.py +++ b/tests/test_cwltool.py @@ -4,6 +4,8 @@ from pathlib import Path from unittest import TestCase +import nbformat + from ipython2cwl.cwltoolextractor import AnnotatedIPython2CWLToolConverter from ipython2cwl.iotypes import CWLStringInput, CWLFilePathOutput @@ -257,20 +259,68 @@ def test_AnnotatedIPython2CWLToolConverter_output_file_annotation(self): tool ) - # def test_AnnotatedIPython2CWLToolConverter_exclamation_mark_command(self): - # printed_message = '' - # annotated_python_script = os.linesep.join([ - # '!ls -la', - # 'global printed_message', - # f"msg: {CWLStringInput.__name__} = 'original'", - # "print('message:', msg)", - # "printed_message = msg" - # ]) - # exec(annotated_python_script) - # self.assertEqual('original', globals()['printed_message']) - # converter = AnnotatedIPython2CWLToolConverter(annotated_python_script) - # new_script = converter._wrap_script_to_method(converter._tree, converter._variables) - # print('\n' + new_script, '\n') - # exec(new_script) - # locals()['main']('new message') - # self.assertEqual('new message', globals()['printed_message']) + def test_AnnotatedIPython2CWLToolConverter_exclamation_mark_command(self): + printed_message = '' + annotated_python_jn_node = nbformat.from_dict( + { + "cells": [ + { + "cell_type": "code", + "execution_count": None, + "metadata": {}, + "outputs": [], + "source": os.linesep.join([ + "!ls -la\n", + "global printed_message\n", + "msg: CWLStringInput = 'original'\n", + "print('message:', msg)\n", + "printed_message = msg" + ]) + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 + }, + ) + converter = AnnotatedIPython2CWLToolConverter.from_jupyter_notebook_node(annotated_python_jn_node) + new_script = converter._wrap_script_to_method(converter._tree, converter._variables) + new_script_without_magics = os.linesep.join( + [line for line in new_script.splitlines() if not line.strip().startswith('get_ipython')] + ) + print('\n' + new_script, '\n') + exec(new_script_without_magics) + + annotated_python_jn_node.cells[0].source = os.linesep.join([ + '!ls -la', + 'global printed_message', + f'msg: {CWLStringInput.__name__} = """original\n!ls -la"""', + "print('message:', msg)", + "printed_message = msg" + ]) + converter = AnnotatedIPython2CWLToolConverter.from_jupyter_notebook_node(annotated_python_jn_node) + new_script = converter._wrap_script_to_method(converter._tree, converter._variables) + new_script_without_magics = os.linesep.join( + [line for line in new_script.splitlines() if not line.strip().startswith('get_ipython')]) + print('\n' + new_script, '\n') + exec(new_script_without_magics) + locals()['main']('original\n!ls -l') + self.assertEqual('original\n!ls -l', globals()['printed_message']) From a571fe04233ce1550d036438f47753b149f69674 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 19:44:37 +0100 Subject: [PATCH 21/26] fix issue with nbconvert --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9c786b3..9b8960c 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ def get_version(rel_path): 'PyYAML>=5.3.1', 'gitpython>=3.1.3', 'jupyter-repo2docker>=0.11.0', - 'nbconvert>=5.6.1' + 'nbconvert==5.6.1' ], test_suite='tests', ) From 1a10131156aa13e4124bce6eb76bb0794e6bb5bc Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 19:46:06 +0100 Subject: [PATCH 22/26] rm TODO --- ipython2cwl/repo2cwl.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index e2e43e7..b2b2695 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -68,7 +68,6 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi def existing_path(path: str): - # TODO: validate that the path exists and it is a directory path = Path(path) if not path.is_dir(): raise ValueError('Directory does not exists') From 86ae407947124b972c2afa58ac1d7065a462296a Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 20:00:50 +0100 Subject: [PATCH 23/26] add ipython as requirements --- .travis.yml | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b626821..8e9efa0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ before_install: install: - pip install -r test-requirements.txt - python setup.py install + - pip freeze script: - pycodestyle --max-line-length=119 $(find ipython2cwl -name '*.py') - coverage run --source ipython2cwl -m unittest discover tests diff --git a/setup.py b/setup.py index 9b8960c..69ae0f4 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,8 @@ def get_version(rel_path): 'PyYAML>=5.3.1', 'gitpython>=3.1.3', 'jupyter-repo2docker>=0.11.0', - 'nbconvert==5.6.1' + 'nbconvert==5.6.1', + 'ipython>=7.15.0' ], test_suite='tests', ) From c5cf43a4b5b3e3f0df52977b80102cbb01a51765 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 20:19:11 +0100 Subject: [PATCH 24/26] update doc --- docs/index.rst | 15 ++++++--------- docs/ipython2cwl.rst | 14 -------------- 2 files changed, 6 insertions(+), 23 deletions(-) delete mode 100644 docs/ipython2cwl.rst diff --git a/docs/index.rst b/docs/index.rst index 2e7d704..c470269 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,25 +16,22 @@ IPython2CWL is a tool for converting `IPython `_ Jupyter N from ipython2cwl.iotypes import CWLFilePathInput, CWLFilePathOutput import csv - input_filename: CWLFilePathInput = 'data.csv' + input_filename: 'CWLFilePathInput' = 'data.csv' with open(input_filename) as f: csv_reader = csv.reader(f) data = [line for line in csv_reader] number_of_lines = len(data) - result_file: CWLFilePathOutput = 'number_of_lines.txt' + result_file: 'CWLFilePathOutput' = 'number_of_lines.txt' with open(result_file, 'w') as f: f.write(str(number_of_lines)) ------------------------------------------------------------------------------------------ - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - - ipython2cwl +IPython2CWL is based on `repo2docker `_, the same tool +used by `mybinder `_. Now, by writing Jupyter Notebook and publish them, including repo2docker +configuration, the community can not only execute the notebooks remotely but also to use them as steps in scientific +workflows. Indices and tables ================== diff --git a/docs/ipython2cwl.rst b/docs/ipython2cwl.rst deleted file mode 100644 index cbd0496..0000000 --- a/docs/ipython2cwl.rst +++ /dev/null @@ -1,14 +0,0 @@ -IPython2CWL Module -=================== - -IPython2CWL -=============== -.. automodule:: ipython2cwl.ipython2cwl - :members: - :undoc-members: - -CWLTool -=============== -.. automodule:: ipython2cwl.cwltool - :members: - :undoc-members: From 8a1611b5c0ad218a00d31be5a0fb0b3e697a9315 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 20:53:03 +0100 Subject: [PATCH 25/26] fix micro issues on cloning from remote git --- ipython2cwl/repo2cwl.py | 7 ++++--- setup.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index b2b2695..8910042 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -100,16 +100,17 @@ def repo2cwl(argv: Optional[List[str]] = None): supported_schemes = {'file', 'http', 'https', 'ssh'} if uri.scheme not in supported_schemes: raise ValueError(f'Supported schema uris: {supported_schemes}') - local_git_directory = os.path.join(tempfile.mkdtemp(prefix='repo2cwl'), 'repo') + local_git_directory = os.path.join(tempfile.mkdtemp(prefix='repo2cwl_'), 'repo') if uri.scheme == 'file': if not os.path.isdir(uri.path): raise ValueError(f'Directory does not exists') logger.info(f'copy repo to temp directory: {local_git_directory}') shutil.copytree(uri.path, local_git_directory) + local_git = git.Repo(local_git_directory) else: logger.info(f'cloning repo to temp directory: {local_git_directory}') - git.Git(local_git_directory).clone(uri.geturl()) - local_git = git.Repo(local_git_directory) + local_git = git.Repo.clone_from(uri.geturl(), local_git_directory) + image_id, cwl_tools = _repo2cwl(local_git) logger.info(f'Generated image id: {image_id}') for tool in cwl_tools: diff --git a/setup.py b/setup.py index 69ae0f4..30f33ff 100644 --- a/setup.py +++ b/setup.py @@ -49,8 +49,8 @@ def get_version(rel_path): ], entry_points={ 'console_scripts': [ - 'jupyter-jn2cwl=ipython2cwl.ipython2cwl:main', - 'jupyter-jnrepo2cwl=ipython2cwl.repo2cwl:repo2cwl', + # 'jupyter-jn2cwl=ipython2cwl.ipython2cwl:main', + 'jupyter-repo2cwl=ipython2cwl.repo2cwl:repo2cwl', ], }, install_requires=[ From 716cc4d5cd40bb2b0c0fbe223d765d4d9ff50cd3 Mon Sep 17 00:00:00 2001 From: Giannis Doukas Date: Sat, 27 Jun 2020 20:53:16 +0100 Subject: [PATCH 26/26] update the docs --- docs/index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index c470269..b0be4a9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,6 +33,11 @@ used by `mybinder `_. Now, by writing Jupyter Notebook an configuration, the community can not only execute the notebooks remotely but also to use them as steps in scientific workflows. +* Install ipython2cwl +* Ensure that you have docker running +* Create a directory to store the generated cwl files, for example cwlbuild +* Execute :code:`jupyter repo2cwl https://github.com/giannisdoukas/cwl-annotated-jupyter-notebook.git -o cwlbuild` + Indices and tables ==================