diff --git a/.gitignore b/.gitignore
index 4af9dd9..de7ce12 100644
--- a/.gitignore
+++ b/.gitignore
@@ -243,3 +243,5 @@ cython_debug/
/external_examples/
/tests/jn/output/
tmp.py
+/html/
+cwlbuild
\ No newline at end of file
diff --git a/.readthedocs.yml b/.readthedocs.yml
new file mode 100644
index 0000000..7eb135b
--- /dev/null
+++ b/.readthedocs.yml
@@ -0,0 +1,2 @@
+python:
+ setup_py_install: true
diff --git a/.travis.yml b/.travis.yml
index 36a1576..8e9efa0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,3 +1,5 @@
+services:
+ - docker
language: python
python:
- "3.6"
@@ -10,6 +12,7 @@ before_install:
install:
- pip install -r test-requirements.txt
- python setup.py install
+ - pip freeze
script:
- pycodestyle --max-line-length=119 $(find ipython2cwl -name '*.py')
- coverage run --source ipython2cwl -m unittest discover tests
@@ -27,4 +30,5 @@ matrix:
- virtualenv -p python3 venv
- source venv/bin/activate
- pip3 install -U -r test-requirements.txt
- script: coverage run --source ipython2cwl -m unittest discover tests
\ No newline at end of file
+ script: coverage run --source ipython2cwl -m unittest discover tests
+ env: TRAVIS_IGNORE_DOCKER=true
\ No newline at end of file
diff --git a/README.md b/README.md
index b7c4ad1..d809e95 100644
--- a/README.md
+++ b/README.md
@@ -2,4 +2,5 @@
[](https://travis-ci.com/giannisdoukas/ipython2cwl)
[](https://coveralls.io/github/giannisdoukas/ipython2cwl?branch=dev)
+[](https://ipython2cwl.readthedocs.io/en/latest/?badge=latest)
diff --git a/docs/conf.py b/docs/conf.py
new file mode 100644
index 0000000..3ce62ae
--- /dev/null
+++ b/docs/conf.py
@@ -0,0 +1,53 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+# -- Project information -----------------------------------------------------
+
+project = 'ipython2cwl'
+copyright = '2020, Yannis Doukas'
+author = 'Yannis Doukas'
+
+# The full version, including alpha/beta/rc tags
+release = "0.1"
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.autodoc']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+master_doc = 'index'
\ No newline at end of file
diff --git a/docs/index.rst b/docs/index.rst
new file mode 100644
index 0000000..b0be4a9
--- /dev/null
+++ b/docs/index.rst
@@ -0,0 +1,46 @@
+IPython2CWL: Convert Jupyter Notebook to CWL
+================================================================================
+
+.. image:: https://travis-ci.com/giannisdoukas/ipython2cwl.svg?branch=master
+ :target: https://travis-ci.com/giannisdoukas/ipython2cwl
+.. image:: https://coveralls.io/repos/github/giannisdoukas/ipython2cwl/badge.svg?branch=master
+ :target: https://coveralls.io/github/giannisdoukas/ipython2cwl?branch=master
+
+
+------------------------------------------------------------------------------------------
+
+IPython2CWL is a tool for converting `IPython `_ Jupyter Notebooks to
+`CWL `_ Command Line Tools by simply providing typing annotation.
+
+.. code-block:: python
+
+ from ipython2cwl.iotypes import CWLFilePathInput, CWLFilePathOutput
+ import csv
+ input_filename: 'CWLFilePathInput' = 'data.csv'
+ with open(input_filename) as f:
+ csv_reader = csv.reader(f)
+ data = [line for line in csv_reader]
+ number_of_lines = len(data)
+ result_file: 'CWLFilePathOutput' = 'number_of_lines.txt'
+ with open(result_file, 'w') as f:
+ f.write(str(number_of_lines))
+
+
+------------------------------------------------------------------------------------------
+
+IPython2CWL is based on `repo2docker `_, the same tool
+used by `mybinder `_. Now, by writing Jupyter Notebook and publish them, including repo2docker
+configuration, the community can not only execute the notebooks remotely but also to use them as steps in scientific
+workflows.
+
+* Install ipython2cwl
+* Ensure that you have docker running
+* Create a directory to store the generated cwl files, for example cwlbuild
+* Execute :code:`jupyter repo2cwl https://github.com/giannisdoukas/cwl-annotated-jupyter-notebook.git -o cwlbuild`
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/ipython2cwl/cwltool.py b/ipython2cwl/cwltoolextractor.py
similarity index 90%
rename from ipython2cwl/cwltool.py
rename to ipython2cwl/cwltoolextractor.py
index a9a65b4..b782ff2 100644
--- a/ipython2cwl/cwltool.py
+++ b/ipython2cwl/cwltoolextractor.py
@@ -9,7 +9,9 @@
from typing import Dict, Any
import astor
+import nbconvert
import yaml
+from nbformat.notebooknode import NotebookNode
from .iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput
from .requirements_manager import RequirementsManager
@@ -51,7 +53,8 @@ def __init__(self, *args, **kwargs):
def visit_AnnAssign(self, node):
try:
- if isinstance(node.annotation, ast.Name) and node.annotation.id in self.input_type_mapper:
+ if (isinstance(node.annotation, ast.Name) and node.annotation.id in self.input_type_mapper) or \
+ (isinstance(node.annotation, ast.Str) and node.annotation.s in self.input_type_mapper):
mapper = self.input_type_mapper[node.annotation.id]
self.extracted_nodes.append(
(node, mapper[0], mapper[1], True, True, False)
@@ -72,7 +75,8 @@ def visit_AnnAssign(self, node):
(node, mapper[0] + '[]', mapper[1], True, True, False)
)
return None
- elif isinstance(node.annotation, ast.Name) and node.annotation.id in self.output_type_mapper:
+ elif (isinstance(node.annotation, ast.Name) and node.annotation.id in self.output_type_mapper) or \
+ (isinstance(node.annotation, ast.Str) and node.annotation.s in self.output_type_mapper):
self.extracted_nodes.append(
(node, None, None, None, False, True)
)
@@ -121,6 +125,9 @@ class AnnotatedIPython2CWLToolConverter:
"""The annotated python code to convert."""
def __init__(self, annotated_ipython_code: str):
+ """Creates an AnnotatedIPython2CWLToolConverter. If the annotated_ipython_code contains magic commands use the
+ from_jupyter_notebook_node method"""
+
self._code = annotated_ipython_code
extractor = AnnotatedVariablesExtractor()
self._tree = ast.fix_missing_locations(extractor.visit(ast.parse(self._code)))
@@ -137,6 +144,12 @@ def __init__(self, annotated_ipython_code: str):
node.value.s)
)
+ @classmethod
+ def from_jupyter_notebook_node(cls, node: NotebookNode) -> 'AnnotatedIPython2CWLToolConverter':
+ python_exporter = nbconvert.PythonExporter()
+ code = python_exporter.from_notebook_node(node)[0]
+ return cls(code)
+
@classmethod
def _wrap_script_to_method(cls, tree, variables) -> str:
main_template_code = os.linesep.join([
diff --git a/ipython2cwl/ipython2cwl.py b/ipython2cwl/ipython2cwl.py
index c7bad11..0d1d8ff 100644
--- a/ipython2cwl/ipython2cwl.py
+++ b/ipython2cwl/ipython2cwl.py
@@ -1,9 +1,19 @@
import argparse
+import json
+from io import StringIO
from pathlib import Path
from typing import List, Optional
+import nbconvert
import nbformat
-from .cwltool import AnnotatedIPython2CWLToolConverter
+
+from .cwltoolextractor import AnnotatedIPython2CWLToolConverter
+
+
+def jn2code(notebook):
+ exporter = nbconvert.PythonExporter()
+ script = exporter.from_file(StringIO(json.dumps(notebook)))
+ return script
def main(argv: Optional[List[str]] = None):
@@ -17,10 +27,7 @@ def main(argv: Optional[List[str]] = None):
notebook = nbformat.read(args.jn[0], as_version=4)
output: Path = args.output
args.jn[0].close()
- script_code = '\n'.join(
- [f"\n\n# --------- cell - {i} ---------\n\n{cell.source}" for i, cell in
- enumerate(filter(lambda c: c.cell_type == 'code', notebook.cells), start=1)]
- )
+ script_code = jn2code(notebook)
converter = AnnotatedIPython2CWLToolConverter(script_code)
converter.compile(output)
diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py
new file mode 100644
index 0000000..8910042
--- /dev/null
+++ b/ipython2cwl/repo2cwl.py
@@ -0,0 +1,163 @@
+import argparse
+import logging
+import os
+import shutil
+import stat
+import sys
+import tempfile
+from pathlib import Path
+from typing import List, Optional, Tuple, Dict
+from urllib.parse import urlparse, ParseResult
+
+import git
+import nbformat
+import yaml
+from git import Repo
+from repo2docker import Repo2Docker
+
+from .cwltoolextractor import AnnotatedIPython2CWLToolConverter
+
+logger = logging.getLogger('repo2cwl')
+
+
+def _get_notebook_paths_from_dir(dir_path: str):
+ notebooks_paths = []
+ for path, subdirs, files in os.walk(dir_path):
+ for name in files:
+ if name.endswith('.ipynb'):
+ notebooks_paths.append(os.path.join(path, name))
+ return notebooks_paths
+
+
+def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bin_absolute_path: str, image_id: str) \
+ -> Tuple[Optional[Dict], Optional[str]]:
+ with open(notebook_path) as fd:
+ notebook = nbformat.read(fd, as_version=4)
+
+ converter = AnnotatedIPython2CWLToolConverter.from_jupyter_notebook_node(notebook)
+
+ if len(converter._variables) == 0:
+ logger.info(f"Notebook {notebook_path} does not contains typing annotations. skipping...")
+ return None, None
+ script_relative_path = os.path.relpath(notebook_path, git_directory_absolute_path)[:-6]
+ script_relative_parent_directories = script_relative_path.split(os.sep)
+ if len(script_relative_parent_directories) > 1:
+ script_absolute_name = os.path.join(bin_absolute_path, os.sep.join(script_relative_parent_directories[:-1]))
+ os.makedirs(
+ script_absolute_name,
+ exist_ok=True)
+ script_absolute_name = os.path.join(script_absolute_name, os.path.basename(script_relative_path))
+ else:
+ script_absolute_name = os.path.join(bin_absolute_path, script_relative_path)
+ script = os.linesep.join([
+ '#!/usr/bin/env ipython',
+ '"""',
+ 'DO NOT EDIT THIS FILE',
+ 'THIS FILE IS AUTO-GENERATED BY THE ipython2cwl.',
+ 'FOR MORE INFORMATION CHECK https://github.com/giannisdoukas/ipython2cwl',
+ '"""',
+ converter._wrap_script_to_method(converter._tree, converter._variables)
+ ])
+ with open(script_absolute_name, 'w') as fd:
+ fd.write(script)
+ tool = converter.cwl_command_line_tool(image_id)
+ in_git_dir_script_file = os.path.join(bin_absolute_path, script_relative_path)
+ tool_st = os.stat(in_git_dir_script_file)
+ os.chmod(in_git_dir_script_file, tool_st.st_mode | stat.S_IEXEC)
+ return tool, script_relative_path
+
+
+def existing_path(path: str):
+ path = Path(path)
+ if not path.is_dir():
+ raise ValueError('Directory does not exists')
+ return path
+
+
+def parser_arguments(argv: List[str]):
+ parser = argparse.ArgumentParser()
+ parser.add_argument('repo', type=lambda uri: urlparse(uri, scheme='file'), nargs=1)
+ parser.add_argument('-o', '--output', help='Output directory to store the generated cwl files',
+ type=existing_path,
+ required=True)
+ return parser.parse_args(argv[1:])
+
+
+def setup_logger():
+ handler = logging.StreamHandler(sys.stdout)
+ handler.setLevel(logging.INFO)
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
+
+
+def repo2cwl(argv: Optional[List[str]] = None):
+ setup_logger()
+ argv = sys.argv if argv is None else argv
+ args = parser_arguments(argv)
+ uri: ParseResult = args.repo[0]
+ output_directory: Path = args.output
+ supported_schemes = {'file', 'http', 'https', 'ssh'}
+ if uri.scheme not in supported_schemes:
+ raise ValueError(f'Supported schema uris: {supported_schemes}')
+ local_git_directory = os.path.join(tempfile.mkdtemp(prefix='repo2cwl_'), 'repo')
+ if uri.scheme == 'file':
+ if not os.path.isdir(uri.path):
+ raise ValueError(f'Directory does not exists')
+ logger.info(f'copy repo to temp directory: {local_git_directory}')
+ shutil.copytree(uri.path, local_git_directory)
+ local_git = git.Repo(local_git_directory)
+ else:
+ logger.info(f'cloning repo to temp directory: {local_git_directory}')
+ local_git = git.Repo.clone_from(uri.geturl(), local_git_directory)
+
+ image_id, cwl_tools = _repo2cwl(local_git)
+ logger.info(f'Generated image id: {image_id}')
+ for tool in cwl_tools:
+ base_command_script_name = f'{tool["baseCommand"][len("/app/cwl/bin/"):].replace("/", "_")}.cwl'
+ tool_filename = str(output_directory.joinpath(base_command_script_name))
+ with open(tool_filename, 'w') as f:
+ logger.info(f'Creating CWL command line tool: {tool_filename}')
+ yaml.safe_dump(tool, f)
+
+ logger.info(f'Cleaning local temporary directory {local_git_directory}...')
+ shutil.rmtree(local_git_directory)
+
+
+def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]:
+ """
+ Takes a Repo mounted to a local directory. That function will create new files and it will commit the changes.
+ Do not use that function for Repositories you do not want to change them.
+ :param git_directory_path:
+ :return: The generated build image id & the cwl description
+ """
+ r2d = Repo2Docker()
+ r2d.target_repo_dir = os.path.join(os.path.sep, 'app')
+ r2d.repo = git_directory_path.tree().abspath
+ bin_path = os.path.join(r2d.repo, 'cwl', 'bin')
+ os.makedirs(bin_path, exist_ok=True)
+ notebooks_paths = _get_notebook_paths_from_dir(r2d.repo)
+
+ tools = []
+ for notebook in notebooks_paths:
+ cwl_command_line_tool, script_name = _store_jn_as_script(
+ notebook,
+ git_directory_path.tree().abspath,
+ bin_path,
+ r2d.output_image_spec
+ )
+ if cwl_command_line_tool is None:
+ continue
+ cwl_command_line_tool['baseCommand'] = os.path.join('/app', 'cwl', 'bin', script_name)
+ tools.append(cwl_command_line_tool)
+ git_directory_path.index.commit("auto-commit")
+
+ r2d.build()
+ # fix dockerImageId
+ for cwl_command_line_tool in tools:
+ cwl_command_line_tool['hints']['DockerRequirement']['dockerImageId'] = r2d.output_image_spec
+ return r2d.output_image_spec, tools
+
+
+if __name__ == '__main__':
+ repo2cwl()
diff --git a/setup.py b/setup.py
index 8ac2afa..30f33ff 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,6 @@
import os.path
from setuptools import setup
-from setuptools import find_packages
name = 'ipython2cwl'
@@ -50,13 +49,18 @@ def get_version(rel_path):
],
entry_points={
'console_scripts': [
- 'jupyter-jn2cwl=ipython2cwl.ipython2cwl:main',
+ # 'jupyter-jn2cwl=ipython2cwl.ipython2cwl:main',
+ 'jupyter-repo2cwl=ipython2cwl.repo2cwl:repo2cwl',
],
},
install_requires=[
'nbformat>=5.0.6',
'astor>=0.8.1',
- 'PyYAML>=5.3.1'
+ 'PyYAML>=5.3.1',
+ 'gitpython>=3.1.3',
+ 'jupyter-repo2docker>=0.11.0',
+ 'nbconvert==5.6.1',
+ 'ipython>=7.15.0'
],
test_suite='tests',
)
diff --git a/test-requirements.txt b/test-requirements.txt
index e4d5a20..15475e2 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,4 +1,6 @@
pycodestyle>=2.6.0
coverage>=5.1
coveralls>=2.0.0
-virtualenv>=3.1.0
\ No newline at end of file
+virtualenv>=3.1.0
+gitpython>=3.1.3
+docker>=4.2.1
diff --git a/tests/non-annotated.ipynb b/tests/non-annotated.ipynb
new file mode 100644
index 0000000..cac765d
--- /dev/null
+++ b/tests/non-annotated.ipynb
@@ -0,0 +1,72 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import matplotlib\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dataset = 'example.csv'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data = pd.read_csv(dataset)\n",
+ "# original data\n",
+ "fig = data.plot()\n",
+ "\n",
+ "original_image = 'original_data.png'\n",
+ "fig.figure.savefig(original_image)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# transform data\n",
+ "data.sort_values(by='Random B', ascending=False, inplace=True, ignore_index=True)\n",
+ "fig = data.plot()\n",
+ "\n",
+ "after_transform_data = 'new_data.png'\n",
+ "fig.figure.savefig(after_transform_data)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/tests/simple.ipynb b/tests/simple.ipynb
new file mode 100644
index 0000000..a9d7b61
--- /dev/null
+++ b/tests/simple.ipynb
@@ -0,0 +1,73 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import matplotlib\n",
+ "from ipython2cwl.iotypes import CWLFilePathInput, CWLFilePathOutput"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dataset: CWLFilePathInput = 'example.csv'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data = pd.read_csv(dataset)\n",
+ "# original data\n",
+ "fig = data.plot()\n",
+ "\n",
+ "original_image: CWLFilePathOutput = 'original_data.png'\n",
+ "fig.figure.savefig(original_image)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# transform data\n",
+ "data.sort_values(by='Random B', ascending=False, inplace=True, ignore_index=True)\n",
+ "fig = data.plot()\n",
+ "\n",
+ "after_transform_data: CWLFilePathOutput = 'new_data.png'\n",
+ "fig.figure.savefig(after_transform_data)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}
diff --git a/tests/test_cwltool.py b/tests/test_cwltool.py
index c4e9bd1..9c71751 100644
--- a/tests/test_cwltool.py
+++ b/tests/test_cwltool.py
@@ -4,7 +4,9 @@
from pathlib import Path
from unittest import TestCase
-from ipython2cwl.cwltool import AnnotatedIPython2CWLToolConverter
+import nbformat
+
+from ipython2cwl.cwltoolextractor import AnnotatedIPython2CWLToolConverter
from ipython2cwl.iotypes import CWLStringInput, CWLFilePathOutput
@@ -256,3 +258,69 @@ def test_AnnotatedIPython2CWLToolConverter_output_file_annotation(self):
},
tool
)
+
+ def test_AnnotatedIPython2CWLToolConverter_exclamation_mark_command(self):
+ printed_message = ''
+ annotated_python_jn_node = nbformat.from_dict(
+ {
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": None,
+ "metadata": {},
+ "outputs": [],
+ "source": os.linesep.join([
+ "!ls -la\n",
+ "global printed_message\n",
+ "msg: CWLStringInput = 'original'\n",
+ "print('message:', msg)\n",
+ "printed_message = msg"
+ ])
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+ },
+ )
+ converter = AnnotatedIPython2CWLToolConverter.from_jupyter_notebook_node(annotated_python_jn_node)
+ new_script = converter._wrap_script_to_method(converter._tree, converter._variables)
+ new_script_without_magics = os.linesep.join(
+ [line for line in new_script.splitlines() if not line.strip().startswith('get_ipython')]
+ )
+ print('\n' + new_script, '\n')
+ exec(new_script_without_magics)
+
+ annotated_python_jn_node.cells[0].source = os.linesep.join([
+ '!ls -la',
+ 'global printed_message',
+ f'msg: {CWLStringInput.__name__} = """original\n!ls -la"""',
+ "print('message:', msg)",
+ "printed_message = msg"
+ ])
+ converter = AnnotatedIPython2CWLToolConverter.from_jupyter_notebook_node(annotated_python_jn_node)
+ new_script = converter._wrap_script_to_method(converter._tree, converter._variables)
+ new_script_without_magics = os.linesep.join(
+ [line for line in new_script.splitlines() if not line.strip().startswith('get_ipython')])
+ print('\n' + new_script, '\n')
+ exec(new_script_without_magics)
+ locals()['main']('original\n!ls -l')
+ self.assertEqual('original\n!ls -l', globals()['printed_message'])
diff --git a/tests/test_ipython2cwl_from_repo.py b/tests/test_ipython2cwl_from_repo.py
new file mode 100644
index 0000000..a0cef3b
--- /dev/null
+++ b/tests/test_ipython2cwl_from_repo.py
@@ -0,0 +1,106 @@
+import os
+import shutil
+import tempfile
+from io import StringIO
+from unittest import TestCase, skipIf
+
+import docker
+import yaml
+from git import Repo
+
+from ipython2cwl.repo2cwl import _repo2cwl
+
+
+class Test2CWLFromRepo(TestCase):
+ maxDiff = None
+ here = os.path.abspath(os.path.dirname(__file__))
+
+ @skipIf("TRAVIS_IGNORE_DOCKER" in os.environ and os.environ["TRAVIS_IGNORE_DOCKER"] == "true",
+ "Skipping this test on Travis CI.")
+ def test_docker_build(self):
+ # setup a simple git repo
+ git_dir = tempfile.mkdtemp()
+ jn_repo = Repo.init(git_dir)
+ shutil.copy(
+ os.path.join(self.here, 'simple.ipynb'),
+ os.path.join(git_dir, 'simple.ipynb'),
+ )
+ with open(os.path.join(git_dir, 'requirements.txt'), 'w') as f:
+ f.write('pandas\n')
+ f.write('matplotlib\n')
+ jn_repo.index.add('simple.ipynb')
+ jn_repo.index.add('requirements.txt')
+ jn_repo.index.commit("initial commit")
+
+ print(git_dir)
+
+ dockerfile_image_id, cwl_tool = _repo2cwl(jn_repo)
+ self.assertEqual(1, len(cwl_tool))
+ docker_client = docker.from_env()
+ script = docker_client.containers.run(dockerfile_image_id, '/app/cwl/bin/simple', entrypoint='/bin/cat')
+ self.assertIn('fig.figure.savefig(after_transform_data)', script.decode())
+ self.assertDictEqual(
+ {
+ 'cwlVersion': "v1.1",
+ 'class': 'CommandLineTool',
+ 'baseCommand': '/app/cwl/bin/simple',
+ 'hints': {
+ 'DockerRequirement': {'dockerImageId': dockerfile_image_id}
+ },
+ 'inputs': {
+ 'dataset': {
+ 'type': 'File',
+ 'inputBinding': {
+ 'prefix': '--dataset'
+ }
+ }
+ },
+ 'outputs': {
+ 'original_image': {
+ 'type': 'File',
+ 'outputBinding': {
+ 'glob': 'original_data.png'
+ }
+ },
+ 'after_transform_data': {
+ 'type': 'File',
+ 'outputBinding': {
+ 'glob': 'new_data.png'
+ }
+ }
+ },
+ },
+ cwl_tool[0]
+ )
+ cwl = StringIO()
+ yaml.safe_dump(cwl_tool[0], cwl)
+ cwl_code = cwl.getvalue()
+ print(cwl_code)
+
+ # test for non-annotated jn
+ shutil.copy(
+ os.path.join(self.here, 'non-annotated.ipynb'),
+ os.path.join(git_dir, 'non-annotated.ipynb'),
+ )
+ jn_repo.index.add("non-annotated.ipynb")
+ jn_repo.index.commit("add non annotated notebook")
+ dockerfile_image_id, new_cwl_tool = _repo2cwl(jn_repo)
+ self.assertEqual(1, len(new_cwl_tool))
+ cwl_tool[0]['hints']['DockerRequirement'].pop('dockerImageId')
+ new_cwl_tool[0]['hints']['DockerRequirement'].pop('dockerImageId')
+ self.assertListEqual(cwl_tool, new_cwl_tool)
+
+ # test with jn with same name in different directory
+ os.makedirs(os.path.join(git_dir, 'subdir'), exist_ok=True)
+ shutil.copy(
+ os.path.join(self.here, 'simple.ipynb'),
+ os.path.join(git_dir, 'subdir', 'simple.ipynb'),
+ )
+ jn_repo.index.add("subdir/simple.ipynb")
+ jn_repo.index.commit("add second jn with the same name")
+ dockerfile_image_id, new_cwl_tool = _repo2cwl(jn_repo)
+ base_commands = [tool['baseCommand'] for tool in new_cwl_tool]
+ base_commands.sort()
+ self.assertListEqual(base_commands, ['/app/cwl/bin/simple', '/app/cwl/bin/subdir/simple'])
+ script = docker_client.containers.run(dockerfile_image_id, '/app/cwl/bin/subdir/simple', entrypoint='/bin/cat')
+ self.assertIn('fig.figure.savefig(after_transform_data)', script.decode())