cwl: support for initial work dir requirement

(closes #228)

renku/models/_datastructures.py

@@ -17,6 +17,11 @@
 # limitations under the License.
 """Base classes for Model objects used in Python SDK."""
 
+import os
+from collections import deque
+
+from renku._compat import Path
+
 
 class Model(object):
     """Abstract response of a single object."""
@@ -83,3 +88,46 @@ def __getitem__(self, key):
                 self._called = True
                 return dict.__getitem__(self, key)
             raise
+
+
+class DirectoryTree(dict):
+    r"""Create a safe directory tree from paths.
+
+    Example usage:
+
+    >>> directory = DirectoryTree()
+    >>> directory.add('foo/bar/baz')
+    >>> directory.add('foo/bar/bay')
+    >>> directory.add('foo/fooo/foooo')
+    >>> print('\n'.join(sorted(directory)))
+    foo/bar
+    foo/fooo
+
+    """
+
+    def add(self, value):
+        """Create a safe directory from a value."""
+        path = Path(str(value))
+        directory = path.parent
+        if directory and directory != directory.parent:
+            destination = self
+            for part in directory.parts:
+                destination = destination.setdefault(part, {})
+
+    def __iter__(self):
+        """Yield all stored directories."""
+        filter = {
+            os.path.sep,
+        }
+        queue = deque()
+        queue.append((self, []))
+
+        while queue:
+            data, parents = queue.popleft()
+            for key, value in dict.items(data):
+                if key in filter:
+                    continue
+                if value:
+                    queue.append((value, parents + [key]))
+                else:
+                    yield os.path.sep.join(parents + [key])

renku/models/cwl/command_line_tool.py

@@ -209,6 +209,18 @@ def watch(self, repo=None, no_output=False):
 
             tool.inputs = list(inputs.values())
             tool.outputs = outputs
+
+            from .process_requirements import InitialWorkDirRequirement, \
+                InlineJavascriptRequirement
+            initial_work_dir_requirement = InitialWorkDirRequirement.from_tool(
+                tool
+            )
+            if initial_work_dir_requirement:
+                tool.requirements.extend([
+                    InlineJavascriptRequirement(),
+                    initial_work_dir_requirement,
+                ])
+
             repo.track_paths_in_storage(*paths)
 
     @command_line.validator

renku/models/cwl/process_requirements.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2018 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Process requirements modify the semantics or runtime environment."""
+
+import attr
+
+from .._datastructures import DirectoryTree
+from ._ascwl import CWLClass
+from .types import DIRECTORY_EXPRESSION, Dirent
+
+
+class ProcessRequirement(object):
+    """Declare a prerequisite that may or must be fulfilled."""
+
+
+@attr.s
+class InlineJavascriptRequirement(CWLClass):
+    """Indicate that runner must support inline Javascript expressions."""
+
+
+@attr.s
+class InitialWorkDirRequirement(ProcessRequirement, CWLClass):
+    """Define a list of files and subdirectories that must be created."""
+
+    listing = attr.ib(default=attr.Factory(list))  # File, Directory
+
+    @classmethod
+    def from_tool(cls, tool):
+        """Create a directory structure based on tool inputs and outputs."""
+        directories = DirectoryTree()
+        inputs = {input_.id: input_ for input_ in tool.inputs}
+
+        # TODO enable for extra tool inputs when there is no inputBinding
+        # for input_ in tool.inputs:
+        #     # NOTE use with CWL 1.1
+        #     # if intput_.type == 'stdin':
+        #     #     stream = getattr(tool, input_.type)
+        #     #     directories[stream]
+        #     if input_.type == 'File':
+        #         directories.add(input_.default.path)
+        #     # TODO add Directory
+
+        for output in tool.outputs:
+            # NOTE output streams should be handled automatically
+            # if output.type in {'stdout', 'stderr'}:
+            #     stream = getattr(tool, output.type)
+            #     directories.add(stream)
+            if output.type == 'File':
+                glob = output.outputBinding.glob
+                # TODO better support for Expression
+                if glob.startswith('$(inputs.'):
+                    input_id = glob[len('$(inputs.'):-1]
+                    input_ = inputs.get(input_id)
+                    if input_ is not None:
+                        directories.add(input_.default)
+                elif glob:
+                    directories.add(glob)
+            # TODO add Directory
+
+        requirement = cls()
+
+        for directory in directories:
+            requirement.listing.append(
+                Dirent(
+                    entryname=directory,
+                    entry=DIRECTORY_EXPRESSION,
+                    writable=True,
+                )
+            )
+
+        if requirement.listing:
+            return requirement

renku/models/cwl/types.py

@@ -17,13 +17,14 @@
 # limitations under the License.
 """Represent the Common Workflow Language types."""
 
+import json
 import os
 
 import attr
 
 from renku._compat import Path
 
-from ._ascwl import CWLClass
+from ._ascwl import CWLClass, ascwl
 
 
 @attr.s
@@ -38,3 +39,33 @@ def __str__(self):
         return os.path.relpath(
             os.path.realpath(str(self.path)), os.path.realpath(os.getcwd())
         )
+
+
+@attr.s
+class Directory(CWLClass):
+    """Represent a directory."""
+
+    # TODO add validation to allow only directories
+    path = attr.ib(default=None)
+    listing = attr.ib(default=attr.Factory(list))
+
+    def __str__(self):
+        """Simple conversion to string."""
+        # TODO refactor to use `basedir`
+        return os.path.relpath(
+            os.path.realpath(str(self.path)), os.path.realpath(os.getcwd())
+        )
+
+
+DIRECTORY_EXPRESSION = '$({0})'.format(
+    json.dumps(ascwl(Directory(), filter=lambda _, x: x is not None))
+)
+
+
+@attr.s
+class Dirent(object):
+    """Define a file or subdirectory."""
+
+    entryname = attr.ib(default=None)
+    entry = attr.ib(default=None)
+    writable = attr.ib(default=False)

tests/test_cli.py

@@ -603,11 +603,11 @@ def test_status_with_submodules(base_runner):
 def test_unchanged_output(runner):
     """Test detection of unchanged output."""
     cmd = ['run', 'touch', '1']
-    result = runner.invoke(cli.cli, cmd)
+    result = runner.invoke(cli.cli, cmd, catch_exceptions=False)
     assert result.exit_code == 0
 
     cmd = ['run', 'touch', '1']
-    result = runner.invoke(cli.cli, cmd)
+    result = runner.invoke(cli.cli, cmd, catch_exceptions=False)
     assert result.exit_code == 1
 
 
@@ -643,7 +643,9 @@ def test_modified_output(project, runner, capsys):
     """Test detection of changed file as output."""
     cwd = Path(project)
     source = cwd / 'source.txt'
-    output = cwd / 'result.txt'
+    data = cwd / 'data' / 'results'
+    data.mkdir(parents=True)
+    output = data / 'result.txt'
 
     repo = git.Repo(project)
     cmd = ['run', 'cp', '-r', str(source), str(output)]