feat(models): reference context for relative paths

(closes #452)

Author: jirikuncar

renku/api/datasets.py

@@ -56,8 +56,7 @@ def datasets(self):
         """Return mapping from path to dataset."""
         result = {}
         for path in self.renku_datasets_path.rglob(self.METADATA):
-            with path.open('r') as fp:
-                result[path] = Dataset.from_jsonld(yaml.load(fp))
+            result[path] = Dataset.from_yaml(path)
         return result
 
     @contextmanager
@@ -80,7 +79,7 @@ def with_dataset(self, name=None):
                 if path.exists():
                     with path.open('r') as f:
                         source = yaml.load(f) or {}
-                    dataset = Dataset.from_jsonld(source)
+                    dataset = Dataset.from_jsonld(source, __reference__=path)
 
             if dataset is None:
                 source = {}

renku/api/repository.py

@@ -256,10 +256,10 @@ def with_metadata(self):
             if self.renku_metadata_path.exists():
                 with metadata_path.open('r') as f:
                     source = yaml.load(f) or {}
-                metadata = Project.from_jsonld(source)
             else:
                 source = {}
-                metadata = Project()
+
+            metadata = Project.from_jsonld(source, __reference__=metadata_path)
 
             yield metadata
 

renku/cli/_format/graph.py

@@ -280,7 +280,6 @@ def color(p):
 
 def makefile(graph):
     """Format graph as Makefile."""
-    from renku._compat import Path
     from renku.models.provenance.activities import ProcessRun, WorkflowRun
 
     for activity in graph.activities.values():
@@ -293,14 +292,11 @@ def makefile(graph):
 
         for step in steps:
             click.echo(' '.join(step.outputs) + ': ' + ' '.join(step.inputs))
-
             tool = step.process
-            basedir = Path(step.path).parent
-
             click.echo(
                 '\t@' + ' '.join(tool.to_argv()) + ' ' + ' '.join(
                     tool.STD_STREAMS_REPR[key] + ' ' + str(path)
-                    for key, path in tool._std_streams(basedir=basedir).items()
+                    for key, path in tool._std_streams().items()
                 )
             )
 

renku/models/_jsonld.py

@@ -29,6 +29,7 @@
 from pyld import jsonld as ld
 
 from renku._compat import Path
+from renku.models._locals import ReferenceMixin, with_reference
 
 KEY = '__json_ld'
 KEY_CLS = '__json_ld_cls'
@@ -62,7 +63,10 @@ def wrap(cls):
         jsonld_cls = attr.s(cls, **attrs_kwargs)
 
         if not issubclass(jsonld_cls, JSONLDMixin):
-            jsonld_cls = make_type(cls.__name__, (jsonld_cls, JSONLDMixin), {})
+            jsonld_cls = attr.s(
+                make_type(cls.__name__, (jsonld_cls, JSONLDMixin), {}),
+                **attrs_kwargs
+            )
 
         # Merge types
         for subcls in jsonld_cls.mro():
@@ -291,13 +295,13 @@ def convert_value(v):
     return rv
 
 
-class JSONLDMixin(object):
+class JSONLDMixin(ReferenceMixin):
     """Mixin for loading a JSON-LD data."""
 
     __type_registry__ = {}
 
     @classmethod
-    def from_jsonld(cls, data):
+    def from_jsonld(cls, data, __reference__=None):
         """Instantiate a JSON-LD class from data."""
         if isinstance(data, cls):
             return data
@@ -328,10 +332,33 @@ def from_jsonld(cls, data):
         # assert compacted['@type'] == cls._jsonld_type, '@type must be equal'
         # TODO update self(not cls)._jsonld_context with data['@context']
         fields = cls._jsonld_fields
-        return cls(
-            **{k.lstrip('_'): v
-               for k, v in compacted.items() if k in fields}
-        )
+
+        if __reference__:
+            with with_reference(__reference__):
+                self = cls(
+                    **{
+                        k.lstrip('_'): v
+                        for k, v in compacted.items() if k in fields
+                    }
+                )
+        else:
+            self = cls(
+                **{
+                    k.lstrip('_'): v
+                    for k, v in compacted.items() if k in fields
+                }
+            )
+        return self
+
+    @classmethod
+    def from_yaml(cls, path):
+        """Return an instance from a YAML file."""
+        import yaml
+
+        with path.open(mode='r') as fp:
+            self = cls.from_jsonld(yaml.load(fp) or {}, __reference__=path)
+
+        return self
 
 
 s = attrs

renku/models/_locals.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2019 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Manage local contexts."""
+
+from contextlib import contextmanager
+
+import attr
+from werkzeug.local import LocalStack
+
+_current_reference = LocalStack()
+
+current_reference = _current_reference()
+
+
+@contextmanager
+def with_reference(path):
+    """Manage reference stack."""
+    _current_reference.push(path)
+    yield
+    if path != _current_reference.pop():
+        raise RuntimeError('current_reference has been modified')
+
+
+def has_reference():
+    """Check if the current reference is bounded."""
+    return _current_reference.top is not None
+
+
+@attr.s(cmp=False)
+class ReferenceMixin:
+    """Define an automatic ``__reference__`` attribute."""
+
+    __reference__ = attr.ib(init=False, kw_only=True, repr=False)
+
+    @__reference__.default
+    def default_reference(self):
+        """Create a default reference path."""
+        return current_reference._get_current_object() if has_reference(
+        ) else None

renku/models/cwl/_ascwl.py

@@ -25,6 +25,7 @@
 from attr._make import fields
 
 from renku._compat import Path
+from renku.models._locals import ReferenceMixin, with_reference
 
 
 class CWLType(type):
@@ -40,15 +41,34 @@ def __init__(cls, name, bases, namespace):
         cls.registry[name] = cls
 
 
-class CWLClass(object, metaclass=CWLType):
+class CWLClass(ReferenceMixin, metaclass=CWLType):
     """Include ``class`` field in serialized object."""
 
     @classmethod
-    def from_cwl(cls, data):
+    def from_cwl(cls, data, __reference__=None):
         """Return an instance from CWL data."""
         class_name = data.get('class', None)
         cls = cls.registry.get(class_name, cls)
-        return cls(**{k: v for k, v in iteritems(data) if k != 'class'})
+
+        if __reference__:
+            with with_reference(__reference__):
+                self = cls(
+                    **{k: v
+                       for k, v in iteritems(data) if k != 'class'}
+                )
+        else:
+            self = cls(**{k: v for k, v in iteritems(data) if k != 'class'})
+        return self
+
+    @classmethod
+    def from_yaml(cls, path):
+        """Return an instance from a YAML file."""
+        import yaml
+
+        with path.open(mode='r') as fp:
+            self = cls.from_cwl(yaml.load(fp), __reference__=path)
+
+        return self
 
 
 def mapped(cls, key='id', **kwargs):
@@ -114,6 +134,9 @@ def convert_value(v):
         return v
 
     for a in attrs:
+        if a.name.startswith('__'):
+            continue
+
         a_name = a.name.rstrip('_')
         v = getattr(inst, a.name)
         if filter is not None and not filter(a, v):

renku/models/cwl/command_line_tool.py

@@ -142,7 +142,7 @@ def to_argv(self, job=None):
             if i.inputBinding:
                 args.append((i.inputBinding.position, i))
 
-        for p, v in sorted(args):
+        for _, v in sorted(args):
             argv.extend(v.to_argv())
 
         return argv

renku/models/cwl/parameter.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright 2018 - Swiss Data Science Center (SDSC)
+# Copyright 2018-2019 - Swiss Data Science Center (SDSC)
 # A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
 # Eidgenössische Technische Hochschule Zürich (ETHZ).
 #
@@ -119,10 +119,11 @@ def from_cwl(cls, data):
             data = {'type': data}
         return cls(**data)
 
-    def to_argv(self):
+    def to_argv(self, **kwargs):
         """Format command input parameter as shell argument."""
-        return self.inputBinding.to_argv(default=self.default
-                                         ) if self.inputBinding else []
+        return self.inputBinding.to_argv(
+            default=self.default, **kwargs
+        ) if self.inputBinding else []
 
 
 @attr.s

renku/models/cwl/types.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright 2018 - Swiss Data Science Center (SDSC)
+# Copyright 2018-2019 - Swiss Data Science Center (SDSC)
 # A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
 # Eidgenössische Technische Hochschule Zürich (ETHZ).
 #
@@ -27,35 +27,34 @@
 from ._ascwl import CWLClass, ascwl
 
 
-@attr.s
-class File(CWLClass):
-    """Represent a file."""
-
-    path = attr.ib(converter=Path)
+class PathFormatterMixin:
+    """Format path property."""
 
     def __str__(self):
         """Simple conversion to string."""
-        # TODO refactor to use `basedir`
+        reference = self.__reference__
+        if reference:
+            return str(os.path.normpath(str(reference.parent / self.path)))
         return os.path.relpath(
             os.path.realpath(str(self.path)), os.path.realpath(os.getcwd())
         )
 
 
 @attr.s
-class Directory(CWLClass):
+class File(CWLClass, PathFormatterMixin):
+    """Represent a file."""
+
+    path = attr.ib(converter=Path)
+
+
+@attr.s
+class Directory(CWLClass, PathFormatterMixin):
     """Represent a directory."""
 
     # TODO add validation to allow only directories
     path = attr.ib(default=None)
     listing = attr.ib(default=attr.Factory(list))
 
-    def __str__(self):
-        """Simple conversion to string."""
-        # TODO refactor to use `basedir`
-        return os.path.relpath(
-            os.path.realpath(str(self.path)), os.path.realpath(os.getcwd())
-        )
-
 
 DIRECTORY_EXPRESSION = '$({0})'.format(
     json.dumps(ascwl(Directory(), filter=lambda _, x: x is not None))

renku/models/provenance/activities.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright 2018 - Swiss Data Science Center (SDSC)
+# Copyright 2018-2019 - Swiss Data Science Center (SDSC)
 # A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
 # Eidgenössische Technische Hochschule Zürich (ETHZ).
 #
@@ -21,9 +21,9 @@
 from collections import OrderedDict
 
 import attr
-import yaml
 from git import NULL_TREE
 
+from renku._compat import Path
 from renku.models import _jsonld as jsonld
 from renku.models.cwl import WORKFLOW_STEP_RUN_TYPES
 from renku.models.cwl._ascwl import CWLClass
@@ -408,8 +408,6 @@ class WorkflowRun(ProcessRun):
     @children.default
     def default_children(self):
         """Load children from process."""
-        import yaml
-
         basedir = os.path.dirname(self.path) if self.path is not None else None
 
         def _load(step):
@@ -418,12 +416,12 @@ def _load(step):
                 return step.run
 
             if self.commit:
+                import yaml
                 data = (self.commit.tree / basedir /
                         step.run).data_stream.read()
-            else:
-                with step.run.open('r') as f:
-                    data = f.read()
-            return CWLClass.from_cwl(yaml.load(data))
+                return CWLClass.from_cwl(yaml.load(data))
+
+            return CWLClass.from_yaml(step.run)
 
         return {step.id: _load(step) for step in self.process.steps}
 
@@ -593,8 +591,10 @@ def from_git_commit(commit, client, path=None):
                 path = file_
 
     if path:
+        import yaml
+
         data = (commit.tree / path).data_stream.read()
-        process = CWLClass.from_cwl(yaml.load(data))
+        process = CWLClass.from_cwl(yaml.load(data), __reference__=Path(path))
 
         return process.create_run(
             commit=commit,