Merge branch '2131-workflow-visualize' of github.com:SwissDataScience…

…Center/renku-python into 2131-workflow-visualize

.github/workflows/test_deploy.yml

@@ -291,6 +291,7 @@ jobs:
     - name: Install dependencies
       run: |
         brew update || true
+        brew unlink git-lfs || true
         curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
         brew install shellcheck node || brew link --overwrite node
         python -m pip install --upgrade pip
@@ -323,6 +324,7 @@ jobs:
     - name: Install dependencies
       run: |
         brew update || true
+        brew unlink git-lfs || true
         curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
         brew install shellcheck node || brew link --overwrite node
         python -m pip install --upgrade pip
@@ -361,6 +363,7 @@ jobs:
       - name: Install dependencies
         run: |
           brew update || true
+          brew unlink git-lfs || true
           curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
           brew install shellcheck node || brew link --overwrite node
           python -m pip install --upgrade pip
@@ -399,6 +402,7 @@ jobs:
       - name: Install dependencies
         run: |
           brew update || true
+          brew unlink git-lfs || true
           curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
           brew install shellcheck node || brew link --overwrite node
           python -m pip install --upgrade pip
@@ -487,6 +491,7 @@ jobs:
     - name: Install dependencies
       run: |
         brew update || true
+        brew unlink git-lfs || true
         curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
         brew install shellcheck node || brew link --overwrite node
         python -m pip install --upgrade pip

renku/core/management/datasets.py

@@ -65,7 +65,6 @@
 from renku.core.utils import communication
 from renku.core.utils.git import (
     add_to_git,
-    find_previous_commit,
     get_oauth_url,
     get_object_hash,
     get_renku_repo_url,
@@ -928,14 +927,8 @@ def update_dataset_git_files(self, files: List[DynamicProxy], ref, delete=False)
 
                 checksum = get_object_hash(repo, based_on.path)
 
-                # NOTE: Use commit sha to compare if available for compatibility
-                if based_on.commit_sha is not None:
-                    remote_commit = find_previous_commit(repo, based_on.path)
-                    found = bool(remote_commit)
-                    changed = found and based_on.commit_sha != remote_commit.hexsha
-                else:
-                    found = bool(checksum)
-                    changed = found and based_on.checksum != checksum
+                found = bool(checksum)
+                changed = found and based_on.checksum != checksum
 
                 src = Path(repo.working_dir) / based_on.path
                 dst = self.renku_path.parent / file.entity.path

renku/core/management/migrations/utils/conversion.py

@@ -62,7 +62,7 @@ def _create_remote_entity(dataset_file: Optional[old_datasets.DatasetFile]) -> O
     if not dataset_file:
         return
     commit_sha = dataset_file._label.rsplit("@", maxsplit=1)[-1]
-    return RemoteEntity(commit_sha=commit_sha, path=dataset_file.path, url=dataset_file.url)
+    return RemoteEntity(checksum=commit_sha, path=dataset_file.path, url=dataset_file.url)
 
 
 def _convert_dataset_file(dataset_file: old_datasets.DatasetFile, client, revision: str) -> Optional[DatasetFile]:

renku/core/models/dataset.py

@@ -19,6 +19,7 @@
 
 import copy
 import os
+import posixpath
 from datetime import datetime
 from pathlib import Path
 from typing import Dict, List, Optional, Union
@@ -182,38 +183,32 @@ def is_absolute(self):
 class RemoteEntity(Slots):
     """Reference to an Entity in a remote repo."""
 
-    __slots__ = ("checksum", "commit_sha", "id", "path", "url")
+    __slots__ = ("checksum", "id", "path", "url")
 
-    def __init__(
-        self, *, checksum: str = None, commit_sha: str = None, id: str = None, path: Union[Path, str], url: str
-    ):
+    def __init__(self, *, checksum: str, id: str = None, path: Union[Path, str], url: str):
         super().__init__()
-        entity_hash = commit_sha or checksum
-        assert entity_hash, "One of checksum or commit_sha must be provided."
-        assert checksum is None or commit_sha is None, "Either checksum or commit_sha must be provided, not both."
-
-        # NOTE: For compatibility we use commit_sha when migrating old projects. For all new instances use checksum.
-        self.commit_sha: str = commit_sha
         self.checksum: str = checksum
-        self.id: str = id or RemoteEntity.generate_id(entity_hash, path)
+        self.id: str = id or RemoteEntity.generate_id(checksum=checksum, path=path, url=url)
         self.path: str = str(path)
         self.url: str = url
 
     @staticmethod
-    def generate_id(commit_sha: str, path: Union[Path, str]) -> str:
+    def generate_id(checksum: str, path: Union[Path, str], url: str) -> str:
         """Generate an id."""
+        parsed_url = urlparse(url)
+        prefix = quote(posixpath.join(parsed_url.netloc, parsed_url.path))
         path = quote(str(path))
-        return f"/remote-entity/{commit_sha}/{path}"
+        return f"/remote-entity/{prefix}/{checksum}/{path}"
 
     def __eq__(self, other):
         if self is other:
             return True
         if not isinstance(other, RemoteEntity):
             return False
-        return self.commit_sha == other.commit_sha and self.path == other.path and self.url == other.url
+        return self.checksum == other.checksum and self.path == other.path and self.url == other.url
 
     def __hash__(self):
-        return hash((self.commit_sha, self.path, self.url))
+        return hash((self.checksum, self.path, self.url))
 
 
 class DatasetFile(Slots):
@@ -661,11 +656,11 @@ class RemoteEntitySchema(JsonLDSchema):
     class Meta:
         """Meta class."""
 
-        rdf_type = [prov.Entity, schema.DigitalDocument]
+        rdf_type = [prov.Entity]
         model = RemoteEntity
         unknown = EXCLUDE
 
-    commit_sha = fields.String(renku.commit_sha)
+    checksum = fields.String(renku.checksum)
     id = fields.Id()
     path = fields.String(prov.atLocation)
     url = fields.String(schema.url)

renku/data/shacl_shape.json

@@ -294,6 +294,14 @@
                "datatype": {
                   "@id": "xsd:string"
                }
+            },
+            {
+               "nodeKind": "sh:Literal",
+               "path": "schema:url",
+               "datatype": {
+                  "@id": "xsd:string"
+               },
+               "maxCount": 1
             }
          ]
       },
@@ -573,7 +581,7 @@
             {
                "path": "schema:isBasedOn",
                "sh:class": {
-                  "@id": "schema:DigitalDocument"
+                  "@id": "prov:Entity"
                },
                "maxCount": 1
             },

setup.py

@@ -165,7 +165,7 @@ def run(self):
     "click>=7.0,<8.0.2",
     "cryptography>=3.4.1,<3.5",
     "cwlgen>=0.4.0,<=0.4.2",
-    "cwltool>=3.1.20210816212154,<3.1.20210921111717",
+    "cwltool==3.1.20210922203925",
     "cwl-utils>=0.10",
     "environ_config>=18.2.0,<21.3.0",
     "filelock>=3.0.0,<=3.0.12",

tests/cli/test_integration_datasets.py

@@ -877,6 +877,9 @@ def test_add_data_from_git(runner, client, params, path, load_dataset_with_injec
     assert file.source == remote
     assert file.based_on.url == remote
 
+    result = runner.invoke(cli, ["graph", "export", "--format", "json-ld", "--strict"])
+    assert 0 == result.exit_code, format_result_exception(result)
+
 
 @pytest.mark.integration
 @pytest.mark.parametrize(
@@ -986,7 +989,6 @@ def test_dataset_update(client, runner, params, load_dataset_with_injection):
     assert after.based_on.id != before.based_on.id
     assert after.based_on.path == before.based_on.path
     assert after.based_on.url == url
-    assert after.based_on.commit_sha is None
     assert after.based_on.checksum in after.based_on.id
 
     result = runner.invoke(cli, ["doctor"])

tests/cli/test_migrate.py

@@ -218,7 +218,7 @@ def test_comprehensive_dataset_migration(
     assert isinstance(file_.based_on, RemoteEntity)
     assert file_.source == file_.based_on.url
     assert "Makefile" == file_.based_on.path
-    assert "49f331d7388785208ccfb3cfb9156b226d9b59ea" == file_.based_on.commit_sha
+    assert "49f331d7388785208ccfb3cfb9156b226d9b59ea" == file_.based_on.checksum
 
     file_ = dataset.find_file("data/mixed/data.txt")
     assert file_.entity.id.endswith("/data/mixed/data.txt")