fix(core): various migration issues (#2491)

renku/core/management/migrate.py

@@ -57,6 +57,7 @@
     is_using_temporary_datasets_path,
     read_project_version,
 )
+from renku.core.management.workflow.plan_factory import RENKU_TMP
 from renku.core.utils import communication
 
 SUPPORTED_PROJECT_VERSION = 9
@@ -157,16 +158,28 @@ def migrate(
             except (Exception, BaseException) as e:
                 raise MigrationError("Couldn't execute migration") from e
             n_migrations_executed += 1
-    if n_migrations_executed > 0 and not is_using_temporary_datasets_path():
-        client._project = None  # NOTE: force reloading of project metadata
-        client.project.version = str(version)
-        project_gateway.update_project(client.project)
+    if not is_using_temporary_datasets_path():
+        if n_migrations_executed > 0:
+            client._project = None  # NOTE: force reloading of project metadata
+            client.project.version = str(version)
+            project_gateway.update_project(client.project)
 
-        communication.echo(f"Successfully applied {n_migrations_executed} migrations.")
+            communication.echo(f"Successfully applied {n_migrations_executed} migrations.")
+
+        _remove_untracked_renku_files(renku_path=client.renku_path)
 
     return n_migrations_executed != 0, template_updated, docker_updated
 
 
+def _remove_untracked_renku_files(renku_path):
+    from renku.core.management.datasets import DatasetsApiMixin
+
+    untracked_paths = [RENKU_TMP, DatasetsApiMixin.CACHE, "vendors"]
+    for path in untracked_paths:
+        path = renku_path / path
+        shutil.rmtree(path, ignore_errors=True)
+
+
 @inject.autoparams()
 def _update_template(client_dispatcher: IClientDispatcher, project_gateway: IProjectGateway, check_only=False):
     """Update local files from the remote template."""

renku/core/management/migrations/utils/conversion.py

@@ -153,6 +153,20 @@ def convert_derived_from(derived_from: Optional[old_datasets.Url]) -> Optional[U
 
         return Url(url_id=Dataset.generate_id(identifier=Path(path).name))
 
+    def convert_license(license):
+        if not license:
+            return license
+        elif isinstance(license, (Url, str)):
+            return license
+        elif isinstance(license, dict) and len(license) == 1:
+            return list(license.values())[0]
+        elif isinstance(license, dict) and "@id" in license:
+            return license["@id"]
+        elif isinstance(license, list) and len(license) == 1:
+            return license[0]
+
+        return str(license)
+
     tags = [_convert_dataset_tag(tag) for tag in (dataset.tags or [])]
 
     return (
@@ -169,7 +183,7 @@ def convert_derived_from(derived_from: Optional[old_datasets.Url]) -> Optional[U
             images=[_convert_image_object(image) for image in (dataset.images or [])],
             in_language=_convert_language(dataset.in_language),
             keywords=dataset.keywords,
-            license=dataset.license,
+            license=convert_license(dataset.license),
             name=dataset.name,
             project_id=client.project.id,
             initial_identifier=_convert_dataset_identifier(dataset.initial_identifier),

renku/data/shacl_shape.json

@@ -400,6 +400,12 @@
                   "@id": "schema:Language"
                }
             },
+            {
+               "path": "schema:image",
+               "sh:class": {
+                  "@id": "schema:ImageObject"
+               }
+            },
             {
                "nodeKind": "sh:Literal",
                "path": "schema:keywords",
@@ -505,6 +511,35 @@
             }
          ]
       },
+      {
+         "@id": "_:ImageObjectShape",
+         "@type": "sh:NodeShape",
+         "ignoredProperties": [
+            {
+               "@id": "rdf:type"
+            }
+         ],
+         "closed": true,
+         "targetClass": "schema:ImageObject",
+         "property": [
+            {
+               "path": "schema:contentUrl",
+               "nodeKind": "sh:Literal",
+               "datatype": {
+                  "@id": "xsd:string"
+               },
+               "maxCount": 1
+            },
+            {
+               "path": "schema:position",
+               "nodeKind": "sh:Literal",
+               "datatype": {
+                  "@id": "xsd:integer"
+               },
+               "maxCount": 1
+            }
+         ]
+      },
       {
          "@id": "_:inLanguageShape",
          "@type": "sh:NodeShape",
@@ -770,9 +805,18 @@
             },
             {
                "path": "prov:agent",
-               "sh:class": {
-                  "@id": "prov:SoftwareAgent"
-               },
+               "or": [
+                  {
+                     "sh:class": {
+                        "@id": "prov:SoftwareAgent"
+                     }
+                  },
+                  {
+                     "sh:class": {
+                        "@id": "schema:Person"
+                     }
+                  }
+               ],
                "minCount": 1,
                "maxCount": 1
             }

tests/cli/test_migrate.py

@@ -24,9 +24,11 @@
 import pytest
 
 from renku.cli import cli
+from renku.core.management import RENKU_HOME
 from renku.core.management.client import LocalClient
 from renku.core.management.dataset.datasets_provenance import DatasetsProvenance
 from renku.core.management.migrate import SUPPORTED_PROJECT_VERSION, get_migrations
+from renku.core.management.workflow.plan_factory import RENKU_TMP
 from renku.core.metadata.repository import Repository
 from renku.core.models.dataset import RemoteEntity
 from tests.utils import format_result_exception
@@ -165,6 +167,11 @@ def test_migrations_runs(isolated_runner, old_project):
     assert 0 == result.exit_code, format_result_exception(result)
     assert "No migrations required." in result.output
 
+    tmp_path = os.path.join(RENKU_HOME, RENKU_TMP)
+    paths = [c.b_path for c in old_project.head.commit.get_changes() if tmp_path in c.b_path]
+
+    assert 0 == len(paths), ", ".join(paths)
+
 
 @pytest.mark.migration
 def test_migration_version():