feat(service): add support for doctor check in cache migration endpoint.

renku/command/checks/activities.py

@@ -58,7 +58,7 @@ def check_migrated_activity_ids(fix, activity_gateway: IActivityGateway, **_):
         wrong_activities = []
 
     if not wrong_activities:
-        return True, None
+        return True, False, None
 
     problems = (
         WARNING
@@ -68,7 +68,7 @@ def check_migrated_activity_ids(fix, activity_gateway: IActivityGateway, **_):
         + "\n"
     )
 
-    return False, problems
+    return False, True, problems
 
 
 @inject.autoparams("activity_gateway")
@@ -95,7 +95,7 @@ def check_activity_dates(fix, activity_gateway: IActivityGateway, **_):
             invalid_activities.append(activity)
 
     if not invalid_activities:
-        return True, None
+        return True, False, None
     if not fix:
         ids = [a.id for a in invalid_activities]
         message = (
@@ -104,13 +104,13 @@ def check_activity_dates(fix, activity_gateway: IActivityGateway, **_):
             + "\n\t"
             + "\n\t".join(ids)
         )
-        return False, message
+        return False, True, message
 
     fix_activity_dates(activities=invalid_activities)
     project_context.database.commit()
     communication.info("Activity dates were fixed")
 
-    return True, None
+    return True, False, None
 
 
 def fix_activity_dates(activities):

renku/command/checks/datasets.py

@@ -43,7 +43,7 @@ def check_dataset_old_metadata_location(**_):
     old_metadata = get_pre_0_3_4_datasets_metadata()
 
     if not old_metadata:
-        return True, None
+        return True, False, None
 
     problems = (
         WARNING + "There are metadata files in the old location."
@@ -52,7 +52,7 @@ def check_dataset_old_metadata_location(**_):
         + "\n"
     )
 
-    return False, problems
+    return False, False, problems
 
 
 @inject.autoparams("dataset_gateway")
@@ -79,7 +79,7 @@ def check_missing_files(dataset_gateway: IDatasetGateway, **_):
                 missing[dataset.name].append(file_.entity.path)
 
     if not missing:
-        return True, None
+        return True, False, None
 
     problems = WARNING + "There are missing files in datasets."
 
@@ -91,7 +91,7 @@ def check_missing_files(dataset_gateway: IDatasetGateway, **_):
             + "\n\t  ".join(click.style(path, fg="red") for path in files)
         )
 
-    return False, problems
+    return False, False, problems
 
 
 @inject.autoparams("dataset_gateway")
@@ -130,7 +130,7 @@ def fix_or_report(dataset):
                 break
 
     if not invalid_datasets:
-        return True, None
+        return True, False, None
 
     problems = (
         WARNING
@@ -140,7 +140,7 @@ def fix_or_report(dataset):
         + "\n"
     )
 
-    return False, problems
+    return False, True, problems
 
 
 @inject.autoparams("dataset_gateway")
@@ -193,9 +193,9 @@ def check_dataset_files_outside_datadir(fix, dataset_gateway: IDatasetGateway, *
             + "\n\t".join(click.style(file.entity.path, fg="yellow") for file in invalid_files)
             + "\n"
         )
-        return False, problems
+        return False, True, problems
 
-    return True, None
+    return True, False, None
 
 
 @inject.autoparams("dataset_gateway")
@@ -222,7 +222,7 @@ def check_external_files(fix, dataset_gateway: IDatasetGateway, **_):
                 datasets[dataset.name].append(file)
 
     if not external_files:
-        return True, None
+        return True, False, None
 
     external_files_str = "\n\t".join(sorted(external_files))
 
@@ -232,7 +232,7 @@ def check_external_files(fix, dataset_gateway: IDatasetGateway, **_):
             "Use 'renku dataset rm' or rerun 'renku doctor' with '--fix' flag to remove them:\n\t"
             f"{external_files_str}\n"
         )
-        return False, problems
+        return False, True, problems
 
     communication.info(
         "The following external files were deleted from the project. You need to add them later manually using a "
@@ -242,4 +242,4 @@ def check_external_files(fix, dataset_gateway: IDatasetGateway, **_):
     for name, files in datasets.items():
         file_unlink(name=name, yes=True, dataset_files=files)
 
-    return True, None
+    return True, False, None

renku/command/checks/githooks.py

@@ -41,7 +41,7 @@ def check_git_hooks_installed(**_):
         hook_path = get_hook_path(name=hook, path=project_context.path)
         if not hook_path.exists():
             message = WARNING + "Git hooks are not installed. " 'Use "renku githooks install" to install them. \n'
-            return False, message
+            return False, False, message
 
         with hook_path.open() as file_:
             actual_hook = _extract_renku_hook(file_)
@@ -50,16 +50,16 @@ def check_git_hooks_installed(**_):
 
         if not expected_hook:
             message = WARNING + "Cannot check for existence of Git hooks.\n"
-            return False, message
+            return False, False, message
 
         if actual_hook != expected_hook:
             message = (
                 WARNING + "Git hooks are outdated or not installed.\n"
                 '  (use "renku githooks install --force" to update them) \n'
             )
-            return False, message
+            return False, False, message
 
-    return True, None
+    return True, False, None
 
 
 def _extract_renku_hook(file):

renku/command/checks/migration.py

@@ -35,6 +35,6 @@ def check_migration(**_):
             ERROR + "Project version is not supported by your version of Renku.\n" + "  (upgrade your Renku version)\n"
         )
     else:
-        return True, None
+        return True, False, None
 
-    return False, problems
+    return False, False, problems

renku/command/checks/project.py

@@ -42,21 +42,25 @@ def check_project_id_group(fix, project_gateway: IProjectGateway, **_):
     )
 
     if namespace is None or name is None:
-        return True, None
+        return True, False, None
 
     generated_id = Project.generate_id(namespace=namespace, name=name)
 
     if generated_id == current_project.id:
-        return True, None
+        return True, False, None
 
     if fix:
         communication.info(f"Fixing project id '{current_project.id}' -> '{generated_id}'")
         current_project.id = generated_id
         project_gateway.update_project(current_project)
-        return True, None
+        return True, False, None
 
-    return True, (
-        WARNING
-        + "Project id doesn't match id created based on the current Git remote (use 'renku doctor --fix' to fix it):"
-        f"\n\t'{current_project.id}' -> '{generated_id}'"
+    return (
+        False,
+        True,
+        (
+            WARNING
+            + "Project id doesn't match id based on the current Git remote (use 'renku doctor --fix' to fix it):"
+            f"\n\t'{current_project.id}' -> '{generated_id}'"
+        ),
     )

renku/command/checks/storage.py

@@ -29,19 +29,19 @@ def check_lfs_info(**_):
         Tuple of whether project structure is valid and string of found problems.
     """
     if not check_external_storage():
-        return True, None
+        return True, False, None
 
     files = check_lfs_migrate_info()
 
     if not files:
-        return True, None
+        return True, False, None
 
     message = (
         WARNING
         + "Git history contains large files - consider moving them "
-        + "to external storage like git LFS\n\t"
+        + "to external storage like git LFS using 'renku storage migrate'\n\t"
         + "\n\t".join(files)
         + "\n"
     )
 
-    return False, message
+    return False, False, message

renku/command/checks/validate_shacl.py

@@ -16,15 +16,13 @@
 """Check KG structure using SHACL."""
 
 import pyld
-import yaml
 
 from renku.command.command_builder import inject
 from renku.command.schema.dataset import dump_dataset_as_jsonld
 from renku.command.schema.project import ProjectSchema
 from renku.command.util import WARNING
 from renku.core.interface.dataset_gateway import IDatasetGateway
 from renku.core.util.shacl import validate_graph
-from renku.core.util.yaml import NoDatesSafeLoader
 from renku.domain_model.project_context import project_context
 
 
@@ -78,11 +76,11 @@ def check_project_structure(**_):
     conform, graph, t = _check_shacl_structure(data)
 
     if conform:
-        return True, None
+        return True, False, None
 
     problems = f"{WARNING}Invalid structure of project metadata\n\t{_shacl_graph_to_string(graph)}"
 
-    return False, problems
+    return False, False, problems
 
 
 @inject.autoparams("dataset_gateway")
@@ -116,16 +114,9 @@ def check_datasets_structure(dataset_gateway: IDatasetGateway, **_):
         problems.append(f"{dataset.name}\n\t{_shacl_graph_to_string(graph)}\n")
 
     if ok:
-        return True, None
+        return True, False, None
 
-    return False, "\n".join(problems)
-
-
-def _check_shacl_structure_for_path(path):
-    with path.open(mode="r") as fp:
-        data = yaml.load(fp, Loader=NoDatesSafeLoader) or {}
-
-    return _check_shacl_structure(data)
+    return False, False, "\n".join(problems)
 
 
 def _check_shacl_structure(data):

renku/command/checks/workflow.py

@@ -26,7 +26,7 @@
 from renku.infrastructure.gateway.activity_gateway import reindex_catalog
 
 
-def check_activity_catalog(fix, force, **_) -> Tuple[bool, Optional[str]]:
+def check_activity_catalog(fix, force, **_) -> Tuple[bool, bool, Optional[str]]:
     """Check if the activity-catalog needs to be rebuilt.
 
     Args:
@@ -44,25 +44,25 @@ def check_activity_catalog(fix, force, **_) -> Tuple[bool, Optional[str]]:
     # NOTE: If len(activity_catalog) > 0 then either the project is fixed or it used a fixed Renku version but still has
     # broken metadata. ``force`` allows to rebuild the metadata in the latter case.
     if (len(relations) == 0 or len(activity_catalog) > 0) and not (force and fix):
-        return True, None
+        return True, False, None
 
     if not fix:
         problems = (
             WARNING + "The project's workflow metadata needs to be rebuilt (use 'renku doctor --fix' to rebuild it).\n"
         )
 
-        return False, problems
+        return False, True, problems
 
     with communication.busy("Rebuilding workflow metadata ..."):
         reindex_catalog(database=database)
 
     communication.info("Workflow metadata was rebuilt")
 
-    return True, None
+    return True, False, None
 
 
 @inject.autoparams("plan_gateway")
-def check_plan_modification_date(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, Optional[str]]:
+def check_plan_modification_date(fix, plan_gateway: IPlanGateway, **_) -> Tuple[bool, bool, Optional[str]]:
     """Check if all plans have modification date set for them.
 
     Args:
@@ -81,21 +81,21 @@ def check_plan_modification_date(fix, plan_gateway: IPlanGateway, **_) -> Tuple[
             to_be_processed.append(plan)
 
     if not to_be_processed:
-        return True, None
+        return True, False, None
     if not fix:
         ids = [plan.id for plan in to_be_processed]
         message = (
             WARNING
             + "The following workflows have incorrect modification date (use 'renku doctor --fix' to fix them):\n\t"
             + "\n\t".join(ids)
         )
-        return False, message
+        return False, True, message
 
     fix_plan_dates(plans=to_be_processed, plan_gateway=plan_gateway)
     project_context.database.commit()
     communication.info("Workflow modification dates were fixed")
 
-    return True, None
+    return True, False, None
 
 
 def fix_plan_dates(plans: List[AbstractPlan], plan_gateway):

renku/command/doctor.py

@@ -44,22 +44,25 @@ def _doctor_check(fix: bool, force: bool):
     from renku.command import checks
 
     is_ok = True
+    fixes_available = False
     problems = []
 
     for check in checks.__all__:
         try:
-            ok, problems_ = getattr(checks, check)(fix=fix, force=force)
+            ok, has_fix, problems_ = getattr(checks, check)(fix=fix, force=force)
         except Exception:
             ok = False
+            has_fix = False
             tb = "\n\t".join(traceback.format_exc().split("\n"))
             problems_ = f"{ERROR}Exception raised when running {check}\n\t{tb}"
 
         is_ok &= ok
+        fixes_available |= has_fix
 
         if problems_:
             problems.append(problems_)
 
-    return is_ok, "\n".join(problems)
+    return is_ok, fixes_available, "\n".join(problems)
 
 
 def doctor_check_command(with_fix):

renku/ui/cli/doctor.py

@@ -58,11 +58,15 @@ def doctor(ctx, fix, force):
     command = doctor_check_command(with_fix=fix)
     if fix:
         command = command.with_communicator(communicator)
-    is_ok, problems = command.build().execute(fix=fix, force=force).output
+    is_ok, fixes_available, problems = command.build().execute(fix=fix, force=force).output
 
     if is_ok:
         click.secho("Everything seems to be ok.", fg=color.GREEN)
         ctx.exit(0)
 
     click.echo(problems)
+
+    if fixes_available:
+        click.echo("Run with '--fix' flag to try and fix these issues.")
+
     ctx.exit(1)