feat(cli): gc command (#2866)

docs/cheatsheet_hash

@@ -1,2 +1,2 @@
-8f835f7c5c76b60f8cd9785fd1e0cf31  cheatsheet.tex
+3e7d7567c17eba64945b70a3c1009343  cheatsheet.tex
 c70c179e07f04186ec05497564165f11  sdsc_cheatsheet.cls

docs/reference/commands.rst

@@ -62,6 +62,13 @@ Renku Command Line
 
 .. automodule:: renku.ui.cli.dataset
 
+.. _cli-gc:
+
+``renku gc``
+------------
+
+.. automodule:: renku.ui.cli.gc
+
 .. _cli-graph:
 
 ``renku graph``

renku/command/command_builder/client_dispatcher.py

@@ -17,11 +17,11 @@
 # limitations under the License.
 """Renku client dispatcher."""
 from pathlib import Path
-from typing import Optional, Union
+from typing import Union
 
 from renku.core import errors
+from renku.core.constant import RENKU_HOME
 from renku.core.interface.client_dispatcher import IClientDispatcher
-from renku.core.management import RENKU_HOME
 from renku.core.management.client import LocalClient
 
 
@@ -35,7 +35,7 @@ def __init__(self):
         self.client_stack = []
 
     @property
-    def current_client(self) -> Optional[LocalClient]:
+    def current_client(self) -> LocalClient:
         """Get the currently active client."""
         if len(self.client_stack) == 0:
             raise errors.ConfigurationError("No client configured for injection")

renku/command/gc.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Garbage collection and cleanup command."""
+
+from renku.command.command_builder.command import Command
+
+
+def gc_command():
+    """Command to clean up project's caches."""
+    from renku.core.misc.gc import remove_caches
+
+    return Command().command(remove_caches).lock_project()

renku/command/init.py

@@ -26,10 +26,10 @@
 from renku.command.command_builder.command import Command, inject
 from renku.command.git import set_git_home
 from renku.core import errors
+from renku.core.constant import RENKU_HOME
 from renku.core.interface.client_dispatcher import IClientDispatcher
 from renku.core.interface.database_dispatcher import IDatabaseDispatcher
 from renku.core.interface.database_gateway import IDatabaseGateway
-from renku.core.management import RENKU_HOME
 from renku.core.migration.utils import OLD_METADATA_PATH
 from renku.core.template.template import (
     FileAction,

renku/core/constant.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2017-2022 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku core constants."""
+
+CACHE = "cache"
+"""Directory to cache transient data."""
+
+RENKU_HOME = ".renku"
+"""Project directory name."""
+
+RENKU_TMP = "tmp"

renku/core/dataset/constant.py

@@ -19,16 +19,13 @@
 
 from pathlib import Path
 
-from renku.core.management import RENKU_HOME
+from renku.core.constant import RENKU_HOME
 from renku.core.management.repository import RepositoryApiMixin
 from renku.domain_model.refs import LinkReference
 
 POINTERS = "pointers"
 """Directory for storing external pointer files."""
 
-CACHE = "cache"
-"""Directory to cache transient data."""
-
 DATASET_IMAGES = "dataset_images"
 """Directory for dataset images."""
 

renku/core/dataset/dataset_add.py

@@ -30,7 +30,8 @@
 
 from renku.command.command_builder.command import inject
 from renku.core import errors
-from renku.core.dataset.constant import CACHE, renku_pointers_path
+from renku.core.constant import CACHE
+from renku.core.dataset.constant import renku_pointers_path
 from renku.core.dataset.context import DatasetContext
 from renku.core.dataset.datasets_provenance import DatasetsProvenance
 from renku.core.dataset.pointer_file import create_external_file

renku/core/errors.py

@@ -23,7 +23,7 @@
 
 import click
 
-from renku.core.management import RENKU_HOME
+from renku.core.constant import RENKU_HOME
 
 
 class RenkuException(Exception):

renku/core/interface/client_dispatcher.py

@@ -19,7 +19,10 @@
 
 from abc import ABC
 from pathlib import Path
-from typing import Union
+from typing import TYPE_CHECKING, Union
+
+if TYPE_CHECKING:
+    from renku.core.management.client import LocalClient
 
 
 class IClientDispatcher(ABC):
@@ -29,7 +32,7 @@ class IClientDispatcher(ABC):
     """
 
     @property
-    def current_client(self):
+    def current_client(self) -> "LocalClient":
         """Get the currently active client."""
         raise NotImplementedError
 

renku/core/management/__init__.py

@@ -16,6 +16,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Renku repository management."""
-
-RENKU_HOME = ".renku"
-"""Project directory name."""

renku/core/management/config.py

@@ -25,7 +25,7 @@
 import click
 import portalocker
 
-from renku.core.management import RENKU_HOME
+from renku.core.constant import RENKU_HOME
 from renku.domain_model.enums import ConfigFilter
 
 APP_NAME = "Renku"

renku/core/management/migrate.py

@@ -37,6 +37,7 @@
 from packaging.version import Version
 
 from renku.command.command_builder.command import inject
+from renku.core.constant import RENKU_TMP
 from renku.core.errors import (
     DockerfileUpdateError,
     MigrationError,
@@ -55,7 +56,6 @@
     read_project_version,
 )
 from renku.core.util import communication
-from renku.core.workflow.plan_factory import RENKU_TMP
 
 try:
     import importlib_resources
@@ -165,7 +165,7 @@ def migrate(
 
 
 def _remove_untracked_renku_files(renku_path):
-    from renku.core.dataset.constant import CACHE
+    from renku.core.constant import CACHE
 
     untracked_paths = [RENKU_TMP, CACHE, "vendors"]
     for path in untracked_paths:

renku/core/management/repository.py

@@ -30,9 +30,9 @@
 from renku.command.command_builder import inject
 from renku.core import errors
 from renku.core.compat import Path
+from renku.core.constant import RENKU_HOME
 from renku.core.interface.database_gateway import IDatabaseGateway
 from renku.core.interface.project_gateway import IProjectGateway
-from renku.core.management import RENKU_HOME
 from renku.core.management.git import GitCore
 from renku.core.util.git import default_path
 from renku.domain_model.enums import ConfigFilter

renku/core/migration/m_0003__2_initial.py

@@ -21,7 +21,7 @@
 import urllib
 from pathlib import Path
 
-from renku.core.management import RENKU_HOME
+from renku.core.constant import RENKU_HOME
 from renku.core.management.repository import DEFAULT_DATA_DIR as DATA_DIR
 from renku.core.migration.models.v3 import Collection, Dataset, Project, get_client_datasets
 from renku.core.migration.models.v9 import generate_file_id, generate_label

renku/core/misc/__init__.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2017-2022 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Various management functionalities."""

renku/core/misc/gc.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2020 - Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Project cleanup management."""
+
+from renku.command.command_builder.command import inject
+from renku.core.constant import CACHE, RENKU_HOME, RENKU_TMP
+from renku.core.interface.client_dispatcher import IClientDispatcher
+
+
+@inject.autoparams()
+def remove_caches(client_dispatcher: IClientDispatcher):
+    """Remove caches and temporary files.
+
+    Args:
+        client_dispatcher(IClientDispatcher):  Injected client dispatcher.
+
+    """
+    client = client_dispatcher.current_client
+
+    cache_paths = [CACHE, RENKU_TMP]
+    paths = [client.path / RENKU_HOME / p for p in cache_paths]
+
+    client.repository.clean(paths=paths)

renku/core/template/template.py

@@ -155,6 +155,7 @@ def copy_template_metadata_to_client():
         except OSError as e:
             # TODO: Use a general cleanup strategy: https://github.com/SwissDataScienceCenter/renku-python/issues/736
             if cleanup:
+                client.repository.reset(hard=True)
                 client.repository.clean()
 
             raise errors.TemplateUpdateError(f"Cannot write to '{destination}'") from e

renku/core/util/contexts.py

@@ -106,7 +106,7 @@ def measure(message="TOTAL"):
 
 def click_context(path, command):
     """Provide a click context with repo path injected."""
-    from renku.core.management import RENKU_HOME
+    from renku.core.constant import RENKU_HOME
     from renku.core.management.client import LocalClient
 
     return click.Context(

renku/core/util/git.py

@@ -137,7 +137,7 @@ def get_cache_directory_for_repository(client, url) -> Path:
         Path: The path of the cache.
 
     """
-    from renku.core.dataset.constant import CACHE
+    from renku.core.constant import CACHE
 
     return client.renku_path / CACHE / get_full_repository_path(url)
 

renku/core/util/metadata.py

@@ -84,8 +84,8 @@ def construct_creator(creator: Union[dict, str], ignore_email) -> Tuple[Optional
 
 def is_external_file(path: Union[Path, str], client_path: Path):
     """Checks if a path is an external file."""
+    from renku.core.constant import RENKU_HOME
     from renku.core.dataset.constant import POINTERS
-    from renku.core.management import RENKU_HOME
 
     path = client_path / path
     if not path.is_symlink() or not is_subpath(path=path, base=client_path):

renku/core/workflow/plan_factory.py

@@ -31,9 +31,9 @@
 
 from renku.command.command_builder.command import inject
 from renku.core import errors
+from renku.core.constant import RENKU_HOME, RENKU_TMP
 from renku.core.interface.client_dispatcher import IClientDispatcher
 from renku.core.interface.project_gateway import IProjectGateway
-from renku.core.management import RENKU_HOME
 from renku.core.util.git import is_path_safe
 from renku.core.util.metadata import is_external_file
 from renku.core.util.os import get_absolute_path, get_relative_path, is_subpath
@@ -50,8 +50,6 @@
 
 STARTED_AT = int(time.time() * 1000)
 
-RENKU_TMP = "tmp"
-
 
 class PlanFactory:
     """Factory for creating a plan from a command line call."""

renku/core/workflow/providers/toil.py

@@ -37,11 +37,10 @@
 
 from renku.command.echo import progressbar
 from renku.core import errors
+from renku.core.constant import RENKU_HOME, RENKU_TMP
 from renku.core.errors import WorkflowExecuteError
-from renku.core.management.config import RENKU_HOME
 from renku.core.plugin import hookimpl
 from renku.core.plugin.provider import RENKU_ENV_PREFIX
-from renku.core.workflow.plan_factory import RENKU_TMP
 from renku.domain_model.workflow.parameter import CommandParameterBase
 from renku.domain_model.workflow.plan import Plan
 from renku.domain_model.workflow.provider import IWorkflowProvider

renku/domain_model/template.py

@@ -29,7 +29,7 @@
 import yaml
 
 from renku.core import errors
-from renku.core.management import RENKU_HOME
+from renku.core.constant import RENKU_HOME
 from renku.core.util.os import get_safe_relative_path, hash_file
 
 TEMPLATE_MANIFEST = "manifest.yaml"

renku/infrastructure/repository.py

@@ -225,10 +225,9 @@ def checkout(self, reference: Union["Branch", "Tag", str]):
         """Check-out a specific reference."""
         self.run_git_command("checkout", reference)
 
-    def clean(self):
-        """Remove all untracked files and reset the repo."""
-        self.reset(hard=True)
-        self.run_git_command("clean", "-xdff")
+    def clean(self, paths: List[Union[Path, str]] = None):
+        """Remove untracked files."""
+        self.run_git_command("clean", "-xdff", paths)
 
     def fetch(
         self,