feat(api): project status

.github/workflows/test_deploy.yml

@@ -532,7 +532,8 @@ jobs:
           curl -L https://raw.githubusercontent.com/Homebrew/homebrew-core/43842898fd3ff43273466052722f5ba2789196cb/Formula/git-lfs.rb > git-lfs.rb && brew install git-lfs.rb && rm git-lfs.rb
           brew install shellcheck node || brew link --overwrite node
           python -m pip install --upgrade pip
-          python -m pip install wheel
+          python -m pip install wheel poetry-dynamic-versioning
+          poetry-dynamic-versioning
           make download-templates
           python -m pip install .[all]
           git config --global --add user.name "Renku Bot"

renku/command/command_builder/client_dispatcher.py

@@ -44,17 +44,18 @@ def current_client(self) -> LocalClient:
 
     def push_client_to_stack(
         self, path: Union[Path, str], renku_home: str = RENKU_HOME, external_storage_requested: bool = True
-    ) -> None:
+    ) -> LocalClient:
         """Create and push a new client to the stack."""
         if isinstance(path, str):
             path = Path(path)
 
         new_client = LocalClient(path=path)
-        self.client_stack.append(new_client)
+        self.push_created_client_to_stack(new_client)
+
+        return new_client
 
     def push_created_client_to_stack(self, client: LocalClient) -> None:
         """Push an already created client to the stack."""
-
         self.client_stack.append(client)
 
     def pop_client(self) -> None:

renku/command/command_builder/command.py

@@ -21,7 +21,7 @@
 import functools
 import threading
 from collections import defaultdict
-from typing import Any, Callable, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
 
 import click
 import inject
@@ -30,6 +30,9 @@
 from renku.core.util.communication import CommunicationCallback
 from renku.core.util.git import default_path
 
+if TYPE_CHECKING:
+    from renku.core.management.client import LocalClient
+
 _LOCAL = threading.local()
 
 
@@ -167,6 +170,7 @@ def __init__(self) -> None:
         self._finalized: bool = False
         self._track_std_streams: bool = False
         self._working_directory: Optional[str] = None
+        self._client: Optional["LocalClient"] = None
 
     def __getattr__(self, name: str) -> Any:
         """Bubble up attributes of wrapped builders."""
@@ -197,11 +201,15 @@ def _injection_pre_hook(self, builder: "Command", context: dict, *args, **kwargs
 
         ctx = click.get_current_context(silent=True)
         if ctx is None:
-            dispatcher.push_client_to_stack(path=default_path(self._working_directory or "."))
+            if self._client:
+                dispatcher.push_created_client_to_stack(self._client)
+            else:
+                self._client = dispatcher.push_client_to_stack(path=default_path(self._working_directory or "."))
             ctx = click.Context(click.Command(builder._operation))  # type: ignore
         else:
-            client = ctx.ensure_object(LocalClient)
-            dispatcher.push_created_client_to_stack(client)
+            if not self._client:
+                self._client = ctx.ensure_object(LocalClient)
+            dispatcher.push_created_client_to_stack(self._client)
 
         context["bindings"] = {IClientDispatcher: dispatcher, "IClientDispatcher": dispatcher}
         context["constructor_bindings"] = {}
@@ -400,6 +408,13 @@ def track_std_streams(self) -> "Command":
 
         return self
 
+    @check_finalized
+    def with_client(self, client: "LocalClient") -> "Command":
+        """Set a client."""
+        self._client = client
+
+        return self
+
     @check_finalized
     def with_git_isolation(self) -> "Command":
         """Whether to run in git isolation or not."""

renku/command/status.py

@@ -17,75 +17,10 @@
 # limitations under the License.
 """Renku ``status`` command."""
 
-from collections import defaultdict
-from pathlib import Path
-from typing import Dict, Set
-
-from renku.command.command_builder import inject
 from renku.command.command_builder.command import Command
-from renku.core.interface.client_dispatcher import IClientDispatcher
-from renku.core.util.os import get_relative_path_to_cwd, get_relative_paths
-from renku.core.workflow.activity import (
-    get_all_modified_and_deleted_activities_and_entities,
-    get_downstream_generating_activities,
-    is_activity_valid,
-)
+from renku.core.workflow.run import get_status
 
 
 def get_status_command():
     """Show a status of the repository."""
-    return Command().command(_get_status).require_migration().with_database(write=False)
-
-
-@inject.autoparams()
-def _get_status(ignore_deleted: bool, client_dispatcher: IClientDispatcher, paths=None):
-    def mark_generations_as_stale(activity):
-        for generation in activity.generations:
-            generation_path = get_relative_path_to_cwd(client.path / generation.entity.path)
-            stale_outputs[generation_path].add(usage_path)
-
-    client = client_dispatcher.current_client
-
-    ignore_deleted = ignore_deleted or client.get_value("renku", "update_ignore_delete")
-
-    modified, deleted = get_all_modified_and_deleted_activities_and_entities(client.repository)
-
-    modified = {(a, e) for a, e in modified if is_activity_valid(a)}
-    deleted = {(a, e) for a, e in deleted if is_activity_valid(a)}
-
-    if not modified and not deleted:
-        return None, None, None, None
-
-    paths = paths or []
-    paths = get_relative_paths(base=client.path, paths=[Path.cwd() / p for p in paths])
-
-    modified_inputs: Set[str] = set()
-    stale_outputs: Dict[str, Set[str]] = defaultdict(set)
-    stale_activities: Dict[str, Set[str]] = defaultdict(set)
-
-    for start_activity, entity in modified:
-        usage_path = get_relative_path_to_cwd(client.path / entity.path)
-
-        # NOTE: Add all downstream activities if the modified entity is in paths; otherwise, add only activities that
-        # chain-generate at least one of the paths
-        generation_paths = [] if not paths or entity.path in paths else paths
-
-        activities = get_downstream_generating_activities(
-            starting_activities={start_activity},
-            paths=generation_paths,
-            ignore_deleted=ignore_deleted,
-            client_path=client.path,
-        )
-        if activities:
-            modified_inputs.add(usage_path)
-
-            for activity in activities:
-                if len(activity.generations) == 0:
-                    stale_activities[activity.id].add(usage_path)
-                else:
-                    mark_generations_as_stale(activity)
-
-    deleted_paths = {e.path for _, e in deleted}
-    deleted_paths = {get_relative_path_to_cwd(client.path / d) for d in deleted_paths if not paths or d in paths}
-
-    return stale_outputs, stale_activities, modified_inputs, deleted_paths
+    return Command().command(get_status).require_migration().with_database(write=False)

renku/core/interface/client_dispatcher.py

@@ -38,7 +38,7 @@ def current_client(self) -> "LocalClient":
 
     def push_client_to_stack(
         self, path: Union[Path, str], renku_home: str = ".renku", external_storage_requested: bool = True
-    ) -> None:
+    ) -> "LocalClient":
         """Create and push a new client to the stack."""
         raise NotImplementedError
 

renku/core/management/repository.py

@@ -21,6 +21,7 @@
 import shutil
 from contextlib import contextmanager
 from fnmatch import fnmatch
+from pathlib import Path
 from typing import Any, Optional
 from uuid import uuid4
 
@@ -29,7 +30,6 @@
 
 from renku.command.command_builder import inject
 from renku.core import errors
-from renku.core.compat import Path
 from renku.core.constant import RENKU_HOME
 from renku.core.interface.database_gateway import IDatabaseGateway
 from renku.core.interface.project_gateway import IProjectGateway

renku/core/management/storage.py

@@ -44,6 +44,20 @@
 from .repository import RepositoryApiMixin  # type: ignore
 
 
+class RenkuGitWildMatchPattern(pathspec.patterns.GitWildMatchPattern):
+    """Custom GitWildMatchPattern matcher."""
+
+    __slots__ = ("pattern",)
+
+    def __init__(self, pattern, include=None):
+        """Initialize RenkuRegexPattern."""
+        super().__init__(pattern, include)
+        self.pattern = pattern
+
+
+pathspec.util.register_pattern("renku_gitwildmatch", RenkuGitWildMatchPattern)
+
+
 def check_external_storage_wrapper(fn):
     """Check availability of external storage on methods that need it.
 

renku/core/workflow/activity.py

@@ -15,7 +15,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Renku workflow commands."""
+"""Activity management."""
 
 import itertools
 from collections import defaultdict
@@ -374,7 +374,8 @@ def get_modified_activities(
         for usage in activity.usages:
             entity = usage.entity
             current_checksum = hashes.get(entity.path, None)
-            if current_checksum is None:
+            usage_path = repository.path / usage.entity.path
+            if current_checksum is None or not usage_path.exists():
                 deleted.add((activity, entity))
             elif current_checksum != entity.checksum:
                 modified.add((activity, entity))