feat(core): renku clone with credentials (#2517)

renku/cli/__init__.py

@@ -79,7 +79,7 @@
 from renku.cli.graph import graph
 from renku.cli.init import init as init_command
 from renku.cli.log import log
-from renku.cli.login import login, logout, token
+from renku.cli.login import credentials, login, logout
 from renku.cli.migrate import check_immutable_template_files, migrate, migrationscheck
 from renku.cli.move import move
 from renku.cli.project import project
@@ -119,7 +119,7 @@ def get_entry_points(name: str):
 #: Monkeypatch Click application.
 click_completion.init()
 
-WARNING_UNPROTECTED_COMMANDS = ["clone", "init", "help", "login", "logout", "service"]
+WARNING_UNPROTECTED_COMMANDS = ["clone", "init", "help", "login", "logout", "service", "credentials"]
 
 
 def _uuid_representer(dumper, data):
@@ -239,7 +239,7 @@ def help(ctx):
 cli.add_command(save)
 cli.add_command(status)
 cli.add_command(storage)
-cli.add_command(token)
+cli.add_command(credentials)
 cli.add_command(update)
 cli.add_command(workflow)
 cli.add_command(service)

renku/cli/clone.py

@@ -45,6 +45,10 @@
         $ git remote remove origin
         $ git remote add origin <new-repository-url>
         $ git push --mirror origin
+
+To clone private repositories with an HTTPS address, you first need to log into
+a Renku deployment using the :ref:`cli-login` command. ``renku clone`` will use
+the stored credentials when available.
 """
 
 import click
@@ -61,6 +65,6 @@ def clone(no_pull_data, url, path):
 
     click.echo(f"Cloning {url} ...")
     project_clone_command().build().execute(
-        url=url, path=path, skip_smudge=no_pull_data, progress=get_git_progress_instance()
+        url=url, path=path, skip_smudge=no_pull_data, progress=get_git_progress_instance(), use_renku_credentials=True
     )
     click.secho("OK", fg="green")

renku/cli/login.py

@@ -94,9 +94,9 @@ def logout(endpoint):
 @click.command(hidden=True)
 @click.option("--hostname", default=None, hidden=True, help="Remote hostname.")
 @click.argument("command")
-def token(command, hostname):
-    """A git credential helper for returning renku token."""
-    from renku.core.commands.login import token_command
+def credentials(command, hostname):
+    """A git credential helper for returning renku user/token."""
+    from renku.core.commands.login import credentials_command
 
     communicator = ClickCallback()
-    token_command().with_communicator(communicator).build().execute(command=command, hostname=hostname)
+    credentials_command().with_communicator(communicator).build().execute(command=command, hostname=hostname)

renku/core/commands/clone.py

@@ -16,12 +16,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Clone a Renku repo along with all Renku-specific initializations."""
-
 from renku.core.management.command_builder import inject
 from renku.core.management.command_builder.command import Command
 from renku.core.management.interface.client_dispatcher import IClientDispatcher
 from renku.core.management.interface.database_dispatcher import IDatabaseDispatcher
-from renku.core.utils.git import clone_repository
+from renku.core.utils.git import clone_renku_repository
 
 
 @inject.autoparams()
@@ -38,13 +37,14 @@ def _project_clone(
     config=None,
     raise_git_except=False,
     checkout_revision=None,
+    use_renku_credentials=False,
 ):
     """Clone Renku project repo, install Git hooks and LFS."""
     from renku.core.management.migrate import is_renku_project
 
     install_lfs = client_dispatcher.current_client.external_storage_requested
 
-    repository = clone_repository(
+    repository = clone_renku_repository(
         url=url,
         path=path,
         install_githooks=install_githooks,
@@ -56,6 +56,7 @@ def _project_clone(
         config=config,
         raise_git_except=raise_git_except,
         checkout_revision=checkout_revision,
+        use_renku_credentials=use_renku_credentials,
     )
 
     client_dispatcher.push_client_to_stack(path=repository.path, external_storage_requested=install_lfs)

renku/core/commands/log.py

@@ -21,9 +21,6 @@
 from renku.core.management.command_builder import Command, inject
 from renku.core.management.interface.activity_gateway import IActivityGateway
 
-CONFIG_SECTION = "http"
-RENKU_BACKUP_PREFIX = "renku-backup"
-
 
 def log_command():
     """Return a command for getting a log of renku commands."""

renku/core/commands/login.py

@@ -22,17 +22,21 @@
 import urllib
 import uuid
 import webbrowser
+from typing import TYPE_CHECKING
 
 from renku.core import errors
 from renku.core.management.command_builder import Command, inject
 from renku.core.management.interface.client_dispatcher import IClientDispatcher
 from renku.core.models.enums import ConfigFilter
 from renku.core.utils import communication
-from renku.core.utils.git import get_remote, get_renku_repo_url
+from renku.core.utils.git import RENKU_BACKUP_PREFIX, create_backup_remote, get_remote, get_renku_repo_url
 from renku.core.utils.urls import parse_authentication_endpoint
 
+if TYPE_CHECKING:
+    from renku.core.metadata.repository import Repository
+
+
 CONFIG_SECTION = "http"
-RENKU_BACKUP_PREFIX = "renku-backup"
 
 
 def login_command():
@@ -83,8 +87,22 @@ def _login(endpoint, git_login, yes, client_dispatcher: IClientDispatcher):
         _store_token(parsed_endpoint.netloc, access_token)
 
         if git_login:
-            _store_git_credential_helper(parsed_endpoint.netloc)
-            _swap_git_remote(parsed_endpoint, remote_name, remote_url)
+            _set_git_credential_helper(repository=client.repository, hostname=parsed_endpoint.netloc)
+            backup_remote_name, backup_exists, remote = create_backup_remote(
+                repository=client.repository, remote_name=remote_name, url=remote_url
+            )
+            if backup_exists:
+                communication.echo(f"Backup remote '{backup_remote_name}' already exists. Ignoring '--git' flag.")
+            elif not remote:
+                communication.error(f"Cannot create backup remote '{backup_remote_name}' for '{remote_url}'")
+            else:
+                _set_renku_url_for_remote(
+                    repository=client.repository,
+                    remote_name=remote_name,
+                    remote_url=remote_url,
+                    hostname=parsed_endpoint.netloc,
+                )
+
     else:
         communication.error(
             f"Remote host did not return an access token: {parsed_endpoint.geturl()}, "
@@ -93,7 +111,6 @@ def _login(endpoint, git_login, yes, client_dispatcher: IClientDispatcher):
         sys.exit(1)
 
 
-@inject.autoparams()
 def _parse_endpoint(endpoint):
     parsed_endpoint = parse_authentication_endpoint(endpoint=endpoint)
     if not parsed_endpoint:
@@ -115,33 +132,19 @@ def _store_token(netloc, access_token, client_dispatcher: IClientDispatcher):
     os.chmod(client.global_config_path, 0o600)
 
 
-@inject.autoparams()
-def _store_git_credential_helper(netloc, client_dispatcher: IClientDispatcher):
-    with client_dispatcher.current_client.repository.get_configuration(writable=True) as config:
-        config.set_value("credential", "helper", f"!renku token --hostname {netloc}")
-
+def _set_git_credential_helper(repository: "Repository", hostname):
+    with repository.get_configuration(writable=True) as config:
+        config.set_value("credential", "helper", f"!renku credentials --hostname {hostname}")
 
-@inject.autoparams()
-def _swap_git_remote(parsed_endpoint, remote_name, remote_url, client_dispatcher: IClientDispatcher):
-    client = client_dispatcher.current_client
-    backup_remote_name = f"{RENKU_BACKUP_PREFIX}-{remote_name}"
-
-    if backup_remote_name in [r.name for r in client.repository.remotes]:
-        communication.echo(f"Backup remove '{backup_remote_name}' already exists. Ignoring '--git' flag.")
-        return
 
-    new_remote_url = get_renku_repo_url(remote_url, deployment_hostname=parsed_endpoint.netloc)
+def _set_renku_url_for_remote(repository: "Repository", remote_name: str, remote_url: str, hostname: str):
+    """Set renku repository URL for ``remote_name``."""
+    new_remote_url = get_renku_repo_url(remote_url, deployment_hostname=hostname)
 
     try:
-        client.repository.remotes.add(name=backup_remote_name, url=remote_url)
-    except errors.GitCommandError:
-        communication.error(f"Cannot create backup remote '{backup_remote_name}' for '{remote_url}'")
-    else:
-        try:
-            client.repository.remotes[remote_name].set_url(url=new_remote_url)
-        except errors.GitCommandError as e:
-            client.repository.remotes.remove(backup_remote_name)
-            raise errors.GitError(f"Cannot change remote url for '{remote_name}' to {new_remote_url}") from e
+        repository.remotes[remote_name].set_url(url=new_remote_url)
+    except errors.GitCommandError as e:
+        raise errors.GitError(f"Cannot change remote url for '{remote_name}' to '{new_remote_url}'") from e
 
 
 @inject.autoparams()
@@ -174,24 +177,24 @@ def _logout(endpoint, client_dispatcher: IClientDispatcher):
     else:
         key = "*"
 
-    client_dispatcher.current_client.remove_value(section=CONFIG_SECTION, key=key, global_only=True)
-    _remove_git_credential_helper()
-    _restore_git_remote()
+    client = client_dispatcher.current_client
+    client.remove_value(section=CONFIG_SECTION, key=key, global_only=True)
+    _remove_git_credential_helper(client=client)
+    _restore_git_remote(client=client)
 
 
-@inject.autoparams()
-def _remove_git_credential_helper(client_dispatcher: IClientDispatcher):
-    with client_dispatcher.current_client.repository.get_configuration(writable=True) as config:
+def _remove_git_credential_helper(client):
+    if not client.repository:  # Outside a renku project
+        return
+
+    with client.repository.get_configuration(writable=True) as config:
         try:
             config.remove_value("credential", "helper")
         except errors.GitError:  # NOTE: If already logged out, an exception is raised
             pass
 
 
-@inject.autoparams()
-def _restore_git_remote(client_dispatcher: IClientDispatcher):
-    client = client_dispatcher.current_client
-
+def _restore_git_remote(client):
     if not client.repository:  # Outside a renku project
         return
 
@@ -211,13 +214,13 @@ def _restore_git_remote(client_dispatcher: IClientDispatcher):
             communication.error(f"Cannot delete backup remote '{backup_remote}'")
 
 
-def token_command():
+def credentials_command():
     """Return a command as git credential helper."""
-    return Command().command(_token)
+    return Command().command(_credentials)
 
 
 @inject.autoparams()
-def _token(command, hostname, client_dispatcher: IClientDispatcher):
+def _credentials(command, hostname, client_dispatcher: IClientDispatcher):
     if command != "get":
         return
 

renku/core/commands/providers/renku.py

@@ -359,9 +359,10 @@ def _fetch_dataset(self, client_dispatcher: IClientDispatcher, database_dispatch
                     url=url,
                     path=get_cache_directory_for_repository(client=client, url=url),
                     gitlab_token=self._gitlab_token,
-                    renku_token=self._renku_token,
                     deployment_hostname=parsed_uri.netloc,
                     depth=None,
+                    reuse_existing_repository=True,
+                    use_renku_credentials=True,
                 )
             except errors.GitError:
                 pass

renku/core/errors.py

@@ -325,6 +325,10 @@ class GitError(RenkuException):
     """Raised when a Git operation fails."""
 
 
+class InvalidGitURL(GitError):
+    """Raise when a Git URL is not valid."""
+
+
 class GitCommandError(GitError):
     """Raised when a Git command fails."""
 

renku/core/metadata/repository.py

@@ -696,6 +696,7 @@ def clone_from(
         progress: Optional[Callable] = None,
         no_checkout: bool = False,
         env: dict = None,
+        clone_options: List[str] = None,
     ) -> "Repository":
         """Clone a remote repository and create an instance."""
         try:
@@ -708,6 +709,7 @@ def clone_from(
                 progress=progress,
                 no_checkout=no_checkout,
                 env=env,
+                multi_options=clone_options,
             )
         except git.GitCommandError as e:
             raise errors.GitCommandError(

renku/core/models/git.py

@@ -27,7 +27,7 @@
 from renku.core import errors
 from renku.core.utils.os import is_ascii, normalize_to_ascii
 
-_RE_PROTOCOL = r"(?P<protocol>(git\+)?(https?|git|ssh|rsync))\://"
+_RE_SCHEME = r"(?P<scheme>(git\+)?(https?|git|ssh|rsync))\://"
 
 _RE_USERNAME = r"(?:(?P<username>.+)@)?"
 
@@ -43,14 +43,14 @@
 
 _RE_PORT = r":(?P<port>\d+)"
 
-_RE_PATHNAME = r"(?P<pathname>(([\w\-\~\.]+)/)*?(((?P<owner>([\w\-\.]+/?)+)/)?(?P<name>[\w\-\.]+)(\.git)?)?)/*"
+_RE_PATHNAME = r"(?P<path>(([\w\-\~\.]+)/)*?(((?P<owner>([\w\-\.]+/?)+)/)?(?P<name>[\w\-\.]+)(\.git)?)?)/*"
 
 _RE_PATHNAME_WITH_GITLAB = (
-    r"(?P<pathname>((((gitlab/){0,1}|([\w\-\~\.]+/)*?)(?P<owner>([\w\-\.]+/)*[\w\-\.]+)/)?"
+    r"(?P<path>((((gitlab/){0,1}|([\w\-\~\.]+/)*?)(?P<owner>([\w\-\.]+/)*[\w\-\.]+)/)?"
     r"(?P<name>[\w\-\.]+)(\.git)?)?)/*"
 )
 
-_RE_UNIXPATH = r"(file\://)?(?P<pathname>\/$|((?=\/)|\.|\.\.)(\/(?=[^/\0])[^/\0]+)*\/?)"
+_RE_UNIXPATH = r"(file\://)?(?P<path>\/$|((?=\/)|\.|\.\.)(\/(?=[^/\0])[^/\0]+)*\/?)"
 
 
 def _build(*parts):
@@ -61,9 +61,9 @@ def _build(*parts):
 #: Define possible repository URLs.
 _REPOSITORY_URLS = (
     # https://user:pass@example.com/owner/repo.git
-    _build(_RE_PROTOCOL, _RE_USERNAME_PASSWORD, _RE_HOSTNAME, r"/", _RE_PATHNAME_WITH_GITLAB),
+    _build(_RE_SCHEME, _RE_USERNAME_PASSWORD, _RE_HOSTNAME, r"/", _RE_PATHNAME_WITH_GITLAB),
     # https://user:pass@example.com:gitlab/owner/repo.git
-    _build(_RE_PROTOCOL, _RE_USERNAME_PASSWORD, _RE_HOSTNAME, _RE_PORT, r"/", _RE_PATHNAME_WITH_GITLAB),
+    _build(_RE_SCHEME, _RE_USERNAME_PASSWORD, _RE_HOSTNAME, _RE_PORT, r"/", _RE_PATHNAME_WITH_GITLAB),
     # git@example.com:owner/repo.git
     _build(_RE_USERNAME, _RE_HOSTNAME, r":", _RE_PATHNAME_WITH_GITLAB),
     # /path/to/repo
@@ -84,10 +84,8 @@ class GitURL(object):
 
     # Initial value
     href = attr.ib()
-    # Parsed protocols
-    pathname = attr.ib(default=None)
-    protocols = attr.ib(default=attr.Factory(list), init=False)
-    protocol = attr.ib(default="ssh")
+    path = attr.ib(default=None)
+    scheme = attr.ib(default="ssh")
     hostname = attr.ib(default="localhost")
     username = attr.ib(default=None)
     password = attr.ib(default=None)
@@ -99,18 +97,13 @@ class GitURL(object):
 
     def __attrs_post_init__(self):
         """Derive basic information."""
-        if self.protocol:
-            protocols = self.protocol.split("+")
-            self.protocols = protocols
-            self.protocol = protocols[-1]
-
-        if not self.name and self.pathname:
-            self.name = filter_repo_name(Path(self.pathname).name)
+        if not self.name and self.path:
+            self.name = filter_repo_name(Path(self.path).name)
 
         self.slug = normalize_to_ascii(self.name)
 
     @classmethod
-    def parse(cls, href):
+    def parse(cls, href) -> "GitURL":
         """Derive URI components."""
         if not is_ascii(href):
             raise UnicodeError(f"`{href}` is not a valid Git remote")
@@ -123,7 +116,7 @@ def parse(cls, href):
             # NOTE: use known gitlab url to simplify regex to make detection more robust
             gitlab_url = urlparse(gitlab_url)
             gitlab_re = _build(
-                _RE_PROTOCOL,
+                _RE_SCHEME,
                 _RE_USERNAME_PASSWORD,
                 r"(?P<hostname>" + re.escape(gitlab_url.hostname) + ")",
                 r":(?P<port>" + str(gitlab_url.port) + ")" if gitlab_url.port else "",
@@ -138,14 +131,14 @@ def parse(cls, href):
             if matches:
                 return cls(href=href, regex=regex, **matches.groupdict())
         else:
-            raise errors.GitConfigurationError(f"`{href}` is not a valid Git remote")
+            raise errors.InvalidGitURL(f"`{href}` is not a valid Git URL")
 
     @property
     def instance_url(self):
         """Get the url of the git instance."""
         url = urlparse(self.href)
 
-        path = self.pathname.split(self.owner, 1)[0]
+        path = self.path.split(self.owner, 1)[0]
         url = url._replace(path=path)
 
         return url.geturl()