diff --git a/renku/cli/__init__.py b/renku/cli/__init__.py index 3a33d746d6..e6dfe7d750 100644 --- a/renku/cli/__init__.py +++ b/renku/cli/__init__.py @@ -69,6 +69,7 @@ import click_completion import yaml +from renku.cli.clone import clone from renku.cli.config import config from renku.cli.dataset import dataset from renku.cli.doctor import doctor @@ -187,6 +188,7 @@ def help(ctx): # Register subcommands: +cli.add_command(clone) cli.add_command(config) cli.add_command(dataset) cli.add_command(doctor) diff --git a/renku/cli/clone.py b/renku/cli/clone.py new file mode 100644 index 0000000000..e946406fab --- /dev/null +++ b/renku/cli/clone.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018-2019- Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Clone a Renku project. + +Cloning a Renku project +~~~~~~~~~~~~~~~~~~~~~~~ + +To clone a Renku project and set up required Git hooks and Git LFS use +``renku clone`` command. + +.. code-block:: console + + $ renku clone git+ssh://host.io/namespace/project.git + + +""" + +import click + +from renku.core.commands.clone import renku_clone +from renku.core.commands.echo import GitProgress + + +@click.command() +@click.option( + '--pull-data', is_flag=True, help='Pull data from Git-LFS.', default=False +) +@click.argument('url') +@click.argument('path', required=False, default=None) +def clone(pull_data, url, path): + """Clone a Renku repository.""" + click.echo('Cloning {} ...'.format(url)) + + skip_smudge = not pull_data + renku_clone( + url=url, path=path, skip_smudge=skip_smudge, progress=GitProgress() + ) + click.secho('OK', fg='green') diff --git a/renku/cli/githooks.py b/renku/cli/githooks.py index 1cf2da6e0b..3ced3f7c91 100644 --- a/renku/cli/githooks.py +++ b/renku/cli/githooks.py @@ -38,14 +38,9 @@ """ -import stat -from pathlib import Path - import click -from renku.core.commands.client import pass_local_client - -HOOKS = ('pre-commit', ) +from renku.core.commands.githooks import install_githooks, uninstall_githooks @click.group() @@ -55,42 +50,14 @@ def githooks(): @githooks.command() @click.option('--force', is_flag=True, help='Override existing hooks.') -@pass_local_client -def install(client, force): +def install(force): """Install Git hooks.""" - import pkg_resources - from git.index.fun import hook_path as get_hook_path - - for hook in HOOKS: - hook_path = Path(get_hook_path(hook, client.repo.git_dir)) - if hook_path.exists(): - if not force: - click.echo( - 'Hook already exists. Skipping {0}'.format(str(hook_path)), - err=True - ) - continue - else: - hook_path.unlink() - - # Make sure the hooks directory exists. - hook_path.parent.mkdir(parents=True, exist_ok=True) - - Path(hook_path).write_bytes( - pkg_resources.resource_string( - 'renku.data', '{hook}.sh'.format(hook=hook) - ) - ) - hook_path.chmod(hook_path.stat().st_mode | stat.S_IEXEC) + install_githooks(force) + click.secho('OK', fg='green') @githooks.command() -@pass_local_client -def uninstall(client): +def uninstall(): """Uninstall Git hooks.""" - from git.index.fun import hook_path as get_hook_path - - for hook in HOOKS: - hook_path = Path(get_hook_path(hook, client.repo.git_dir)) - if hook_path.exists(): - hook_path.unlink() + uninstall_githooks() + click.secho('OK', fg='green') diff --git a/renku/core/commands/clone.py b/renku/core/commands/clone.py new file mode 100644 index 0000000000..d3805640a5 --- /dev/null +++ b/renku/core/commands/clone.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018-2019- Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Clone a Renku repo along with all Renku-specific initializations.""" + +from renku.core.management.clone import clone + +from .client import pass_local_client + + +@pass_local_client +def renku_clone( + client, + url, + path=None, + install_githooks=True, + skip_smudge=True, + progress=None +): + """Clone Renku project repo, install Git hooks and LFS.""" + install_lfs = client.use_external_storage + clone( + url=url, + path=path, + install_githooks=install_githooks, + install_lfs=install_lfs, + skip_smudge=skip_smudge, + progress=progress + ) diff --git a/renku/core/commands/echo.py b/renku/core/commands/echo.py index b4ce1f62ee..93204341de 100644 --- a/renku/core/commands/echo.py +++ b/renku/core/commands/echo.py @@ -21,6 +21,7 @@ import os import click +from git.remote import RemoteProgress WARNING = click.style('Warning: ', bold=True, fg='yellow') @@ -45,3 +46,23 @@ def echo_via_pager(*args, **kwargs): show_pos=True, item_show_func=lambda x: x, ) + + +class GitProgress(RemoteProgress): + """Progress printing for GitPython.""" + + def __init__(self): + """Initialize a Git progress printer.""" + super().__init__() + self._previous_line_length = 0 + + def update(self, op_code, cur_count, max_count=None, message=''): + """Callback for printing Git operation status.""" + self._clear_line() + print(self._cur_line, end='\r') + self._previous_line_length = len(self._cur_line) + if (op_code & RemoteProgress.END) != 0: + print() + + def _clear_line(self): + print(self._previous_line_length * ' ', end='\r') diff --git a/renku/core/commands/githooks.py b/renku/core/commands/githooks.py new file mode 100644 index 0000000000..281e65df8e --- /dev/null +++ b/renku/core/commands/githooks.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018-2019- Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Install and uninstall Git hooks.""" + +import click + +from renku.core.management.githooks import install, uninstall + +from .client import pass_local_client +from .echo import WARNING + + +@pass_local_client +def install_githooks(client, force): + """Install Git hooks.""" + warning_messages = install(client=client, force=force) + if warning_messages: + for message in warning_messages: + click.echo(WARNING + message) + + +@pass_local_client +def uninstall_githooks(client): + """Uninstall Git hooks.""" + uninstall(client=client) diff --git a/renku/core/management/clone.py b/renku/core/management/clone.py new file mode 100644 index 0000000000..5c4f900587 --- /dev/null +++ b/renku/core/management/clone.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018-2019- Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Clone a Renku repo along with all Renku-specific initializations.""" + +import os + +from git import GitCommandError, Repo + +from renku.core import errors +from renku.core.management.githooks import install + + +def clone( + url, + path=None, + install_githooks=True, + install_lfs=True, + skip_smudge=True, + recursive=True, + depth=None, + progress=None +): + """Clone Renku project repo, install Git hooks and LFS.""" + from renku.core.management.client import LocalClient + + path = path or '.' + # Clone the project + if skip_smudge: + os.environ['GIT_LFS_SKIP_SMUDGE'] = '1' + try: + repo = Repo.clone_from( + url, path, recursive=recursive, depth=depth, progress=progress + ) + except GitCommandError as e: + raise errors.GitError( + 'Cannot clone remote Renku project: {}'.format(url) + ) from e + + client = LocalClient(path) + + if install_githooks: + install(client=client, force=True) + + if install_lfs: + command = ['git', 'lfs', 'install', '--local', '--force'] + if skip_smudge: + command += ['--skip-smudge'] + try: + repo.git.execute(command=command, with_exceptions=True) + except GitCommandError as e: + raise errors.GitError('Cannot install Git LFS') from e + + return repo diff --git a/renku/core/management/datasets.py b/renku/core/management/datasets.py index c1b867b081..b7ecd3fd3c 100644 --- a/renku/core/management/datasets.py +++ b/renku/core/management/datasets.py @@ -34,6 +34,7 @@ from git import GitCommandError, GitError, Repo from renku.core import errors +from renku.core.management.clone import clone from renku.core.management.config import RENKU_HOME from renku.core.models.datasets import Dataset, DatasetFile, DatasetTag from renku.core.models.git import GitURL @@ -755,9 +756,13 @@ def checkout(repo, ref): repo = Repo(str(repo_path)) if repo.remotes.origin.url == url: try: - repo.git.fetch() + repo.git.fetch(all=True) repo.git.checkout(ref) - repo.git.pull() + try: + repo.git.pull() + except GitError: + # When ref is not a branch, an error is thrown + pass except GitError: # ignore the error and try re-cloning pass @@ -772,25 +777,17 @@ def checkout(repo, ref): format(repo_path) ) + repo = clone(url, path=str(repo_path), install_githooks=False) + + # Because the name of the default branch is not always 'master', we + # create an alias of the default branch when cloning the repo. It + # is used to refer to the default branch later. + renku_ref = 'refs/heads/' + RENKU_BRANCH try: - os.environ['GIT_LFS_SKIP_SMUDGE'] = '1' - repo = Repo.clone_from(url, str(repo_path), recursive=True) - # Because the name of the default branch is not always 'master', we - # create an alias of the default branch when cloning the repo. It - # is used to refer to the default branch later. - renku_ref = 'refs/heads/' + RENKU_BRANCH repo.git.execute([ 'git', 'symbolic-ref', renku_ref, repo.head.reference.path ]) checkout(repo, ref) - # Disable Git LFS smudge filter - repo.git.execute( - command=[ - 'git', 'lfs', 'install', '--local', '--skip-smudge', - '--force' - ], - with_exceptions=False - ) except GitCommandError as e: raise errors.GitError( 'Cannot clone remote Git repo: {}'.format(url) diff --git a/renku/core/management/githooks.py b/renku/core/management/githooks.py new file mode 100644 index 0000000000..b470b98ee3 --- /dev/null +++ b/renku/core/management/githooks.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2018-2019- Swiss Data Science Center (SDSC) +# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Install and uninstall Git hooks.""" + +import stat +from pathlib import Path + +import pkg_resources +from git.index.fun import hook_path as get_hook_path + +HOOKS = ('pre-commit', ) + + +def install(client, force): + """Install Git hooks.""" + warning_messages = [] + for hook in HOOKS: + hook_path = Path(get_hook_path(hook, client.repo.git_dir)) + if hook_path.exists(): + if not force: + warning_messages.append( + 'Hook already exists. Skipping {0}'.format(str(hook_path)) + ) + continue + else: + hook_path.unlink() + + # Make sure the hooks directory exists. + hook_path.parent.mkdir(parents=True, exist_ok=True) + + Path(hook_path).write_bytes( + pkg_resources.resource_string( + 'renku.data', '{hook}.sh'.format(hook=hook) + ) + ) + hook_path.chmod(hook_path.stat().st_mode | stat.S_IEXEC) + + return warning_messages + + +def uninstall(client): + """Uninstall Git hooks.""" + for hook in HOOKS: + hook_path = Path(get_hook_path(hook, client.repo.git_dir)) + if hook_path.exists(): + hook_path.unlink() diff --git a/tests/cli/test_integration_datasets.py b/tests/cli/test_integration_datasets.py index a823ea76db..85ceebfab2 100644 --- a/tests/cli/test_integration_datasets.py +++ b/tests/cli/test_integration_datasets.py @@ -903,3 +903,30 @@ def test_files_are_tracked_in_lfs(runner, client): assert 0 == result.exit_code path = 'data/dataset/{}'.format(FILENAME) assert path in subprocess.check_output(['git', 'lfs', 'ls-files']).decode() + + +@pytest.mark.integration +def test_renku_clone(runner, monkeypatch): + """Test cloning of a Renku repo and existence of required settings.""" + from renku.core.management.storage import StorageApiMixin + + REMOTE = 'git@dev.renku.ch:virginiafriedrich/datasets-test.git' + + with runner.isolated_filesystem() as project_path: + result = runner.invoke(cli, ['clone', REMOTE, project_path]) + assert 0 == result.exit_code + assert (Path(project_path) / 'Dockerfile').exists() + + # Check Git hooks are installed + result = runner.invoke(cli, ['githooks', 'install']) + assert 0 == result.exit_code + assert 'Hook already exists.' in result.output + + # Check Git LFS is enabled + with monkeypatch.context() as monkey: + # Pretend that git-lfs is not installed. + monkey.setattr(StorageApiMixin, 'storage_installed', False) + # Repo is using external storage but it's not installed. + result = runner.invoke(cli, ['run', 'touch', 'output']) + assert 'is not configured' in result.output + assert 1 == result.exit_code