From 3ef6fc39c9ba0a012c2637518bd60786333c4c62 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 11:48:24 -0500 Subject: [PATCH 01/12] Bump libvcs to 0.3.1 - to allow overwriting remote URLs --- poetry.lock | 6 +++--- pyproject.toml | 2 +- requirements/base.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index ca95adef2..18a60a9a8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -379,7 +379,7 @@ description = "vcs abstraction layer" name = "libvcs" optional = false python-versions = "~2.7 || ^3.5" -version = "0.3.1.post1" +version = "0.3.2" [[package]] category = "dev" @@ -1058,7 +1058,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pathlib2", "unittest2", "jaraco.itertools", "func-timeout"] [metadata] -content-hash = "792b8ce7a7270e19326864b99b218f0e4375483d9de7122522058db0d1aa7409" +content-hash = "c8f6cbd05695fb929d3e4fbb8a07742936e22dbac016fb1be80ac131fcce4e15" lock-version = "1.0" python-versions = "~2.7 || ^3.5" @@ -1216,7 +1216,7 @@ kaptan = [ {file = "kaptan-0.5.12.tar.gz", hash = "sha256:1abd1f56731422fce5af1acc28801677a51e56f5d3c3e8636db761ed143c3dd2"}, ] libvcs = [ - {file = "libvcs-0.3.1.post1.tar.gz", hash = "sha256:2bfec57d4048670da300c91c50b156b390668dc9d341e113887cbae5d470241f"}, + {file = "libvcs-0.3.2.tar.gz", hash = "sha256:14d7337304bc164662b191dbaf086fd0abb95e4a18f6246a25fef8a8f6959e97"}, ] markupsafe = [ {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"}, diff --git a/pyproject.toml b/pyproject.toml index 3abdc7263..0d0b1ff8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ authors = ["Tony Narlock "] python = "~2.7 || ^3.5" click = ">=7<8" kaptan = "*" -libvcs = "==0.3.1post1" +libvcs = "==0.3.2" colorama = ">=0.3.9" [tool.poetry.dev-dependencies] diff --git a/requirements/base.txt b/requirements/base.txt index a760b16f0..845191196 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ kaptan>=0.5.9,<1 click>=7<8 colorama>=0.3.9 -libvcs>=0.3.0,<0.4.0 +libvcs>=0.3.2,<0.4.0 From 7c9287f3fb89222fea0fb8f7457aea99313d9741 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 11:49:40 -0500 Subject: [PATCH 02/12] Update CHANGES --- CHANGES | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES b/CHANGES index 135123ce3..9f3f2289d 100644 --- a/CHANGES +++ b/CHANGES @@ -6,6 +6,7 @@ Here you can find the recent changes to vcspull current ------- +- :issue:`231` Add updating / merging of remote URLs (via PR :issue:`231`) - Fix colorama constraint - poetry lockfile: Fix (accidentally pushed lockfile via prerelease version of poetry) From db686a0b9fd71e3486c0b5d6d8017cd85a488854 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 11:50:43 -0500 Subject: [PATCH 03/12] Overwrite remote URLs with state set in config --- vcspull/cli.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/vcspull/cli.py b/vcspull/cli.py index 7431f67e6..d1b13549d 100644 --- a/vcspull/cli.py +++ b/vcspull/cli.py @@ -127,6 +127,25 @@ def update_repo(repo_dict): r = create_repo_from_pip_url(**repo_dict) + remote_settings = repo_dict.get('remotes', []) + if not any(rs for rs in remote_settings if rs['remote_name'] == 'origin'): + remote_settings.append({'remote_name': 'origin', 'url': repo_dict['pip_url']}) + + for remote_setting in remote_settings: + config_remote_name = remote_setting['remote_name'] # From config file + current_remote = r.remotes_get.get(config_remote_name) + + if current_remote is not None and current_remote[0] != remote_setting['url']: + print( + 'Ovewrriting {name} ({current_url}) with {new_url}'.format( + name=config_remote_name, + current_url=current_remote[0], + new_url=remote_setting['url'], + ) + ) + r.remote_set( + name=config_remote_name, url=remote_setting['url'], overwrite=True + ) r.update_repo() From 8a7343a813a89847be4cd6df9377c84067d128b6 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 12:11:41 -0500 Subject: [PATCH 04/12] update_repo: Return repo object --- vcspull/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vcspull/cli.py b/vcspull/cli.py index d1b13549d..3afc48e6d 100644 --- a/vcspull/cli.py +++ b/vcspull/cli.py @@ -147,6 +147,7 @@ def update_repo(repo_dict): name=config_remote_name, url=remote_setting['url'], overwrite=True ) r.update_repo() + return r cli.add_command(update) From e4c55a1b76f594b30d3257d97e2887ebf106cf20 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 12:15:39 -0500 Subject: [PATCH 05/12] tests: Make git dummy repo into factory --- tests/conftest.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f7d00de8b..ff07233a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,17 +34,22 @@ def git_repo(git_repo_kwargs): @pytest.fixture -def git_dummy_repo_dir(tmpdir_repoparent, scope='session'): - """Create a git repo with 1 commit, used as a remote.""" - name = 'dummyrepo' - repo_path = str(tmpdir_repoparent.join(name)) +def create_git_dummy_repo(tmpdir_repoparent): + def fn(repo_name, testfile_filename='testfile.test'): + repo_path = str(tmpdir_repoparent.join(repo_name)) + + run(['git', 'init', repo_name], cwd=str(tmpdir_repoparent)) + + run(['touch', testfile_filename], cwd=repo_path) + run(['git', 'add', testfile_filename], cwd=repo_path) + run(['git', 'commit', '-m', 'test file for %s' % repo_name], cwd=repo_path) - run(['git', 'init', name], cwd=str(tmpdir_repoparent)) + return repo_path - testfile_filename = 'testfile.test' + yield fn - run(['touch', testfile_filename], cwd=repo_path) - run(['git', 'add', testfile_filename], cwd=repo_path) - run(['git', 'commit', '-m', 'test file for %s' % name], cwd=repo_path) - return repo_path +@pytest.fixture +def git_dummy_repo_dir(tmpdir_repoparent, create_git_dummy_repo): + """Create a git repo with 1 commit, used as a remote.""" + return create_git_dummy_repo('dummyrepo') From ad1e9f3942b9e49d55aa58c7270e5bda780fd76c Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 12:40:25 -0500 Subject: [PATCH 06/12] update_repo: Add more protections when iterating through config Also handle case where repo isn't create yet and skip out of git remote configuration if it doesn't exist yet --- vcspull/cli.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vcspull/cli.py b/vcspull/cli.py index 3afc48e6d..65d8a8a61 100644 --- a/vcspull/cli.py +++ b/vcspull/cli.py @@ -122,7 +122,8 @@ def progress_cb(output, timestamp): def update_repo(repo_dict): - repo_dict['pip_url'] = repo_dict.pop('url') + if 'pip_url' not in repo_dict: + repo_dict['pip_url'] = repo_dict.pop('url') repo_dict['progress_callback'] = progress_cb r = create_repo_from_pip_url(**repo_dict) @@ -133,7 +134,10 @@ def update_repo(repo_dict): for remote_setting in remote_settings: config_remote_name = remote_setting['remote_name'] # From config file - current_remote = r.remotes_get.get(config_remote_name) + try: + current_remote = r.remotes_get.get(config_remote_name) + except FileNotFoundError: # git repo doesn't exist yet, so cna't be outdated + break if current_remote is not None and current_remote[0] != remote_setting['url']: print( From f903f26e18c51e02641c0e1a5b90adccc66b56bf Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 12:42:38 -0500 Subject: [PATCH 07/12] Add code comment for clarity --- vcspull/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vcspull/cli.py b/vcspull/cli.py index 65d8a8a61..d112e255b 100644 --- a/vcspull/cli.py +++ b/vcspull/cli.py @@ -126,7 +126,7 @@ def update_repo(repo_dict): repo_dict['pip_url'] = repo_dict.pop('url') repo_dict['progress_callback'] = progress_cb - r = create_repo_from_pip_url(**repo_dict) + r = create_repo_from_pip_url(**repo_dict) # Creates the repo object remote_settings = repo_dict.get('remotes', []) if not any(rs for rs in remote_settings if rs['remote_name'] == 'origin'): @@ -150,7 +150,7 @@ def update_repo(repo_dict): r.remote_set( name=config_remote_name, url=remote_setting['url'], overwrite=True ) - r.update_repo() + r.update_repo() # Creates repo if not exists and fetches return r From b1c4856f729d7fd48025edec1ec8d33bcf3246be Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 12:42:47 -0500 Subject: [PATCH 08/12] Stub out initial test for outdated remotes --- tests/test_repo_object.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/test_repo_object.py b/tests/test_repo_object.py index 204319c19..62432a199 100644 --- a/tests/test_repo_object.py +++ b/tests/test_repo_object.py @@ -8,7 +8,8 @@ from libvcs import BaseRepo, GitRepo, MercurialRepo, SubversionRepo from libvcs.shortcuts import create_repo_from_pip_url -from vcspull.config import extract_repos, filter_repos +from vcspull.cli import update_repo +from vcspull.config import extract_repos, filter_repos, load_configs from .fixtures import example as fixtures @@ -141,3 +142,25 @@ def test_makes_recursive(tmpdir, git_dummy_repo_dir): for r in filter_repos(repos): repo = create_repo_from_pip_url(**r) repo.obtain() + + +def test_updating_remote(tmpdir, create_git_dummy_repo): + """Ensure that directories in pull are made recursively.""" + dummy_repo = create_git_dummy_repo('dummyrepo') + + YAML_CONFIG = """ + {TMP_DIR}/study/myrepo: + my_url: git+file://{REPO_DIR} + """ + + YAML_CONFIG = YAML_CONFIG.format(TMP_DIR=str(tmpdir), REPO_DIR=dummy_repo) + CONFIG_FILENAME = 'myrepos.yaml' + config_file = tmpdir.join(CONFIG_FILENAME) + config_file.write(YAML_CONFIG) + repo_parent = tmpdir.join('study/myrepo') + repo_parent.ensure(dir=True) + + configs = load_configs([str(config_file)]) + + for repo_dict in filter_repos(configs, repo_dir='*', vcs_url='*', name='*'): + update_repo(repo_dict) From 7ea7f6ab0c38acd7606b41ae39a99e32689a8e8c Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 12:58:45 -0500 Subject: [PATCH 09/12] Fix typo, update language with setting outdated remotes --- vcspull/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vcspull/cli.py b/vcspull/cli.py index d112e255b..0e48671dd 100644 --- a/vcspull/cli.py +++ b/vcspull/cli.py @@ -141,7 +141,7 @@ def update_repo(repo_dict): if current_remote is not None and current_remote[0] != remote_setting['url']: print( - 'Ovewrriting {name} ({current_url}) with {new_url}'.format( + 'Updating remote {name} ({current_url}) with {new_url}'.format( name=config_remote_name, current_url=current_remote[0], new_url=remote_setting['url'], From 71213e5afacff57d28dd39b5b04c04e26b922ea3 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 12:59:14 -0500 Subject: [PATCH 10/12] update_repo: Ensure dict isn't mutated --- vcspull/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vcspull/cli.py b/vcspull/cli.py index 0e48671dd..f47c2d929 100644 --- a/vcspull/cli.py +++ b/vcspull/cli.py @@ -9,6 +9,7 @@ import logging import sys +from copy import deepcopy import click @@ -122,6 +123,7 @@ def progress_cb(output, timestamp): def update_repo(repo_dict): + repo_dict = deepcopy(repo_dict) if 'pip_url' not in repo_dict: repo_dict['pip_url'] = repo_dict.pop('url') repo_dict['progress_callback'] = progress_cb From 361b8389bf1e8a6152690fec85c61c0182baee14 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 13:01:18 -0500 Subject: [PATCH 11/12] Add tests for updating remotes --- tests/test_repo_object.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/tests/test_repo_object.py b/tests/test_repo_object.py index 62432a199..92173bf76 100644 --- a/tests/test_repo_object.py +++ b/tests/test_repo_object.py @@ -146,21 +146,36 @@ def test_makes_recursive(tmpdir, git_dummy_repo_dir): def test_updating_remote(tmpdir, create_git_dummy_repo): """Ensure that directories in pull are made recursively.""" - dummy_repo = create_git_dummy_repo('dummyrepo') - YAML_CONFIG = """ - {TMP_DIR}/study/myrepo: - my_url: git+file://{REPO_DIR} - """ + def create_and_load_configs(repo_name): + dummy_repo = create_git_dummy_repo(repo_name) + + YAML_CONFIG = """ + {TMP_DIR}/study/myrepo: + my_url: git+file://{REPO_DIR} + """ - YAML_CONFIG = YAML_CONFIG.format(TMP_DIR=str(tmpdir), REPO_DIR=dummy_repo) - CONFIG_FILENAME = 'myrepos.yaml' - config_file = tmpdir.join(CONFIG_FILENAME) - config_file.write(YAML_CONFIG) - repo_parent = tmpdir.join('study/myrepo') - repo_parent.ensure(dir=True) + YAML_CONFIG = YAML_CONFIG.format(TMP_DIR=str(tmpdir), REPO_DIR=dummy_repo) + CONFIG_FILENAME = 'myrepos.yaml' + config_file = tmpdir.join(CONFIG_FILENAME) + config_file.write(YAML_CONFIG) + repo_parent = tmpdir.join('study/myrepo') + repo_parent.ensure(dir=True) + return config_file + + config_file = create_and_load_configs('dummyrepo') + configs = load_configs([str(config_file)]) + + for repo_dict in filter_repos(configs, repo_dir='*', vcs_url='*', name='*'): + old_repo_remotes = update_repo(repo_dict).remotes_get['origin'] + # Later: Copy dummy repo somewhere else so the commits are common + config_file = create_and_load_configs('new_repo_url') configs = load_configs([str(config_file)]) for repo_dict in filter_repos(configs, repo_dir='*', vcs_url='*', name='*'): - update_repo(repo_dict) + repo_url = repo_dict['url'].replace('git+', '') + r = update_repo(repo_dict) + current_remote_url = r.remotes_get['origin'] + assert current_remote_url[0] == repo_url + assert current_remote_url != old_repo_remotes From e616ed5bf03cba51dd19a01a9a152efc6bc70d50 Mon Sep 17 00:00:00 2001 From: Tony Narlock Date: Sun, 26 Jul 2020 13:04:17 -0500 Subject: [PATCH 12/12] update_repo: Use EnvironmentError for compatibility --- vcspull/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vcspull/cli.py b/vcspull/cli.py index f47c2d929..f062cbaa3 100644 --- a/vcspull/cli.py +++ b/vcspull/cli.py @@ -13,6 +13,7 @@ import click +from libvcs._compat import PY2 from libvcs.shortcuts import create_repo_from_pip_url from .__about__ import __version__ @@ -20,6 +21,9 @@ from .config import filter_repos, find_config_files, load_configs from .log import DebugLogFormatter, RepoFilter, RepoLogFormatter +if PY2: + FileNotFoundError = EnvironmentError + MIN_ASYNC = 3 # minimum amount of repos to sync concurrently MAX_ASYNC = 8 # maximum processes to open:w