From b92a07872191a06bb7565e3e094ff34dfd4a79b5 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 19 Jul 2019 09:53:22 -0400 Subject: [PATCH 1/2] BF: check special remotes of type git by uuid to (not) inform if they are not enabled Matching by the name is not sufficient, since when we just clone such repository it would get an arbitrary (by default "origin") name which would not match the one git-annex calls it by. Checking by annex UUID seems to be the best way to proceed --- datalad/distribution/tests/test_clone.py | 14 ++++++++++++++ datalad/distribution/utils.py | 16 +++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/datalad/distribution/tests/test_clone.py b/datalad/distribution/tests/test_clone.py index d1eb0b4512..053e44b14c 100644 --- a/datalad/distribution/tests/test_clone.py +++ b/datalad/distribution/tests/test_clone.py @@ -16,6 +16,7 @@ slow ) +import logging import os from os.path import join as opj from os.path import isdir @@ -54,6 +55,7 @@ from datalad.tests.utils import ok_startswith from datalad.tests.utils import ok_clean_git from datalad.tests.utils import serve_path_via_http +from datalad.tests.utils import swallow_logs from datalad.tests.utils import use_cassette from datalad.tests.utils import skip_if_no_network from datalad.tests.utils import skip_if @@ -358,3 +360,15 @@ def test_clone_report_permission_issue(tdir): message="could not create work tree dir '%s/%s': Permission denied" % (pdir, get_datasets_topdir()) ) + + +@skip_if_no_network +@with_tempfile +def test_autoenabled_remote_msg(path): + # Verify that no message about a remote not been enabled is displayed + # whenever the remote we clone is the type=git special remote, so the name + # of the remote might not match + with swallow_logs(new_level=logging.INFO) as cml: + res = clone('///repronim/containers', path) + assert_status('ok', res) + assert_not_in("not auto-enabled", cml.out) \ No newline at end of file diff --git a/datalad/distribution/utils.py b/datalad/distribution/utils.py index dd3ba0be19..fc76b8abc7 100644 --- a/datalad/distribution/utils.py +++ b/datalad/distribution/utils.py @@ -157,9 +157,19 @@ def _handle_possible_annex_dataset(dataset, reckless, description=None): remote_names = repo.get_remotes( with_urls_only=False, exclude_special_remotes=False) - for k, v in repo.get_special_remotes().items(): - sr_name = v.get('name', None) - if sr_name and sr_name not in remote_names: + remote_uuids = None # might be necessary to discover known UUIDs + for uuid, config in repo.get_special_remotes().items(): + sr_name = config.get('name', None) + sr_type = config.get('type', None) + if sr_type == 'git': + # determine either there is a registered remote with matching UUID + if remote_uuids is None: + remote_uuids = { + repo.config.get('remote.%s.annex-uuid' % r) + for r in remote_names + } + if (sr_type != 'git' and sr_name and sr_name not in remote_names) or \ + (sr_type == 'git' and uuid and uuid not in remote_uuids): # if it is not listed among the remotes, it wasn't enabled lgr.info( 'access to dataset sibling "%s" not auto-enabled, enable with:\n\t\tdatalad siblings -d "%s" enable -s %s', From e6c3d9eb23bb22c226094e412d0330c2a19e9a77 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 26 Jul 2019 15:28:43 -0400 Subject: [PATCH 2/2] RF: inform about special remotes based on autoenable config + provide debug level msg about autoenable"ed remotes + collide messages on how to enable special remotes into a single message (no need to flood the screen) + do not report anything about remotes UUIDs for which already known --- datalad/distribution/utils.py | 64 +++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/datalad/distribution/utils.py b/datalad/distribution/utils.py index fc76b8abc7..169e4fe1d2 100644 --- a/datalad/distribution/utils.py +++ b/datalad/distribution/utils.py @@ -19,13 +19,14 @@ from six.moves.urllib.parse import unquote as urlunquote +from ..dochelpers import single_or_plural from datalad.support.annexrepo import GitRepo from datalad.support.annexrepo import AnnexRepo from datalad.support.network import DataLadRI from datalad.support.network import URL from datalad.support.network import RI from datalad.support.network import PathRI -from datalad.utils import knows_annex +from datalad.utils import knows_annex, assure_bool lgr = logging.getLogger('datalad.distribution.utils') @@ -152,30 +153,57 @@ def _handle_possible_annex_dataset(dataset, reckless, description=None): repo._init(description=description) if reckless: repo._run_annex_command('untrust', annex_options=['here']) - # go through list of special remotes and issue info message that - # some additional ones are present and were not auto-enabled - remote_names = repo.get_remotes( - with_urls_only=False, - exclude_special_remotes=False) + + srs = {True: [], False: []} # special remotes by "autoenable" key remote_uuids = None # might be necessary to discover known UUIDs + for uuid, config in repo.get_special_remotes().items(): sr_name = config.get('name', None) - sr_type = config.get('type', None) - if sr_type == 'git': - # determine either there is a registered remote with matching UUID + sr_autoenable = config.get('autoenable', False) + try: + sr_autoenable = assure_bool(sr_autoenable) + except ValueError: + # Be resilient against misconfiguration. Here it is only about + # informing the user, so no harm would be done + lgr.warning( + 'Failed to process "autoenable" value %r for sibling %s in ' + 'dataset %s as bool. You might need to enable it later ' + 'manually and/or fix it up to avoid this message in the future.', + sr_autoenable, sr_name, dataset.path) + continue + + # determine either there is a registered remote with matching UUID + if uuid: if remote_uuids is None: remote_uuids = { repo.config.get('remote.%s.annex-uuid' % r) - for r in remote_names + for r in repo.get_remotes() } - if (sr_type != 'git' and sr_name and sr_name not in remote_names) or \ - (sr_type == 'git' and uuid and uuid not in remote_uuids): - # if it is not listed among the remotes, it wasn't enabled - lgr.info( - 'access to dataset sibling "%s" not auto-enabled, enable with:\n\t\tdatalad siblings -d "%s" enable -s %s', - sr_name, - dataset.path, - sr_name) + if uuid not in remote_uuids: + srs[sr_autoenable].append(sr_name) + + if srs[True]: + lgr.debug( + "configuration for %s %s added because of autoenable," + " but no UUIDs for them yet known for dataset %s", + # since we are only at debug level, we could call things their + # proper names + single_or_plural("special remote", "special remotes", len(srs[True]), True), + ", ".join(srs[True]), + dataset.path + ) + + if srs[False]: + # if has no auto-enable special remotes + lgr.info( + 'access to %s %s not auto-enabled, enable with:\n\t\tdatalad siblings -d "%s" enable -s %s', + # but since humans might read it, we better confuse them with our + # own terms! + single_or_plural("dataset sibling", "dataset siblings", len(srs[False]), True), + ", ".join(srs[False]), + dataset.path, + srs[False][0] if len(srs[False]) == 1 else "SIBLING", + ) def _get_installationpath_from_url(url):