Merge remote-tracking branch 'origin/master' into bf-6940-enh-3

* origin/master:
  Switch macOS appveyor builds to Monterey
  Automated deployment to update contributors 2022-12-14
  BF: Don't change ui.backend of test process
  ENH(DX,TST): log at DEBUG about encountered error, unit-test that exception handling block
  RF: Move basic functionality to SpecialRemote
  BF: Don't overwrite AVAILABILITY
  Send ERROR to annex when special remote process fails
  Add Matthias Riße to .zenodo.json
  Update changelog.d/pr-7213.md
  [release-action] Autogenerate changelog snippet for PR 7213
  NF: make extra_remote_settings configurable in git config
  Don't be scary when failing to annex-enable

.all-contributorsrc

@@ -515,6 +515,17 @@
             "avatar_url": [
                 "https://avatars.githubusercontent.com/u/133209?v=4"
             ]
+        },
+        {
+            "login": "asmacdo",
+            "name": "Austin Macdonald",
+            "contributions": [
+                "code"
+            ],
+            "profile": "https://github.com/asmacdo",
+            "avatar_url": [
+                "https://avatars.githubusercontent.com/u/1028657?v=4"
+            ]
         }
     ],
     "contributorsPerLine": 7

.appveyor.yml

@@ -95,7 +95,7 @@ environment:
     - ID: MacP38core
       # ~40min
       DTS: datalad.core datalad.dataset datalad.runner datalad.support
-      APPVEYOR_BUILD_WORKER_IMAGE: macOS
+      APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
       PY: 3.8
       # does not give a functional installation
       # INSTALL_GITANNEX: git-annex -m snapshot
@@ -151,7 +151,7 @@ environment:
           datalad.interface
           datalad.tests
           datalad.ui
-      APPVEYOR_BUILD_WORKER_IMAGE: macOS
+      APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
       PY: 3.8
       INSTALL_GITANNEX: git-annex
       DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets
@@ -161,7 +161,7 @@ environment:
       DTS: >
           datalad.local
           datalad.distributed
-      APPVEYOR_BUILD_WORKER_IMAGE: macOS
+      APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
       PY: 3.8
       INSTALL_GITANNEX: git-annex
       DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets

.tributors

@@ -234,5 +234,9 @@
     "xgui3783": {
         "name": "xgui3783",
         "blog": "https://github.com/xgui3783"
+    },
+    "asmacdo": {
+        "name": "Austin Macdonald",
+        "blog": "https://github.com/asmacdo"
     }
 }

.zenodo.json

@@ -126,6 +126,10 @@
             "affiliation": "Institute of Neuroscience and Medicine, Brain & Behaviour (INM-7), Research Centre Jülich, Jülich, Germany",
             "name": "Szczepanik, Michał",
             "orcid": "0000-0002-4028-2087"
+        },
+        {
+            "affiliation": "Institute of Energy and Climate Research - Stratosphere (IEK-7), Research Centre Jülich, Jülich, Germany",
+            "name": "Riße, Matthias"
         }
     ],
     "grants": [

README.md

@@ -19,7 +19,7 @@
 [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](https://github.com/datalad/datalad/blob/master/CODE_OF_CONDUCT.md)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.808846.svg)](https://doi.org/10.5281/zenodo.808846)
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-46-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-47-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 
@@ -208,6 +208,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://www.mmmccormick.com"><img src="https://avatars.githubusercontent.com/u/25432?v=4?s=100" width="100px;" alt="Matt McCormick"/><br /><sub><b>Matt McCormick</b></sub></a><br /><a href="https://github.com/datalad/datalad/commits?author=thewtex" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/vickychenglau"><img src="https://avatars.githubusercontent.com/u/22065437?v=4?s=100" width="100px;" alt="Vicky C Lau"/><br /><sub><b>Vicky C Lau</b></sub></a><br /><a href="https://github.com/datalad/datalad/commits?author=vickychenglau" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://chris-lamb.co.uk"><img src="https://avatars.githubusercontent.com/u/133209?v=4?s=100" width="100px;" alt="Chris Lamb"/><br /><sub><b>Chris Lamb</b></sub></a><br /><a href="https://github.com/datalad/datalad/commits?author=lamby" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/asmacdo"><img src="https://avatars.githubusercontent.com/u/1028657?v=4?s=100" width="100px;" alt="Austin Macdonald"/><br /><sub><b>Austin Macdonald</b></sub></a><br /><a href="https://github.com/datalad/datalad/commits?author=asmacdo" title="Code">💻</a></td>
     </tr>
   </tbody>
 </table>

changelog.d/pr-7213.md

@@ -0,0 +1,3 @@
+### 🚀 Enhancements and New Features
+
+- New config `datalad.create-sibling-ghlike.extra-remote-settings.NETLOC.KEY=VALUE` allows to add and/or overwrite local configuration for the created sibling by the commands `create-sibling-<gin|gitea|github|gitlab|gogs>`.  [PR #7213](https://github.com/datalad/datalad/pull/7213) (by [@matrss](https://github.com/matrss))

changelog.d/pr-7217.md

@@ -0,0 +1,6 @@
+### 🚀 Enhancements and New Features
+
+- The `siblings` command does not concern the user with messages about 
+  inconsequential failure to annex-enable a remote anymore. 
+  [PR #7217](https://github.com/datalad/datalad/pull/7217)
+  (by [@bpoldrack](https://github.com/bpoldrack))

datalad/customremotes/__init__.py

@@ -20,6 +20,7 @@
     RemoteError as _RemoteError,
 )
 from datalad.support.exceptions import format_exception_with_cause
+from datalad.ui import ui
 
 
 class RemoteError(_RemoteError):
@@ -52,6 +53,12 @@ def __str__(self):
 class SpecialRemote(_SpecialRemote):
     """Common base class for all of DataLad's special remote implementations"""
 
+    def __init__(self, annex):
+        super(SpecialRemote, self).__init__(annex=annex)
+        # instruct annex backend UI to use this remote
+        if ui.backend == 'annex':
+            ui.set_specialremote(self)
+
     def message(self, msg, type='debug'):
         handler = dict(
             debug=self.annex.debug,
@@ -70,3 +77,32 @@ def message(self, msg, type='debug'):
             # If we can't have an actual info message, at least have a
             # debug message.
             self.annex.debug(msg)
+
+    def send_progress(self, progress):
+        """Indicates the current progress of the transfer (in bytes).
+
+        May be repeated any number of times during the transfer process.
+
+        Too frequent updates are wasteful but bear in mind that this is used
+        both to display a progress meter for the user, and for
+        ``annex.stalldetection``. So, sending an update on each 1% of the file
+        may not be frequent enough, as it could appear to be a stall when
+        transferring a large file.
+
+        Parameters
+        ----------
+        progress : int
+            The current progress of the transfer in bytes.
+        """
+        # This method is called by AnnexSpecialRemoteProgressBar through an
+        # obscure process that involves multiple layers of abstractions for
+        # UIs, providers, downloaders, progressbars, which is only happening
+        # within the environment of a running special remote process though
+        # a combination of circumstances.
+        #
+        # The main purpose of this method is to have a place to leave this
+        # comment within the code base of the special remotes, in order to
+        # aid future souls having to sort this out.
+        # (and to avoid having complex code make direct calls to internals
+        # of this class, making things even more complex)
+        self.annex.progress(progress)

datalad/customremotes/archives.py

@@ -91,7 +91,6 @@ class ArchiveAnnexCustomRemote(AnnexCustomRemote):
     URL_SCHEME = SUPPORTED_SCHEMES[0]
     URL_PREFIX = URL_SCHEME + ":"
 
-    AVAILABILITY = "local"
     COST = 500
 
     def __init__(self, annex, path=None, persistent_cache=True, **kwargs):

datalad/customremotes/base.py

@@ -24,24 +24,18 @@
 )
 from datalad.customremotes import SpecialRemote
 
-from datalad.ui import ui
-
 URI_PREFIX = "dl"
 
 
 class AnnexCustomRemote(SpecialRemote):
     # default properties
     COST = 100
-    AVAILABILITY = "LOCAL"
+    AVAILABILITY = "local"
 
     def __init__(self, annex):  # , availability=DEFAULT_AVAILABILITY):
         super().__init__(annex)
         # TODO self.info = {}, self.configs = {}
 
-        # instruct annex backend UI to use this remote
-        if ui.backend == 'annex':
-            ui.set_specialremote(self)
-
         # OPT: a counter to increment upon successful encounter of the scheme
         # (ATM only in gen_URLS but later could also be used in other
         # requests).  This would allow to consider schemes in order of
@@ -76,35 +70,6 @@ def gen_URLS(self, key):
                     yield url
         self.annex.debug("Processed %d URL(s) for key %s", nurls, key)
 
-    def send_progress(self, progress):
-        """Indicates the current progress of the transfer (in bytes).
-
-        May be repeated any number of times during the transfer process.
-
-        Too frequent updates are wasteful but bear in mind that this is used
-        both to display a progress meter for the user, and for
-        ``annex.stalldetection``. So, sending an update on each 1% of the file
-        may not be frequent enough, as it could appear to be a stall when
-        transferring a large file.
-
-        Parameters
-        ----------
-        progress : int
-            The current progress of the transfer in bytes.
-        """
-        # This method is called by AnnexSpecialRemoteProgressBar through an
-        # obscure process that involves multiple layers of abstractions for
-        # UIs, providers, downloaders, progressbars, which is only happening
-        # within the environment of a running special remote process though
-        # a combination of circumstances.
-        #
-        # The main purpose of this method is to have a place to leave this
-        # comment within the code base of the special remotes, in order to
-        # aid future souls having to sort this out.
-        # (and to avoid having complex code make direct calls to internals
-        # of this class, making things even more complex)
-        self.annex.progress(progress)
-
     # Protocol implementation
     def initremote(self):
         pass
@@ -122,7 +87,7 @@ def getcost(self):
         return self.COST
 
     def getavailability(self):
-        return self.AVAILABILITY.lower()
+        return self.AVAILABILITY
 
 
 # this function only has anecdotal value and is not used anywhere

datalad/customremotes/main.py

@@ -18,9 +18,11 @@
     parser_add_version_opt,
 )
 from datalad.cli.utils import setup_exceptionhook
-from datalad.log import lgr
 from datalad.ui import ui
 
+import logging
+lgr = logging.getLogger('datalad.customremotes')
+
 
 def setup_parser(remote_name, description):
     # setup cmdline args parser
@@ -75,5 +77,14 @@ def main(args=None, cls=None, remote_name=None, description=None):
         try:
             _main(args, cls)
         except Exception as exc:
-            lgr.error('%s (%s)', str(exc), exc.__class__.__name__)
+            lgr.debug('%s (%s) - passing ERROR to git-annex and exiting',
+                      str(exc), exc.__class__.__name__)
+            # `SpecialRemote` classes are supposed to catch everything and
+            # turn it into a `RemoteError` resulting in an ERROR message to
+            # annex. If we end up here, something went wrong outside of the
+            # `master.Listen()` call in `_main`.
+            # In any case, exiting the special remote process should be
+            # accompanied by such an ERROR message to annex rather than a log
+            # message.
+            print("ERROR %s (%s)" % (str(exc), exc.__class__.__name__))
             sys.exit(1)

datalad/customremotes/tests/test_main.py

@@ -0,0 +1,44 @@
+# emacs: -*- mode: python; py-indent-offset: 4; tab-width: 4; indent-tabs-mode: nil -*-
+# ex: set sts=4 ts=4 sw=4 et:
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+#
+#   See COPYING file distributed along with the datalad package for the
+#   copyright and license terms.
+#
+# ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
+"""Tests for the "main" driver of the special remotes"""
+
+import logging
+import os
+import pytest
+
+from ..main import main
+
+from ..base import AnnexCustomRemote
+
+from datalad.tests.utils_pytest import (
+    patch,
+    swallow_logs,
+    swallow_outputs,
+)
+from datalad.ui import ui
+
+
+def test_erroring_out():
+    class TooAbstract(AnnexCustomRemote):
+        pass
+
+    # patch to not let `main` change the ui.backend
+    # of the test process (instead of a special remote process it is actually
+    # targeting)
+    with swallow_logs(new_level=logging.DEBUG) as cml, \
+        swallow_outputs() as cmo:
+        with pytest.raises(SystemExit) as cme,\
+                patch.object(ui, "set_backend", autospec=True):
+            main(args=[], cls=TooAbstract)
+        assert cme.value.code == 1
+        assert 'passing ERROR to git-annex' in cml.out
+        # verify basic correct formatting of string to git-annex
+        assert cmo.out.startswith('ERROR ')
+        assert os.linesep not in cmo.out.rstrip()
+        assert cmo.out.endswith('\n')  # This is the case even on Windows.

datalad/distributed/create_sibling_ghlike.py

@@ -18,6 +18,7 @@
 
 import requests
 
+from datalad import cfg as dlcfg
 from datalad.distribution.dataset import (
     EnsureDataset,
     require_dataset,
@@ -157,6 +158,19 @@ def __init__(self, url, credential, require_token=True, token_info=None):
             + f'. {token_info}' if token_info else '',
             require_token,
         )
+        self._set_extra_remote_settings()
+
+    def _set_extra_remote_settings(self):
+        target_name = urlparse(self.api_url).netloc
+        config_section = "datalad.create-sibling-ghlike.extra-remote-settings.{}".format(target_name)
+        target_specific_settings = {
+            option: dlcfg.get_value(config_section, option)
+            for option in dlcfg.options(config_section)
+        }
+        self.extra_remote_settings = {
+            **self.extra_remote_settings,
+            **target_specific_settings,
+        }
 
     @todo_interface_for_extensions
     def _set_request_headers(self, credential_name, auth_info, require_token):

datalad/distributed/create_sibling_gin.py

@@ -71,6 +71,18 @@ class CreateSiblingGin(Interface):
     generated on the platform (Account->Your Settings->Applications->Generate
     New Token).
 
+    This command can be configured with
+    "datalad.create-sibling-ghlike.extra-remote-settings.NETLOC.KEY=VALUE" in
+    order to add any local KEY = VALUE configuration to the created sibling in
+    the local `.git/config` file. NETLOC is the domain of the Gin instance to
+    apply the configuration for.
+    This leads to a behavior that is equivalent to calling datalad's
+    ``siblings('configure', ...)``||``siblings configure`` command with the
+    respective KEY-VALUE pair after creating the sibling.
+    The configuration, like any other, could be set at user- or system level, so
+    users do not need to add this configuration to every sibling created with
+    the service at NETLOC themselves.
+
     .. versionadded:: 0.16
     """
 

datalad/distributed/create_sibling_gitea.py

@@ -94,6 +94,18 @@ class CreateSiblingGitea(Interface):
     In order to be able to use this command, a personal access token has to be
     generated on the platform (Account->Settings->Applications->Generate Token).
 
+    This command can be configured with
+    "datalad.create-sibling-ghlike.extra-remote-settings.NETLOC.KEY=VALUE" in
+    order to add any local KEY = VALUE configuration to the created sibling in
+    the local `.git/config` file. NETLOC is the domain of the Gitea instance to
+    apply the configuration for.
+    This leads to a behavior that is equivalent to calling datalad's
+    ``siblings('configure', ...)``||``siblings configure`` command with the
+    respective KEY-VALUE pair after creating the sibling.
+    The configuration, like any other, could be set at user- or system level, so
+    users do not need to add this configuration to every sibling created with
+    the service at NETLOC themselves.
+
     .. versionadded:: 0.16
     """
 

datalad/distributed/create_sibling_github.py

@@ -129,6 +129,18 @@ class CreateSiblingGithub(Interface):
     generated on the platform (Account->Settings->Developer Settings->Personal
     access tokens->Generate new token).
 
+    This command can be configured with
+    "datalad.create-sibling-ghlike.extra-remote-settings.NETLOC.KEY=VALUE" in
+    order to add any local KEY = VALUE configuration to the created sibling in
+    the local `.git/config` file. NETLOC is the domain of the Github instance to
+    apply the configuration for.
+    This leads to a behavior that is equivalent to calling datalad's
+    ``siblings('configure', ...)``||``siblings configure`` command with the
+    respective KEY-VALUE pair after creating the sibling.
+    The configuration, like any other, could be set at user- or system level, so
+    users do not need to add this configuration to every sibling created with
+    the service at NETLOC themselves.
+
     .. versionchanged:: 0.16
        || REFLOW >>
        The API has been aligned with the some