Merge commit '0.17.2-78-g6d74da13a' (maint)

* commit '0.17.2-78-g6d74da13a': (44 commits)
  BF: fix typo which prevented silently to not show details of filesystems
  In bisect-git-annex do not bother with singularity image (we no longer build it)
  BF(TST): allow for a annex repo version to upgrade if running in adjusted branches
  BF: travis - mark our directory as safe to interact with as root
  ENH: also install dcm2niix via APT for neuroimaging
  Workaround for -crawler: symlink tox.ini into __testhome__ to please coverage
  RF: refactor test_extensions to centralize ext specifics, use extension"s own tox.ini
  Crawler is now using pytest.
  BF(TST,workaround): just xfail failing archives test on NFS
  BF(TST): do not expect log message for guessing Path to be possibly a URL on windows
  RF: just rename internal variable from unclear "op" to "io"
  BF: do not pretend we know what repo version git-annex would upgrade to
  ENH(TST): Disable coverage reporting on travis
  move test dependencies to setup.py
  use `type[...]` instead of `Type[...]`
  reformatting
  modernize type annotations in runner code
  use new annotation style in runner tests
  fix type annotation errors in runner-tests
  BF: add pytest into deps for typing since needed
  ...

.github/workflows/test_extensions.yml

@@ -17,6 +17,7 @@ jobs:
             datalad-crawler,
             datalad-deprecated,
         ]
+
     steps:
     - name: Set up system
       shell: bash
@@ -25,48 +26,93 @@ jobs:
         sudo apt-get update -qq
         sudo apt-get install eatmydata
         sudo eatmydata apt-get install git-annex-standalone
+
     - name: Set up environment
       run: |
         git config --global user.email "test@github.land"
         git config --global user.name "GitHub Almighty"
 
     - uses: actions/checkout@v1
+
     - name: Set up Python 3.7
       uses: actions/setup-python@v1
       with:
         python-version: 3.7
-    - name: Install dependencies
+
+    - name: Install DataLad and dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements-devel.txt
-        # we are using pytest here but extensions still tested with nose
-        pip install nose
-    - name: Install ${{ matrix.extension }} extension
-      run: |
-        pip install https://github.com/datalad/${{ matrix.extension }}/archive/master.zip
-      if: matrix.extension != 'datalad-hirni'
-    - name: Install ${{ matrix.extension }} extension
+
+    - name: Setup ${{ matrix.extension }} extension specific variables
       run: |
-        pip install https://github.com/psychoinformatics-de/${{ matrix.extension }}/archive/master.zip
-      if: matrix.extension == 'datalad-hirni'
-    - name: Install additional dependencies
+        DL_PIP_INSTALLS=mock
+        DL_APT_INSTALLS=dcm2niix
+        DL_REPO=datalad/${{ matrix.extension }}
+        DL_TESTER=nose
+        DL_NEED_SINGULARITY=
+        # TODO: just reuse information from datalad-extensions!!
+        case ${{ matrix.extension }} in
+          datalad-metalad)
+            DL_PIP_INSTALLS+=" nose"
+            DL_TESTER=pytest;;
+          datalad-hirni)
+            DL_REPO=psychoinformatics-de/${{ matrix.extension }}
+            DL_NEED_SINGULARITY=1;;
+          datalad-crawler)
+            DL_TESTER=pytest;;
+          datalad-container)
+            DL_NEED_SINGULARITY=1;;
+        esac
+        {
+        echo "DL_PIP_INSTALLS=$DL_PIP_INSTALLS"
+        echo "DL_APT_INSTALLS=$DL_APT_INSTALLS"
+        echo "DL_REPO=$DL_REPO"
+        echo "DL_TESTER=$DL_TESTER"
+        echo "DL_NEED_SINGULARITY=$DL_NEED_SINGULARITY"
+        echo "DL_PACKAGE=$(echo ${{ matrix.extension }} | tr '-' '_')"
+        } >> "$GITHUB_ENV"
+        
+    - name: Clone and install ${{ matrix.extension }} extension
       run: |
-        pip install mock
-      if: matrix.extension == 'datalad-crawler'
+        git clone https://github.com/$DL_REPO __extension__
+        pip install ./__extension__[devel]
+
     - name: Install Singularity
       run: sudo eatmydata apt-get install singularity-container
-      if: matrix.extension == 'datalad-container' || matrix.extension == 'datalad-hirni'
+      if: env.DL_NEED_SINGULARITY == '1'
+
+    - name: Install additional dependencies and prepare for testing
+      run: |
+        pip install $DL_PIP_INSTALLS
+        [ -z "$DL_APT_INSTALLS" ] || sudo apt-get install $DL_APT_INSTALLS
+        mkdir -p __testhome__
+
     - name: WTF!?
       run: |
         datalad wtf
-    - name: ${{ matrix.extension }} tests
+
+    - name: ${{ matrix.extension }} tests using nose
+      run: |
+        cd __testhome__
+        python -m nose -s -v --with-cov --cover-package datalad ${DL_PACKAGE}
+      if: env.DL_TESTER == 'nose'
+
+    - name: ${{ matrix.extension }} tests using pytest
+      run: |
+        cd __testhome__
+        # -crawler's tox.ini points to itself (but without full path) for coverage
+        # configuration.  So far attempts to "fix" by somehow providing full path did not work.
+        # That is why just symlink it here.
+        ln -s ../__extension__/tox.ini .
+        python -m pytest -c ./tox.ini -s -v --cov=datalad --pyargs ${DL_PACKAGE}
+      if: env.DL_TESTER == 'pytest'
+
+    - name: Prepare coverage
       run: |
-        mkdir -p __testhome__
         cd __testhome__
-        python -m nose -s -v --with-cov --cover-package datalad $(echo ${{ matrix.extension }} | tr '-' '_')
-        # TODO: later whenever some extensions would migrate to pytest -- use pytest
-        # python -m pytest -c ../tox.ini -s -v --cov=datalad --pyargs $(echo ${{ matrix.extension }} | tr '-' '_')
         python -m coverage xml
+
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
       with:

.github/workflows/typing.yml

@@ -0,0 +1,32 @@
+# This workflow is running types checks using `tox -e typing` you could do locally.
+# As type hinting is not yet fully and consistently done through out the project only
+# some (randomly) selected files (listed in tox.ini) will be checked.  Feel welcome
+# to add extra files to the list there. See https://github.com/datalad/datalad/issues/6884
+# for the overall goal/progress towards type hints coverage in DataLad.
+name: Type-check
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  typing:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade tox
+
+      - name: Run type checker
+        run: tox -e typing

.travis.yml

@@ -55,13 +55,21 @@ matrix:
     env:
     - PYTEST_SELECTION=
     - PYTEST_SELECTION_OP=not
+  # Two matrix runs for "recent python and git-annex with the recent supported by git annex
+  # new version of repo"
   - python: '3.10'
     dist: focal
     env:
       - PYTEST_SELECTION=
       - PYTEST_SELECTION_OP=not
-      # current "default" version of git-annex 8.20201007  is not co-installable
-      # with python 3.10, so let's just allow for any git-annex version
+      - DATALAD_REPO_VERSION=10
+      - _DL_ANNEX_INSTALL_SCENARIO="miniconda --python-match minor --batch git-annex -m conda"
+  - python: '3.10'
+    dist: focal
+    env:
+      - PYTEST_SELECTION=
+      - PYTEST_SELECTION_OP=
+      - DATALAD_REPO_VERSION=10
       - _DL_ANNEX_INSTALL_SCENARIO="miniconda --python-match minor --batch git-annex -m conda"
   - if: type = cron
     python: 3.7
@@ -237,6 +245,9 @@ before_install:
 install:
   - git config --global user.email "test@travis.land"
   - git config --global user.name "Travis Almighty"
+  # we are pip sudo install  below and versioneer needs to run git. recent git needs to
+  # be made certain it is safe to do
+  - sudo git config --global --add safe.directory $PWD
   - cd ..; pip install -q codecov; cd -
   - pip install -r requirements-devel.txt
   # So we could test under sudo -E with PATH pointing to installed location
@@ -264,6 +275,7 @@ script:
       -m "${PYTEST_SELECTION:+$PYTEST_SELECTION_OP($PYTEST_SELECTION) and }not(turtle)"
       --doctest-modules
       --cov=datalad
+      --cov-report=
       --pyargs
       $TESTS_TO_PERFORM
   - cd ..