Merge pull request #6813 from bpoldrack/prepare-release

Release 0.17.0

.all-contributorsrc

@@ -438,6 +438,17 @@
             "avatar_url": [
                 "https://avatars.githubusercontent.com/u/28018084?v=4"
             ]
+        },
+        {
+            "login": "jdkent",
+            "name": "James Kent",
+            "contributions": [
+                "code"
+            ],
+            "profile": "https://jdkent.github.io/",
+            "avatar_url": [
+                "https://avatars.githubusercontent.com/u/12564882?v=4"
+            ]
         }
     ],
     "contributorsPerLine": 7

.appveyor.yml

@@ -345,9 +345,9 @@ test_script:
   - cmd: md __testhome__
   - sh: mkdir __testhome__
   - cd __testhome__
-    # run test selecion (--traverse-namespace needed from Python 3.8 onwards)
-  - cmd: python -W error::DeprecationWarning:^datalad -m nose --traverse-namespace -s -v -A "not (turtle)" --with-cov --cover-package datalad %DTS%
-  - sh: PATH=$PWD/../tools/coverage-bin:$PATH python -W error::DeprecationWarning:^datalad -m nose --traverse-namespace -s -v -A "not (turtle)" --with-cov --cover-package datalad ${DTS}
+    # run test selection
+  - cmd: python -m pytest -c ../tox.ini -s -v -m "not (turtle)" --cov=datalad --pyargs %DTS%
+  - sh: PATH=$PWD/../tools/coverage-bin:$PATH python -m pytest -c ../tox.ini -s -v -m "not (turtle)" --cov=datalad --pyargs ${DTS}
 
 
 after_test:

.github/workflows/test-nose.yml

@@ -0,0 +1,54 @@
+name: Test old nose code
+
+on:
+  - pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip wheel
+          pip install -r requirements-devel.txt
+          pip install coverage datalad-installer nose
+          datalad-installer --sudo ok -E /tmp/new.env \
+            miniconda --batch \
+            git-annex=8.20201007 -m conda
+          . /tmp/new.env
+          echo "PATH=$PATH" >> "$GITHUB_ENV"
+
+      - name: Set up environment
+        run: |
+          git config --global user.email "test@github.land"
+          git config --global user.name "GitHub Almighty"
+
+          mkdir -p __testhome__
+
+      - name: WTF
+        run: datalad wtf
+
+      - name: Test nose code
+        run: |
+          python -W error::DeprecationWarning:^datalad \
+            -m nose --with-cov --cover-package datalad --logging-level=INFO \
+            datalad.tests.test_tests_utils
+          coverage report
+          coverage xml
+        working-directory: __testhome__
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v2
+        with:
+          directory: __testhome__
+          fail_ci_if_error: false

.github/workflows/test_crippled.yml

@@ -47,4 +47,4 @@ jobs:
       run: |
         mkdir -p __testhome__
         cd __testhome__
-        python -m nose -s -v -A "not (turtle)" datalad.core datalad.local datalad.distributed datalad.tests.test_utils_cached_dataset
+        python -m pytest -c ../tox.ini -s -v -m "not (turtle)" --pyargs datalad.core datalad.local datalad.distributed datalad.tests.test_utils_cached_dataset

.github/workflows/test_extensions.yml

@@ -39,6 +39,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r requirements-devel.txt
+        # we are using pytest here but extensions still tested with nose
+        pip install nose
     - name: Install ${{ matrix.extension }} extension
       run: |
         pip install https://github.com/datalad/${{ matrix.extension }}/archive/master.zip
@@ -62,6 +64,8 @@ jobs:
         mkdir -p __testhome__
         cd __testhome__
         python -m nose -s -v --with-cov --cover-package datalad $(echo ${{ matrix.extension }} | tr '-' '_')
+        # TODO: later whenever some extensions would migrate to pytest -- use pytest
+        # python -m pytest -c ../tox.ini -s -v --cov=datalad --pyargs $(echo ${{ matrix.extension }} | tr '-' '_')
         python -m coverage xml
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3

.github/workflows/test_macos.yml

@@ -49,7 +49,7 @@ jobs:
       run: |
         mkdir -p __testhome__
         cd __testhome__
-        python -m nose -s -v --with-cov --cover-package datalad --cover-xml datalad.core datalad.support
+        python -m pytest -c ../tox.ini -s -v --cov=datalad --cov-report=xml --pyargs datalad.core datalad.support
 
     - name: Upload coverage report to Codecov
       uses: codecov/codecov-action@v2

.gitignore

@@ -29,3 +29,4 @@ tmp
 .asv
 bin/
 tools/bisect-git-annex.simg
+__testhome__

.travis.yml

@@ -27,11 +27,11 @@ env:
     - DATALAD_LOG_ENV=GIT_SSH_COMMAND
     - TESTS_TO_PERFORM=datalad
     # Should be an array, travis breaks on it, define/adjust in the "before_install"
-    #- NOSE_OPTS=( -v )
+    #- PYTEST_OPTS=( -v )
     # Note, that there's "turtle" as well, which is always excluded from
     # running on Travis.
-    - NOSE_SELECTION="integration or usecase or slow or network"
-    - NOSE_SELECTION_OP="not "   # so it would be "not (integration or usecase)"
+    - PYTEST_SELECTION="integration or usecase or slow or network"
+    - PYTEST_SELECTION_OP="not "   # so it would be "not (integration or usecase)"
     - DATALAD_DATASETS_TOPURL=https://datasets-tests.datalad.org
     # How/which git-annex we install.  conda's build would be the fastest, but it must not
     # get ahead in PATH to not shadow travis' python
@@ -45,48 +45,48 @@ matrix:
     env:
     # Run all tests in a single whoop here
     # We cannot have empty -A selector, so the one which always will be fulfilled
-    - NOSE_SELECTION=
-    - NOSE_SELECTION_OP=not
+    - PYTEST_SELECTION=
+    - PYTEST_SELECTION_OP=not
     # To test https://github.com/datalad/datalad/pull/4342 fix.
     # From our testing in that PR seems to have no effect, but kept around since should not hurt.
     - LC_ALL=ru_RU.UTF-8
   - python: 3.9
     dist: bionic # Xenial doesn't have 3.8 pre-installed (only 3.8-dev).
     env:
-    - NOSE_SELECTION=
-    - NOSE_SELECTION_OP=not
+    - PYTEST_SELECTION=
+    - PYTEST_SELECTION_OP=not
   - python: '3.10'
     dist: focal
     env:
-      - NOSE_SELECTION=
-      - NOSE_SELECTION_OP=not
+      - PYTEST_SELECTION=
+      - PYTEST_SELECTION_OP=not
   - if: type = cron
     python: 3.7
     # Single run for Python 3.7
     env:
     # Run all tests in a single whoop here
     # We cannot have empty -A selector, so the one which always will be fulfilled
-    - NOSE_SELECTION=
-    - NOSE_SELECTION_OP=not
+    - PYTEST_SELECTION=
+    - PYTEST_SELECTION_OP=not
   - python: 3.7
     # Split runs for v6 since a single one is too long now
     env:
-    - NOSE_SELECTION_OP=not
+    - PYTEST_SELECTION_OP=not
     - DATALAD_SSH_MULTIPLEX__CONNECTIONS=0
-    - _DL_ANNEX_INSTALL_SCENARIO="miniconda --batch git-annex=8.20210310 -m conda"
+    - _DL_ANNEX_INSTALL_SCENARIO="miniconda --batch git-annex=10.20220525 -m conda"
   - python: 3.7
     env:
-    - NOSE_SELECTION_OP=""
+    - PYTEST_SELECTION_OP=""
     - DATALAD_SSH_MULTIPLEX__CONNECTIONS=0
     - _DL_ANNEX_INSTALL_SCENARIO="miniconda --batch git-annex=8.20210310 -m conda"
-    # To test https://github.com/datalad/datalad/pull/4342 fix in case of no "not" for NOSE.
+    # To test https://github.com/datalad/datalad/pull/4342 fix in case of no "not" for pytest.
     # From our testing in that PR seems to have no effect, but kept around since should not hurt.
     - LANG=bg_BG.UTF-8
   - python: 3.7
-    # Run slow etc tests under a single tricky scenario
+   # Run slow etc tests under a single tricky scenario
     env:
     - _DL_TMPDIR="/var/tmp/sym link"
-    - NOSE_SELECTION_OP=""
+    - PYTEST_SELECTION_OP=""
     # And the leading - in filenames for the most challenge
     - DATALAD_TESTS_OBSCURE_PREFIX=-
     - DATALAD_LOG_TRACEBACK=collide  # just a smoke test for now
@@ -162,7 +162,7 @@ matrix:
     env:
     # to test operation under root since also would consider FS "crippled" due to
     # ability to rewrite R/O files
-    - NOSE_WRAPPER="sudo -E"
+    - PYTEST_WRAPPER="sudo -E"
     # no key authentication for root:
     - DATALAD_TESTS_SSH=0
   - if: type = cron
@@ -194,11 +194,11 @@ matrix:
 #    - UNSET_S3_SECRETS=1
 
 before_install:
-  - NOSE_OPTS=( )
+  - PYTEST_OPTS=( -v )
   # If we requested to run only not slow (typically <10sec) tests, fail if a test
   # takes 3x more than that - it needs to get @slow or @turtle annotation
-  - if echo "$NOSE_SELECTION_OP($NOSE_SELECTION)" | grep -q "^not.*slow"; then
-      NOSE_OPTS=( "${NOSE_OPTS[@]}" --with-doctest --with-timer --timer-ok 5 --timer-warning 30 --timer-fail error --timer-filter warning,error );
+  - if echo "$PYTEST_SELECTION_OP($PYTEST_SELECTION)" | grep -q "^not.*slow"; then
+      PYTEST_OPTS=( "${PYTEST_OPTS[@]}" --doctest-modules --durations=0 --durations-min=5 --fail-slow 30 );
       export DATALAD_TESTS_SETUP_TESTREPOS=1;
     fi
   # Just in case we need to check if nfs is there etc
@@ -249,14 +249,17 @@ script:
   - mkdir -p __testhome__
   - cd __testhome__
   # Run tests
+  #  Note: adding --log-cli-level=INFO would result in DATALAD_LOG_TARGET=/dev/null being not
+  #  in effect, dumping too many logs.
   - http_proxy=
     PATH=$PWD/../tools/coverage-bin:$PATH
-    $NOSE_WRAPPER python -W error::DeprecationWarning:^datalad
-      -m nose "${NOSE_OPTS[@]}"
-      -A "$NOSE_SELECTION_OP($NOSE_SELECTION) and not(turtle)"
-      --with-doctest
-      --with-cov --cover-package datalad
-      --logging-level=INFO
+    $PYTEST_WRAPPER python
+      -m pytest "${PYTEST_OPTS[@]}"
+      -c ../tox.ini
+      -m "${PYTEST_SELECTION:+$PYTEST_SELECTION_OP($PYTEST_SELECTION) and }not(turtle)"
+      --doctest-modules
+      --cov=datalad
+      --pyargs
       $TESTS_TO_PERFORM
   - cd ..
 
@@ -274,5 +277,5 @@ after_success:
 # for a particular debugging
 #after_failure:
 #  - if [ ! -z "$DATALAD_TESTS_NONETWORK" ]; then sudo route add -net 0.0.0.0 netmask 0.0.0.0 dev lo; fi
-#  - DATALAD_LOG_LEVEL=DEBUG $NOSE_WRAPPER `which nosetests` -s -v --with-doctest --with-cov --cover-package datalad --logging-level=DEBUG
+#  - DATALAD_LOG_LEVEL=DEBUG $PYTEST_WRAPPER `which pytest` -s -v --doctest-modules --cov datalad --log-cli-level=DEBUG
 #  - if [ ! -z "$DATALAD_TESTS_NONETWORK" ]; then sudo route del -net 0.0.0.0 netmask 0.0.0.0 dev lo; fi

.tributors

@@ -166,7 +166,8 @@
     },
     "AKSoo": {
         "name": "Sin Kim",
-        "blog": "https://orcid.org/0000-0003-4652-3758"
+        "blog": "https://orcid.org/0000-0003-4652-3758",
+        "bio": "I like coding, data, and open science. Current interests are adolescent development and psychiatry."
     },
     "DisasterMo": {
         "name": "Michael Burgardt",
@@ -194,5 +195,11 @@
         "name": "Taylor Olson",
         "email": "to.overlake@gmail.com",
         "blog": "https://github.com/taylols"
+    },
+    "jdkent": {
+        "name": "James Kent",
+        "email": "jamesdkent21@gmail.com",
+        "bio": "Postdoc",
+        "blog": "https://jdkent.github.io/"
     }
 }

CHANGELOG.md

@@ -1,3 +1,52 @@
+# 0.17.0 (Thu Jul 7 2022) -- pytest migration
+
+#### 💫 Enhancements and new features
+- "log" progress bar now reports about starting a specific action as well. [#6756](https://github.com/datalad/datalad/pull/6756) (by @yarikoptic)
+- Documentation and behavior of traceback reporting for log messages via `DATALAD_LOG_TRACEBACK` was improved to yield a more compact report. The documentation for this feature has been clarified. [#6746](https://github.com/datalad/datalad/pull/6746) (by @mih)
+- `datalad unlock` gained a progress bar. [#6704](https://github.com/datalad/datalad/pull/6704) (by @adswa)
+- When `create-sibling-gitlab` is called on non-existing subdatasets or paths it now returns an impossible result instead of no feedback at all. [#6701](https://github.com/datalad/datalad/pull/6701) (by @adswa)
+- `datalad wtf` includes a report on file system types of commonly used paths. [#6664](https://github.com/datalad/datalad/pull/6664) (by @adswa)
+- use next generation metadata code in search, if it is available [#6518](https://github.com/datalad/datalad/pull/6518) (by @christian-monch) 
+
+#### 🪓 Deprecations and removals
+- Remove unused and untested log helpers `NoProgressLog` and `OnlyProgressLog`. [#6747](https://github.com/datalad/datalad/pull/6747) (by @mih)
+- Remove unused `sorted_files()` helper. [#6722](https://github.com/datalad/datalad/pull/6722) (by @adswa)
+- Discontinued the value `stdout` for use with the config variable `datalad.log.target` as its use would inevitably break special remote implementations. [#6675](https://github.com/datalad/datalad/pull/6675) (by @bpoldrack)
+- `AnnexRepo.add_urls()` is deprecated in favor of `AnnexRepo.add_url_to_file()` or a direct call to `AnnexRepo.call_annex()`. [#6667](https://github.com/datalad/datalad/pull/6667) (by @mih)
+- `datalad test` command and supporting functionality (e.g., `datalad.test`) were removed. [#](https://github.com/datalad/datalad/pull/6273) (by @jwodder)
+#### 🐛 Bug Fixes
+- `export-archive` does not rely on `normalize_path()` methods anymore and became more robust when called from subdirectories. [#6745](https://github.com/datalad/datalad/pull/6745) (by @adswa)
+- Sanitize keys before checking content availability to ensure that the content availability of files with URL- or custom backend keys is correctly determined and marked. [#6663](https://github.com/datalad/datalad/pull/6663) (by @adswa)
+- Ensure saving a new subdataset to a superdataset yields a valid `.gitmodules` record regardless of whether and how a path constraint is given to the `save()` call. Fixes #6547 [#6790](https://github.com/datalad/datalad/pull/6790) (by @mih)
+- `save` now repairs annex symlinks broken by a `git-mv` operation prior recording a new dataset state. Fixes #4967 [#6795](https://github.com/datalad/datalad/pull/6795) (by @mih)
+
+#### 📝 Documentation
+- API documentation for log helpers, like `log_progress()` is now included in the renderer documentation. [#6746](https://github.com/datalad/datalad/pull/6746) (by @mih)
+- New design document on progress reporting. [#6734](https://github.com/datalad/datalad/pull/6734) (by @mih)
+- Explain downstream consequences of using `--fast` option in `addurls`. [#6684](https://github.com/datalad/datalad/pull/6684) (by @jdkent)
+
+#### 🏠 Internal
+- Inline code of `create-sibling-ria` has been refactored to an internal helper to check for siblings with particular names across dataset hierarchies in `datalad-next`, and is reintroduced into core to modularize the code base further. [#6706](https://github.com/datalad/datalad/pull/6706) (by @adswa)
+- `get_initialized_logger` now lets a given `logtarget` take precendence over `datalad.log.target`. [#6675](https://github.com/datalad/datalad/pull/6675) (by @bpoldrack)
+- Many uses of deprecated call options were replaced with the recommended ones. [#6273](https://github.com/datalad/datalad/pull/6273) (by @jwodder)
+- Get rid of `asyncio` import by defining few noops methods from `asyncio.protocols.SubprocessProtocol` directly in `WitlessProtocol`. [#6648](https://github.com/datalad/datalad/pull/6648) (by @yarikoptic)
+- Consolidate `GitRepo.remove()` and `AnnexRepo.remove()` into a single implementation. [#6783](https://github.com/datalad/datalad/pull/6783) (by @mih)
+#### 🛡 Tests
+- Discontinue use of `with_testrepos` decorator other than for the deprecation cycle for `nose`. [#6690](https://github.com/datalad/datalad/pull/6690) (by @mih @bpoldrack) See [#6144](https://github.com/datalad/datalad/issues/6144) for full list of changes.
+- Remove usage of deprecated `AnnexRepo.add_urls` in tests. [#6683](https://github.com/datalad/datalad/pull/6683) (by @bpoldrack)
+- Minimalistic (adapters, no assert changes, etc) migration from `nose` to `pytest`.
+  Support functionality possibly used by extensions and relying on `nose` helpers is left in place to avoid affecting their run time and defer migration of their test setups.. [#6273](https://github.com/datalad/datalad/pull/6273) (by @jwodder)
+
+#### Authors: 7
+
+- Yaroslav Halchenko (@yarikoptic)
+- Michael Hanke (@mih)
+- Benjamin Poldrack (@bpoldrack)
+- Adina Wagner (@adswa)
+- John T. Wodder (@jwodder)
+- Christian Mönch (@christian-monch)
+- James Kent (@jdkent)
+
 # 0.16.7 (Wed Jul 06 2022)
 
 #### 🐛 Bug Fix