Merge pull request #151 from datalad/tmpl-update

Bring up-to-date with extension template

.appveyor.yml

@@ -3,7 +3,7 @@
 # to create a "native" platform experience, using as few cross-platform
 # helper tools as possible.
 #
-# On Linux/Mac a virtualenv is used for testing. The effective virtual env
+# On Linux/Mac a venv is used for testing. The effective virtual env
 # is available under ~/VENV.
 #
 # All workers support remote login. Login details are shown at the top of each
@@ -64,14 +64,15 @@ environment:
     - ID: Ubu20
       DTS: datalad_xnat
       APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu2004
-      INSTALL_SYSPKGS: python3-virtualenv
+      INSTALL_SYSPKGS: python3-venv
       # system git-annex is way too old, use better one
-      INSTALL_GITANNEX: git-annex -m deb-url --url http://snapshot.debian.org/archive/debian/20210906T204127Z/pool/main/g/git-annex/git-annex_8.20210903-1_amd64.deb
+      INSTALL_GITANNEX: git-annex -m snapshot
+      CODECOV_BINARY: https://uploader.codecov.io/latest/linux/codecov
     # Windows core tests
     - ID: WinP39core
       # ~35 min
       DTS: datalad_xnat
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022
       # Python version specification is non-standard on windows
       PY: 39-x64
       INSTALL_GITANNEX: git-annex -m datalad/packages
@@ -82,12 +83,20 @@ environment:
       PY: 3.8
       INSTALL_GITANNEX: git-annex
       DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets
+      CODECOV_BINARY: https://uploader.codecov.io/latest/macos/codecov
 
 matrix:
   allow_failures:
     - KNOWN2FAIL: 1
 
 
+# do not run the CI if only documentation changes were made
+# documentation builds are tested elsewhere and cheaper
+skip_commits:
+  files:
+    - docs/
+
+
 # it is OK to specify paths that may not exist for a particular test run
 cache:
   # pip cache
@@ -136,13 +145,15 @@ init:
 install:
   # place a debug setup helper at a convenient location
   - cmd: copy tools\ci\appveyor_env_setup.bat C:\\datalad_debug.bat
+  # Missing system software
+  - sh: "[ -n \"$INSTALL_SYSPKGS\" ] && ( [ \"x${APPVEYOR_BUILD_WORKER_IMAGE}\" = \"xmacOS\" ] && brew install -q ${INSTALL_SYSPKGS} || { sudo apt-get update -y && sudo apt-get install --no-install-recommends -y ${INSTALL_SYSPKGS}; } ) || true"
   # If a particular Python version is requested, use env setup (using the
   # appveyor provided environments/installation). Note, these are broken
   # on the ubuntu images
   # https://help.appveyor.com/discussions/problems/28217-appveyor-ubunu-image-with-python3-lzma-module
-  # Otherwise create a virtualenv using the default Python 3, to enable uniform
+  # Otherwise create a venv using the default Python 3, to enable uniform
   # use of python/pip executables below
-  - sh: "[ \"x$PY\" != x ] && . ${HOME}/venv${PY}/bin/activate || virtualenv -p 3 ${HOME}/dlvenv && . ${HOME}/dlvenv/bin/activate; ln -s \"$VIRTUAL_ENV\" \"${HOME}/VENV\""
+  - sh: "[ \"x$PY\" != x ] && . ${HOME}/venv${PY}/bin/activate || python3 -m venv ${HOME}/dlvenv && . ${HOME}/dlvenv/bin/activate; ln -s \"$VIRTUAL_ENV\" \"${HOME}/VENV\""
   - cmd: "set PATH=C:\\Python%PY%;C:\\Python%PY%\\Scripts;%PATH%"
   # deploy the datalad installer, override version via DATALAD_INSTALLER_VERSION
   - cmd:
@@ -152,12 +163,14 @@ install:
       python -m pip install datalad-installer
       )
   - sh: python -m pip install datalad-installer${DATALAD_INSTALLER_VERSION:-}
-  # Missing system software
-  - sh: "[ -n \"$INSTALL_SYSPKGS\" ] && ( [ \"x${APPVEYOR_BUILD_WORKER_IMAGE}\" = \"xmacOS\" ] && brew install -q ${INSTALL_SYSPKGS} || { sudo apt-get update -y && sudo apt-get install --no-install-recommends -y ${INSTALL_SYSPKGS}; } ) || true"
   # Install git-annex on windows, otherwise INSTALL_SYSPKGS can be used
   # deploy git-annex, if desired
   - cmd: IF DEFINED INSTALL_GITANNEX datalad-installer --sudo ok %INSTALL_GITANNEX%
-  - sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer --sudo ok ${INSTALL_GITANNEX}"
+  # in case of a snapshot installation, use the following approach to adjust
+  # the PATH as necessary
+  - sh: "[ -n \"${INSTALL_GITANNEX}\" ] && datalad-installer -E ${HOME}/dlinstaller_env.sh --sudo ok ${INSTALL_GITANNEX}"
+  # add location of datalad installer results to PATH
+  - sh: "[ -f ${HOME}/dlinstaller_env.sh ] && . ${HOME}/dlinstaller_env.sh || true"
 
 
 #before_build:
@@ -184,18 +197,16 @@ test_script:
   - sh: mkdir __testhome__
   - cd __testhome__
   - cmd: python -m pytest -s -v -m "not (turtle)" --cov=datalad_xnat --pyargs %DTS%
-  - sh: python -m pytest -s -v -m "not (turtle)" --cov=datalad_xnat --pyargs ${DTS}
+  - sh:  python -m pytest -s -v -m "not (turtle)" --cov=datalad_xnat --pyargs ${DTS}
+
 
 after_test:
-  # move coverage into the project dir to make default handling applicable
-  - cmd: move .coverage ..
-  - sh: mv .coverage ..
-  - cd ..
-  # use windows codecov uploader package
-  - cmd: choco install codecov
-  - cmd: python -m coverage xml
-  - cmd: codecov -f coverage.xml
-  - sh: bash <(curl -sfS https://codecov.io/bash)
+  - python -m coverage xml
+  - cmd: curl -fsSL -o codecov.exe "https://uploader.codecov.io/latest/windows/codecov.exe"
+  - cmd: .\codecov.exe -f "coverage.xml"
+  - sh: "curl -Os $CODECOV_BINARY"
+  - sh: chmod +x codecov
+  - sh: ./codecov
 
 
 #on_success:

.codeclimate.yml

@@ -0,0 +1,17 @@
+version: "2"
+checks:
+  file-lines:
+    config:
+      threshold: 500
+plugins:
+  bandit:
+    enabled: true
+    checks:
+      assert_used:
+        enabled: false
+exclude_patterns:
+- "_datalad_buildsupport/"
+- "versioneer.py"
+- "*/_version.py"
+- "tools/"
+- "**/tests/"

.codespellrc

@@ -0,0 +1,4 @@
+[codespell]
+skip = .venv,venvs,.git,build,*.egg-info,*.lock,.asv,.mypy_cache,.tox,fixtures,_version.py,*.pem,versioneer.py
+# ignore-words-list = 
+# exclude-file = 

.datalad-release-action.yaml

@@ -0,0 +1,45 @@
+fragment-directory: changelog.d
+
+categories:
+  - name: 💥 Breaking Changes
+    bump: major
+    label: semver-major
+    label-color: C5000B
+    label-description: Increment the major version when merged
+  - name: 🚀 Enhancements and New Features
+    bump: minor
+    label: semver-minor
+    label-color: F1A60E
+    label-description: Increment the minor version when merged
+  - name: 🐛 Bug Fixes
+    label: semver-patch
+    label-color: "870048"
+    label-description: Increment the patch version when merged
+  - name: 🔩 Dependencies
+    label: semver-dependencies
+    label-color: 8732bc
+    label-description: Update one or more dependencies' versions
+  - name: 📝 Documentation
+    label: semver-documentation
+    label-color: cfd3d7
+    label-description: Changes only affect the documentation
+  - name: 🏠 Internal
+    label: semver-internal
+    label-color: "696969"
+    label-description: Changes only affect the internal API
+  - name: 🏎 Performance
+    label: semver-performance
+    label-color: f4b2d8
+    label-description: Improve performance of an existing feature
+  - name: 🧪 Tests
+    label: semver-tests
+    label-color: ffd3cc
+    label-description: Add or improve existing tests
+
+extra-labels:
+  - name: release
+    color: 007f70
+    description: Create a release when this pr is merged
+  - name: CHANGELOG-missing
+    color: 5B0406
+    description: When a PR does not contain add a changelog item, yet

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    commit-message:
+      prefix: "[gh-actions]"
+      include: scope

.github/workflows/add-changelog-snippet.yml

@@ -0,0 +1,33 @@
+name: Add changelog.d snippet
+
+on:
+  # This action should be run in workflows triggered by `pull_request_target`
+  # (not by regular `pull_request`!)
+  pull_request_target:
+    # Run whenever the PR is pushed to, receives a label, or is created with
+    # one or more labels:
+    types: [synchronize, labeled]
+
+# Prevent the workflow from running multiple jobs at once when a PR is created
+# with multiple labels:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  add:
+    runs-on: ubuntu-latest
+    # Only run on PRs that have the "CHANGELOG-missing" label:
+    if: contains(github.event.pull_request.labels.*.name, 'CHANGELOG-missing')
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+      - name: Add changelog snippet
+        uses: datalad/release-action/add-changelog-snippet@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          rm-labels: CHANGELOG-missing

.github/workflows/docbuild.yml

@@ -12,7 +12,7 @@ jobs:
       run: |
         git config --global user.email "test@github.land"
         git config --global user.name "GitHub Almighty"
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:

.github/workflows/release.yml

@@ -0,0 +1,30 @@
+name: Auto-release on PR merge
+
+on:
+  # This action should be run in workflows triggered by `pull_request_target`
+  # (not by regular `pull_request`!)
+  pull_request_target:
+    branches:
+      # Create a release whenever a PR is merged into one of these branches:
+      - main
+    types:
+      - closed
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    # Only run for merged PRs with the "release" label:
+    if: github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release')
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+        with:
+          # Check out all history so that the previous release tag can be
+          # found:
+          fetch-depth: 0
+
+      - name: Prepare release
+        uses: datalad/release-action/release@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          #pypi-token: ${{ secrets.PYPI_TOKEN }}

.github/workflows/test_crippledfs.yml

@@ -27,7 +27,7 @@ jobs:
       run: |
         git config --global user.email "test@github.land"
         git config --global user.name "GitHub Almighty"
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:

.gitignore

@@ -11,3 +11,5 @@ docs/build
 docs/source/generated
 build/
 dist/
+.idea/
+venvs/

CONTRIBUTING.md

@@ -0,0 +1,30 @@
+## Releasing with GitHub Actions and pull requests
+
+New releases of this project are created via a GitHub Actions workflow using
+[datalad/release-action](https://github.com/datalad/release-action), which was
+inspired by [`auto`](https://github.com/intuit/auto).  Whenever a pull request
+is merged into `main` that has the "`release`" label, the workflow updates the
+changelog based on the pull requests since the previous release, commits the
+results, tags the new commit with the next version number, creates a GitHub
+release for the tag, builds an sdist & wheel for the project, and uploads the
+sdist & wheel to PyPI.
+
+### CHANGELOG entries and labelling pull requests
+
+This project uses [scriv](https://github.com/nedbat/scriv/) to maintain the
+[CHANGELOG.md](./CHANGELOG.md) file.  Adding the label `CHANGELOG-missing` to a
+PR triggers a workflow to add a new `scriv` changelog fragment under
+`changelog.d/` using the PR title as the content.  That produced changelog
+snippet can subsequently be tuned to improve the prospective CHANGELOG entry.
+The changelog section that the fragment is added under depends on the `semver-`
+label added to the PR:
+
+- `semver-minor` — for changes corresponding to an increase in the minor
+  version component
+- `semver-patch` — for changes corresponding to an increase in the patch/micro
+  version component
+- `semver-internal` — for changes only affecting the internal API
+- `semver-documentation` — for changes only affecting the documentation
+- `semver-tests` — for changes to tests
+- `semver-dependencies` — for updates to dependency versions
+- `semver-performance` — for performance improvements

MANIFEST.in

@@ -1,6 +1,7 @@
 include CONTRIBUTORS LICENSE versioneer.py
 include datalad_xnat/_version.py
 graft _datalad_buildsupport
+graft datalad_helloworld
 graft docs
 prune docs/build
 global-exclude *.py[cod]

Makefile

@@ -11,8 +11,7 @@ release-pypi:
 	git grep -q '^##.*??? ??' -- CHANGELOG.md && exit 1 || true
 	# avoid upload of stale builds
 	test ! -e dist
-	$(PYTHON) setup.py sdist
-	python setup.py bdist_wheel
+	$(PYTHON) setup.py sdist bdist_wheel
 	twine upload dist/*
 
 update-buildsupport:

README.md

@@ -98,4 +98,4 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
 
 <!-- ALL-CONTRIBUTORS-LIST:END -->
 
-This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

_datalad_buildsupport/formatters.py

@@ -75,7 +75,7 @@ def _mk_title(self, prog):
 
     def _mk_name(self, prog, desc):
         """
-        this method is in consitent with others ... it relies on
+        this method is in consistent with others ... it relies on
         distribution
         """
         desc = desc.splitlines()[0] if desc else 'it is in the name'

_datalad_buildsupport/setup.py

@@ -9,12 +9,11 @@
 import datetime
 import os
 
-from distutils.core import Command
-from distutils.errors import DistutilsOptionError
 from os.path import (
     dirname,
     join as opj,
 )
+from setuptools import Command, DistutilsOptionError
 from setuptools.config import read_configuration
 
 import versioneer
@@ -157,40 +156,6 @@ def run(self):
                     f.write(formatted)
 
 
-class BuildRSTExamplesFromScripts(Command):
-    description = 'Generate RST variants of example shell scripts.'
-
-    user_options = [
-        ('expath=', None, 'path to look for example scripts'),
-        ('rstpath=', None, 'output path for RST files'),
-    ]
-
-    def initialize_options(self):
-        self.expath = opj('docs', 'examples')
-        self.rstpath = opj('docs', 'source', 'generated', 'examples')
-
-    def finalize_options(self):
-        if self.expath is None:
-            raise DistutilsOptionError('\'expath\' option is required')
-        if self.rstpath is None:
-            raise DistutilsOptionError('\'rstpath\' option is required')
-        self.announce('Converting example scripts')
-
-    def run(self):
-        opath = self.rstpath
-        if not os.path.exists(opath):
-            os.makedirs(opath)
-
-        from glob import glob
-        for example in glob(opj(self.expath, '*.sh')):
-            exname = os.path.basename(example)[:-3]
-            with open(opj(opath, '{0}.rst'.format(exname)), 'w') as out:
-                fmt.cmdline_example_to_rst(
-                    open(example),
-                    out=out,
-                    ref='_example_{0}'.format(exname))
-
-
 class BuildConfigInfo(Command):
     description = 'Generate RST documentation for all config items.'