Set up for testing git-annex on non-GitHub clients

.github/workflows/build-ubuntu.yaml

@@ -128,6 +128,40 @@ jobs:
             git-annex[-_]*.*
             dist/build-version
 
+      - name: Clone datalad/git-annex-ci-client-jobs
+        uses: actions/checkout@v2
+        with:
+          repository: datalad/git-annex-ci-client-jobs
+          fetch-depth: 1
+          path: client-jobs
+          ssh-key: ${{ secrets.CLIENT_JOBS_SSH_KEY }}
+
+      - name: Push installer to datalad/git-annex-ci-client-jobs
+        run: |
+          set -ex -o pipefail
+          git checkout --orphan build
+          git rm -rf .
+          cp -i ../git-annex*.deb .
+          git add git-annex*.deb
+          buildno="${{ github.run_number }}"
+          git commit \
+            --gpg-sign=13A1093296154584245E0300C98FC49D36DAB17F \
+            -m "Installer artifact from build $buildno"
+          git ls-remote --heads origin "build-*" | cut -f2 | cut -d/ -f3- > builds.txt
+          git ls-remote --heads origin "result-*" | cut -f2 | cut -d/ -f3- > results.txt
+          for clientid in ndoli smaug
+          do build_branch="build-$clientid-$buildno"
+             result_branch="result-$clientid-$buildno"
+             if grep -Fqx "$result_branch" results.txt
+             then echo "[INFO] Result branch $result_branch already exists; not creating build branch"
+             elif grep -Fqx "$build_branch" builds.txt
+             then echo "[INFO] Build branch $build_branch already exists; leaving alone"
+             else git branch "$build_branch" build
+                  git push origin "$build_branch"
+             fi
+          done
+        working-directory: client-jobs
+
       - name: Create new release
         if: github.event.inputs.commitish != ''
         run: |

.github/workflows/template/Makefile

@@ -1,2 +1,4 @@
+CLIENTS = ../../../clients/clients.yaml
+
 workflows :
-	python3 -m pip_run -q -- mkworkflows.py specs.yml 'build-{{ostype}}.yaml.j2' ..
+	python3 -m pip_run -q -- mkworkflows.py specs.yml 'build-{{ostype}}.yaml.j2' $(CLIENTS) ..

.github/workflows/template/build-{{ostype}}.yaml.j2

@@ -252,6 +252,49 @@ jobs:
             git-annex[-_]*.*
             dist/build-version
 
+    {% if clients %}
+      {% if ostype != "ubuntu" %}
+      - name: Import GPG key
+        run: |
+          # private key for signing - comes from secrets
+          echo '${{ secrets.datalad_builder_gpgkey }}' | gpg --import
+
+      {% endif %}
+      - name: Clone datalad/git-annex-ci-client-jobs
+        uses: actions/checkout@v2
+        with:
+          repository: datalad/git-annex-ci-client-jobs
+          fetch-depth: 1
+          path: client-jobs
+          ssh-key: ${{ secrets.CLIENT_JOBS_SSH_KEY }}
+
+      - name: Push installer to datalad/git-annex-ci-client-jobs
+        run: |
+          set -ex -o pipefail
+          git checkout --orphan build
+          git rm -rf .
+          cp -i ../{{installer_glob}} .
+          git add {{installer_glob}}
+          buildno="${{ github.run_number }}"
+          git commit \
+            --gpg-sign=13A1093296154584245E0300C98FC49D36DAB17F \
+            -m "Installer artifact from build $buildno"
+          git ls-remote --heads origin "build-*" | cut -f2 | cut -d/ -f3- > builds.txt
+          git ls-remote --heads origin "result-*" | cut -f2 | cut -d/ -f3- > results.txt
+          for clientid in {{clients|join(" ")}}
+          do build_branch="build-$clientid-$buildno"
+             result_branch="result-$clientid-$buildno"
+             if grep -Fqx "$result_branch" results.txt
+             then echo "[INFO] Result branch $result_branch already exists; not creating build branch"
+             elif grep -Fqx "$build_branch" builds.txt
+             then echo "[INFO] Build branch $build_branch already exists; leaving alone"
+             else git branch "$build_branch" build
+                  git push origin "$build_branch"
+             fi
+          done
+        working-directory: client-jobs
+
+    {% endif %}
       - name: Create new release
         if: github.event.inputs.commitish != ''
         run: |

.github/workflows/template/mkworkflows.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-__requires__ = ["Jinja2 ~= 2.11", "PyYAML ~= 5.3"]
+__requires__ = ["Jinja2 >= 2.11", "PyYAML >= 5.3"]
 from   pathlib import Path
 import sys
 import jinja2
@@ -19,15 +19,19 @@ def jinja_render(template, context):
         template.replace("${{", PLACEHOLDER),
         trim_blocks   = True,
         lstrip_blocks = True,
-     ).render(context)
+    ).render(context)
     return rendered.replace(PLACEHOLDER, "${{")
 
 def main():
-    specs_file, template_file, workflows_dir = map(Path, sys.argv[1:])
+    specs_file, template_file, clients_file, workflows_dir = map(Path, sys.argv[1:])
     with specs_file.open() as fp:
         specs = yaml.safe_load(fp)
+    with clients_file.open() as fp:
+        clients = list(yaml.safe_load(fp).keys())
     template = template_file.read_text()
     for sp in specs:
+        if sp["ostype"] == "ubuntu":
+            sp["clients"] = clients
         workflow = jinja_render(template, sp)
         filename = jinja_render(template_file.with_suffix("").name, sp)
         (workflows_dir / filename).write_text(workflow + "\n")

.github/workflows/template/specs.yml

@@ -13,6 +13,7 @@
   artifact_install_steps:
     - sudo dpkg -i git-annex*.deb
   test_datalad: true
+  installer_glob: git-annex*.deb
 
 - ostype: macos
   osname: macOS

README.md

@@ -14,3 +14,9 @@ configuration and associated scripts.
 [![Ubuntu](https://github.com/datalad/git-annex/workflows/Build%20git-annex%20on%20Ubuntu/badge.svg)](https://github.com/datalad/git-annex/actions?query=workflow%3A%22Build+git-annex+on+Ubuntu%22)
 [![macOS](https://github.com/datalad/git-annex/workflows/Build%20git-annex%20on%20macOS/badge.svg)](https://github.com/datalad/git-annex/actions?query=workflow%3A%22Build+git-annex+on+macOS%22)
 [![Windows](https://github.com/datalad/git-annex/workflows/Build%20git-annex%20on%20Windows/badge.svg)](https://github.com/datalad/git-annex/actions?query=workflow%3A%22Build+git-annex+on+Windows%22)
+
+## Client Tests
+
+| Client | Test Status |
+| --- | --- |
+| ndoli | ![Overall test status](https://github.com/datalad/git-annex-ci-client-jobs/raw/master/badges/ndoli.svg) ![git-annex-home test status](https://github.com/datalad/git-annex-ci-client-jobs/raw/master/badges/ndoli/git-annex-home.svg) ![git-annex-tmp test status](https://github.com/datalad/git-annex-ci-client-jobs/raw/master/badges/ndoli/git-annex-tmp.svg) |

clients/.gitignore

@@ -0,0 +1,3 @@
+*.lock
+__pycache__/
+venv/

clients/README.md

@@ -0,0 +1,96 @@
+Testing git-annex Builds on Local Clients
+=========================================
+
+In order to test our builds of git-annex on non-GitHub servers (hereafter
+"clients") without the use of a GitHub Actions self-hosted runner, we have
+established the following setup:
+
+- The file `clients/clients.yaml` in this repository stores a list of client
+  IDs and other per-client information
+
+- The GitHub Actions build job pushes its built Debian binary packages to
+  [datalad/git-annex-ci-client-jobs](https://github.com/datalad/git-annex-ci-client-jobs)
+  by creating a `build-$CLIENTID-$BUILDNO` branch for each client containing
+  only the build's `*.deb` file
+
+- On each client, a cronjob is established that fetches the `build-*` branches
+  for that client; for each branch fetched, the `*.deb` is installed, tests are
+  run, the results are saved to a `result-$CLIENTID-$BUILDNO` branch, and the
+  original branch is deleted.
+
+- A workflow on datalad/git-annex-ci-client-jobs is established that is run
+  whenever a `result-*` branch is pushed to.  This workflow uses the latest
+  result statuses to create SVG badges (via shields.io) and save them in the
+  repository; uploads the contents of the `result-*` branches as build
+  artifacts for fetching with [tinuous](https://github.com/con/tinuous); and
+  deletes the `result-*` branch.
+
+- A table (generated by `mktable.py`) is added to the main README in this
+  repository displaying the SVG badges from datalad/git-annex-ci-client-jobs.
+
+Currently, only testing on Debian-based systems is supported.
+
+Setting up a New Client
+-----------------------
+
+- Determine a unique ID for the client.  Client IDs should consist of only
+  ASCII letters, numbers, underscores, and/or hyphens.
+
+- Add an entry to `clients.yaml` for the client containing one or more tests
+  (See "`clients.yaml` Format" below)
+
+- Regenerate the `build-*.yaml` workflows by running `make -C
+  .github/workflows/template`
+
+- Set up permissions for the client to be able to pull from & write to
+  datalad/git-annex-ci-client-jobs
+
+- On the client:
+
+    - Configure a name & e-mail in Git
+
+    - Clone this repository.  Passing `--single-branch` is recommended so as
+      not to include the mirror of git-annex's repository.
+
+    - Import `.github/workflows/tools/datalad-builder-key.asc` into GPG
+
+    - Create a Python virtual environment in this directory in the clone and
+      install the packages listed in `requirements.txt` in it
+
+        - Alternatively, create a conda environment named "`testannex`" using
+          the `environment.yml` file in this directory
+
+    - Create a cronjob with a command of the form:
+
+            cd /path/to/clone/clients && chronic flock -n -E 0 .lock venv/bin/python testannex.py CLIENTID /path/to/job/dir
+
+      where `CLIENTID` is replaced by the ID of the client and
+      `/path/to/job/dir` is replaced by the path to the location at which to
+      clone the jobs repository.
+
+        - Alternatively, if using Conda:
+
+                cd /path/to/clone/clients && chronic flock -n -E 0 .lock ./testannex.sh CLIENTID /path/to/job/dir
+
+
+`clients.yaml` Format
+---------------------
+
+The client configuration file consists of a YAML mapping in which the keys are
+client IDs and the values are sub-mappings with the following fields:
+
+- `tests` — A mapping in which the keys are test names and each value is the
+  body of a script to run for the test.  The test scripts are run in an
+  environment with the following properties:
+
+    - git-annex is installed to a temporary location, the `bin` folder of which
+      is prepended to `PATH`
+
+    - The environment variable `BUILDNO` is set to the run number of the
+      `build-ubuntu.yaml` workflow that produced the package
+
+    - The current working directory is set to the `clients/` directory in the
+      clone of this repository
+
+- `shell` — A string giving the path to the shell to use to run the test
+  bodies; defaults to `"/bin/bash"`

clients/clients.yaml

@@ -0,0 +1,24 @@
+ndoli:
+  tests:
+    git-annex-home: |
+      set -eux
+      source testlib.sh
+      workdir_base "$HOME/git-annex-ci"
+      git annex version
+      timeout 7200 git annex test
+
+    git-annex-tmp: |
+      set -eux
+      source testlib.sh
+      workdir_base /dartfs/rc/lab/D/DBIC/DBIC/archive/tmp
+      git annex version
+      timeout 7200 git annex test
+
+smaug:
+  tests:
+    git-annex: |
+      set -eux
+      source testlib.sh
+      workdir_base /mnt/datasets/datalad/git-annex-build-client
+      git annex version
+      timeout 3600 git annex test

clients/environment.yml

@@ -0,0 +1,10 @@
+name: testannex
+channels:
+  - conda-forge
+dependencies:
+  - git >=2.32
+  - python >=3.8
+  - click >=8.0
+  - click-loglevel >=0.2
+  - pydantic >=1.9.0
+  - ruamel.yaml >=0.15,<1

clients/mktable.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""Generate a table of test status badges for a Markdown README"""
+from testannex import parse_clients
+
+BADGE_BASE = "https://github.com/datalad/git-annex-ci-client-jobs/raw/master/badges"
+
+
+def main():
+    cfg = parse_clients()
+    print("| Client | Test Status |")
+    print("| --- | --- |")
+    for clientid, cl in cfg.items():
+        print(f"| {clientid} |", end="")
+        print(f" ![Overall test status]({BADGE_BASE}/{clientid}.svg)", end="")
+        for test in cl.tests.keys():
+            print(f" ![{test} test status]({BADGE_BASE}/{clientid}/{test}.svg)", end="")
+        print(" |")
+
+
+if __name__ == "__main__":
+    main()

clients/requirements.txt

@@ -0,0 +1,4 @@
+click >= 8.0
+click-loglevel ~= 0.2
+pydantic ~= 1.9
+ruamel.yaml ~= 0.15