continuous_integration.yml

name: continuous integration

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  release:
    types:
      - published
      - edited


jobs:
  test:
    runs-on: self-hosted
    timeout-minutes: 1440  # Set to 24 hours
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # Check out full history for `setuptools_scm`
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: >
            3.12
          check-latest: true
      - name: Install Python dependencies
        run: |
          python -m pip install \
            awscli \
            pygithub \
            setuptools_scm
      - name: Set up metadata
        id: metadata
        shell: python
        run: |
          import os

          from github import Github
          from setuptools_scm import get_version


          pushed_at = "${{ github.event.repository.pushed_at }}"
          ref = "${{ github.ref }}"
          repository = "${{ github.repository }}"
          sha = "${{ github.sha }}"
          url = "${{ github.event.repository.html_url }}"

          registry = "${{ secrets.REGISTRY }}"
          token = "${{ secrets.GITHUB_TOKEN }}"

          repository_data = Github(token).get_repo(repository)
          repository_owner, repository_name = repository.lower().split("/")
          reference, type, name = ref.split("/")[:3]
          if reference != "refs":
              raise ValueError(f"Unknown reference: {reference}")
          if type == "heads":
              tag = {"main": "latest"}.get(name, name)
              push = True
          elif type == "tags":
              tag = name
              push = True
          elif type == "pull":
              tag = f"{type}-{name}"
              push = False
          else:
              raise ValueError(f"Unknown reference type: {type}")

          path = f"{registry}/{repository_name}"
          cache_from = f"type=registry,ref={path}:buildcache"
          output = dict(
              cache_from=cache_from,
              cache_to=f"{cache_from},compression=zstd,mode=max" if push else "",
              labels=[
                  f'org.opencontainers.image.created="{pushed_at}"',
                  f'org.opencontainers.image.authors="Lea Waller <lea@fmri.science>"',
                  f'org.opencontainers.image.url="{url}"',
                  f'org.opencontainers.image.documentation="{url}"',
                  f'org.opencontainers.image.source="{url}"',
                  f'org.opencontainers.image.version="{get_version()}"',
                  f'org.opencontainers.image.revision="{sha}"',
                  f'org.opencontainers.image.licenses="{repository_data.license.spdx_id}"',
                  f'org.opencontainers.image.title="{repository_name}"',
                  f'org.opencontainers.image.description="{repository_data.description}"',
              ],
              push=str(push).lower(),
              build_tag=f"{repository_name}:{tag}",
              push_tags=[
                  f"{path}:{tag}",
                  f"docker.io/{repository_owner}/{repository_name}:{tag}",
              ]
              if push
              else [],
              singularity_name=f"{repository_name}-{tag}.sif",
          )

          with open(os.environ["GITHUB_OUTPUT"], "at") as file_handle:
              for key, value in output.items():
                  if isinstance(value, list):
                      file_handle.write("\n".join([f"{key}<<eof", *value, "eof"]) + "\n")
                  else:
                      file_handle.write(f"{key}={value}\n")

      - name: Login to DockerHub
        uses: docker/login-action@v3
        if: ${{ steps.metadata.outputs.push == 'true' }}
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Login to container registry
        uses: docker/login-action@v3
        if: ${{ steps.metadata.outputs.push == 'true' }}
        with:
          registry: ${{ secrets.REGISTRY }}
          username: ${{ secrets.REGISTRY_USERNAME }}
          password: ${{ secrets.REGISTRY_PASSWORD }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Build container
        uses: docker/build-push-action@v5
        with:
          cache-from: ${{ steps.metadata.outputs.cache_from }}
          cache-to: ${{ steps.metadata.outputs.cache_to }}
          context: "."
          file: "./Dockerfile"
          labels: |
            ${{ steps.metadata.outputs.labels }}
          load: true
          platforms: linux/amd64
          push: false
          tags: |
            ${{ steps.metadata.outputs.build_tag }}

      - name: Cache data
        uses: actions/cache@v4
        with:
          key: resource-cache
          path: /resource-cache
      - name: Setup integration test container
        run: |
          docker run \
            --detach \
            --rm \
            --name="test" \
            --volume="/resource-cache:/var/cache/halfpipe" \
            --volume="/home/runner:/home/runner" \
            --volume="$(pwd):/halfpipe" \
            --workdir="/halfpipe" \
            --entrypoint="tail" \
            ${{ steps.metadata.outputs.build_tag }} \
            --follow /dev/null
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Run integration tests
        shell: docker exec test bash -x -e {0}
        env:
          # Enable pytest-github-actions-annotate-failures
          GITHUB_ACTIONS: true
        run: |
          # Install the Freesurfer license
          cat > ${FREESURFER_HOME}/license.txt <<EOF
          halfpipe@fmri.science
          73053
           *C8FMo8yrzcZA
           FSCA7.lDgNIow
           e1n3o06osRFD3qtBRgsTy9f9bQHGOpY/riIUrHoEx5c=
          EOF
          # Run the integration tests
          py.test -k 'not test_consistency' \
            --cov --cov-report=xml --dist=load \
            --numprocesses=2
      - name: Run test_consistency only on release
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          # Install the Freesurfer license
          cat > ${FREESURFER_HOME}/license.txt <<EOF
          halfpipe@fmri.science
          73053
          *C8FMo8yrzcZA
          FSCA7.lDgNIow
          e1n3o06osRFD3qtBRgsTy9f9bQHGOpY/riIUrHoEx5c=
          EOF
          # Run the tests
          py.test tests/workflows/test_consistency \
            --cov --cov-report=xml --dist=load \
            --numprocesses=2
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          fail_ci_if_error: true
      - name: Stop integration test container
        run: |
          docker kill test

      - name: Push container to registry
        if: ${{ steps.metadata.outputs.push == 'true' }}
        uses: docker/build-push-action@v5
        with:
          cache-from: ${{ steps.metadata.outputs.cache_from }}
          context: "."
          file: "./Dockerfile"
          labels: |
            ${{ steps.metadata.outputs.labels }}
          platforms: linux/amd64
          push: true
          tags: |
            ${{ steps.metadata.outputs.push_tags }}

      - name: Build Singularity 2.x container
        if: ${{ steps.metadata.outputs.push == 'true' }}
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
        run: |
          mkdir --parents singularity-2
          docker run \
            --privileged \
            --rm \
            --volume="/var/run/docker.sock:/var/run/docker.sock" \
            --volume="$(pwd)/singularity-2:/output" \
            quay.io/singularity/docker2singularity:v2.6 \
            --name ${{ steps.metadata.outputs.singularity_name }} \
            ${{ steps.metadata.outputs.build_tag }}
          image_file=$(find singularity-2 -type f | head --lines=1)
          aws s3 mv \
            --only-show-errors \
            --acl public-read \
            ${image_file} \
            s3://singularity/

      - name: Build Singularity container
        if: ${{ steps.metadata.outputs.push == 'true' }}
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          AWS_ENDPOINT_URL: ${{ secrets.AWS_ENDPOINT_URL }}
        run: |
          mkdir --parents singularity
          docker run \
            --rm \
            --volume="/var/run/docker.sock:/var/run/docker.sock" \
            --volume="$(pwd)/singularity:/output" \
            quay.io/singularity/singularity:v4.0.1 \
            build \
            --disable-cache \
            /output/${{ steps.metadata.outputs.singularity_name }} \
            docker-daemon://${{ steps.metadata.outputs.build_tag }}
          image_file=$(find singularity -type f | head --lines=1)
          aws s3 mv \
            --only-show-errors \
            --acl public-read \
            ${image_file} \
            s3://singularity/