tox.yml

name: Run SAGE_ROOT/tox.ini

## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments,
## whenever a GitHub pull request is opened or synchronized in a repository
## where GitHub Actions are enabled.
##
## It builds and checks some sage spkgs as defined in TARGETS.
##
## A job succeeds if there is no error.
##
## The build is run with "make V=0", so the build logs of individual packages are suppressed.
##
## At the end, all package build logs that contain an error are printed out.
##
## After all jobs have finished (or are canceled) and a short delay,
## tar files of all logs are made available as "build artifacts".

#on: [push, pull_request]

on:
  pull_request:
    types: [opened, synchronize]
  push:
    tags:
      - '*'
  workflow_dispatch:
    # Allow to run manually

env:
  TARGETS_PRE: sagelib-build-deps
  TARGETS: build doc-html
  TARGETS_OPTIONAL: ptest

jobs:
  docker:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix:
        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, ubuntu-impish, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, centos-7, centos-8, gentoo, gentoo-python3.7, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386]
        tox_packages_factor: [minimal, standard]
    env:
      TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
      DOCKER_TARGETS: configured with-targets with-targets-optional
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 500
      - name: fetch tags
        run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: free disk space
        run: |
          df -h
          sudo swapoff -a
          sudo rm -f /swapfile
          sudo apt-get clean
          docker rmi $(docker image ls -aq)
          echo "Largest packages:"
          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 50
          sudo apt-get --fix-broken --yes remove $(dpkg-query -f '${Package}\n' -W | grep -E '^(ghc-|google-cloud-sdk|google-chrome|firefox|mysql-server|dotnet-sdk|hhvm|mono)') || echo "(error ignored)"
          df -h
      - name: Install test prerequisites
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
          sudo apt-get clean
          df -h
      - name: Try to login to docker.pkg.github.com
        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
        run: |
            TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
            if [ -z "$TOKEN" ]; then
              TOKEN="${{ secrets.GITHUB_TOKEN }}"
            fi
            if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then
              echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
            fi
      - run: |
          set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
      - name: Copy logs from the docker image or build container
        run: |
          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
          cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
          if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
          if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
        if: always()
      - uses: actions/upload-artifact@v1
        with:
          path: artifacts
          name: ${{ env.LOGS_ARTIFACT_NAME }}
        if: always()
      - name: Print out logs for immediate inspection
        # and markup the output with GitHub Actions logging commands
        run: |
          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
        if: always()
      - name: List docker images
        run: |
          if [ -f .tox/$TOX_ENV/Dockertags ]; then
             cat .tox/$TOX_ENV/Dockertags
          fi
        if: always()

  local-macos:

    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        tox_system_factor: [homebrew-macos, conda-forge-macos]
        tox_packages_factor: [minimal, standard]
        # As of 2021-03, default xcode is 12.4
        # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md#xcode
        xcode_version_factor: [default]
        os: [ macos-10.15, macos-11.0 ]
        include:
          # Test xcode 11.7 only on macos-10.15
          - tox_system_factor: homebrew-macos
            tox_packages_factor: minimal
            xcode_version_factor: 11.7
            os: macos-10.15
          - tox_system_factor: homebrew-macos
            tox_packages_factor: standard
            xcode_version_factor: 11.7
            os: macos-10.15
          # python3_xcode is only accepted if enough packages are available from the system
          # --> to test "minimal", we will need https://trac.sagemath.org/ticket/30949
          - tox_system_factor: homebrew-macos-python3_xcode
            tox_packages_factor: standard
            xcode_version_factor: 11.7
            os: macos-10.15
          - tox_system_factor: homebrew-macos-python3_xcode
            tox_packages_factor: standard
            xcode_version_factor: default
            os: macos-10.15
          - tox_system_factor: homebrew-macos-python3_xcode
            tox_packages_factor: standard
            xcode_version_factor: default
            os: macos-11.0
          - tox_system_factor: homebrew-macos-python3_xcode-nokegonly
            tox_packages_factor: standard
            xcode_version_factor: default
            os: macos-11.0
          # likewise for python3_pythonorg
          - tox_system_factor: homebrew-macos-python3_pythonorg
            tox_packages_factor: standard
            xcode_version_factor: default
            os: macos-11.0
          # conda-forge-macos-environment
          - tox_system_factor: conda-forge-macos
            tox_packages_factor: environment
            xcode_version_factor: default
            os: macos-11.0
          - tox_system_factor: conda-forge-macos
            tox_packages_factor: environment-optional
            xcode_version_factor: default
            os: macos-11.0
    env:
      TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
    steps:
      - uses: actions/checkout@v2
      - name: Select Xcode version
        run: |
          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
      - name: Install test prerequisites
        run: |
          brew install tox
      - name: Install python3 from python.org
        # As of 2020-03-30 (https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md),
        # Python 3.7.7 is installed on GitHub Actions runners.  But we install our own copy from the python.org binary package.
        run: |
          curl -o python3.pkg https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg
          sudo installer -verbose -pkg python3.pkg -target /
        if: contains(matrix.tox_system_factor, 'python3_pythonorg')
      - name: Build and test with tox
        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
        # For doctesting, we use a lower parallelization to avoid timeouts.
        run: |
          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
      - name: Prepare logs artifact
        run: |
          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
        if: always()
      - uses: actions/upload-artifact@v1
        with:
          path: artifacts
          name: ${{ env.LOGS_ARTIFACT_NAME }}
        if: always()
      - name: Print out logs for immediate inspection
        # and markup the output with GitHub Actions logging commands
        run: |
          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
        if: always()

  local-ubuntu:

    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        tox_system_factor: [conda-forge-ubuntu]
        tox_packages_factor: [minimal, standard, environment, environment-optional]
    env:
      TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
    steps:
      - uses: actions/checkout@v2
      - name: Install test prerequisites
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
      - name: Build and test with tox
        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
        # For doctesting, we use a lower parallelization to avoid timeouts.
        run: |
          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
      - name: Prepare logs artifact
        run: |
          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
        if: always()
      - uses: actions/upload-artifact@v1
        with:
          path: artifacts
          name: ${{ env.LOGS_ARTIFACT_NAME }}
        if: always()
      - name: Print out logs for immediate inspection
        # and markup the output with GitHub Actions logging commands
        run: |
          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
        if: always()

  dist:

    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 500
      - name: fetch tags
        run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
      - name: Install bootstrap prerequisites
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian _bootstrap)
      - name: Bootstrap with sage-update-version
        # We set SAGE_ROOT and SAGE_SRC by hand
        # because 'sage -sh' does not work with an unconfigured tree,
        # giving: Error: SAGE_SCRIPTS_DIR is set to a bad value
        run: |
          git config --global user.email "nobody@example.com"
          git config --global user.name "Sage GitHub CI"
          SAGE_ROOT=. SAGE_SRC=./src src/bin/sage-update-version $(git describe) || echo "(ignoring error)"
      - name: make dist
        run: |
          ./configure && make dist
      - uses: actions/upload-artifact@v2
        with:
          path: "dist/*.tar.gz"
          name: dist

  local-macos-nohomebrew:

    needs: [dist]

    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      max-parallel: 4
      matrix:
        os: [ macos-10.15, macos-11.0 ]
        tox_system_factor: [macos-nobootstrap, macos-nobootstrap-python3_pythonorg]
        tox_packages_factor: [minimal]
        # As of 2021-03, default is 12.4
        # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md#xcode
        xcode_version_factor: [default]
        include:
          # Test xcode 11.7 only on macos-10.15
          - tox_system_factor: macos-nobootstrap
            tox_packages_factor: minimal
            xcode_version_factor: 11.7
            os: macos-10.15
          - tox_system_factor: macos-nobootstrap-python3_pythonorg
            tox_packages_factor: minimal
            xcode_version_factor: 11.7
            os: macos-10.15
    env:
      TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-xcode_${{ matrix.xcode_version_factor }}
    steps:
      - uses: actions/checkout@v2
        if: "!contains(matrix.tox_system_factor, 'nobootstrap')"
      - uses: actions/download-artifact@v2
        with:
          path: .
          name: dist
        if: contains(matrix.tox_system_factor, 'nobootstrap')
      - name: Unpack sage dist
        run: |
          tar xf sage*.tar.gz --strip-components=1
        if: contains(matrix.tox_system_factor, 'nobootstrap')
      - name: Move homebrew away
        run: |
          (cd /usr/local && for a in bin etc include lib opt sbin share; do sudo mv $a $a-moved; done)
      - name: Select Xcode version
        run: |
          if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
      - name: Install test prerequisites
        run: |
          sudo /usr/bin/python3 -m pip install tox
      - name: Install python3 from python.org
        # As of 2020-03-30 (https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md),
        # Python 3.7.7 is installed on GitHub Actions runners.  But we install our own copy from the python.org binary package.
        run: |
          curl -o python3.pkg https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg
          sudo installer -verbose -pkg python3.pkg -target /
        if: contains(matrix.tox_system_factor, 'python3_pythonorg')
      - name: Build and test with tox
        # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
        # For doctesting, we use a lower parallelization to avoid timeouts.
        run: |
          MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
      - name: Prepare logs artifact
        run: |
          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
        if: always()
      - uses: actions/upload-artifact@v1
        with:
          path: artifacts
          name: ${{ env.LOGS_ARTIFACT_NAME }}
        if: always()
      - name: Print out logs for immediate inspection
        # and markup the output with GitHub Actions logging commands
        run: |
          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
        if: always()