docker.yml

name: Reusable workflow for Docker-based portability CI

on:
  workflow_call:
    inputs:
      targets_pre:
        default: build/make/Makefile
        type: string
      targets:
        default: build/make/Makefile
        type: string
      targets_optional:
        default: build/make/Makefile
        type: string
      tox_system_factors:
        description: 'Stringified JSON object listing tox system factors'
        type: string
        # 'tox -e update_docker_platforms' updates below
        default: >-
          [
           "ubuntu-trusty-toolchain-gcc_9",
           "ubuntu-xenial-toolchain-gcc_9",
           "ubuntu-bionic-gcc_8",
           "ubuntu-focal",
           "ubuntu-jammy",
           "ubuntu-lunar",
           "ubuntu-mantic",
           "debian-buster-gcc_spkg",
           "debian-bullseye",
           "debian-bookworm",
           "debian-trixie",
           "debian-sid",
           "linuxmint-20.1",
           "linuxmint-20.2",
           "linuxmint-20.3",
           "linuxmint-21",
           "linuxmint-21.1",
           "linuxmint-21.2",
           "fedora-30",
           "fedora-31",
           "fedora-32",
           "fedora-33",
           "fedora-34",
           "fedora-35",
           "fedora-36",
           "fedora-37",
           "fedora-38",
           "fedora-39",
           "centos-7-devtoolset-gcc_11",
           "centos-stream-8-python3.9",
           "centos-stream-9-python3.9",
           "almalinux-8-python3.9",
           "almalinux-9-python3.11",
           "gentoo-python3.10",
           "gentoo-python3.11",
           "archlinux-latest",
           "opensuse-15.3-gcc_11-python3.9",
           "opensuse-15.4-gcc_11-python3.10",
           "opensuse-15.5-gcc_11-python3.11",
           "opensuse-tumbleweed-python3.10",
           "opensuse-tumbleweed",
           "conda-forge-python3.11",
           "ubuntu-bionic-gcc_8-i386",
           "debian-bullseye-i386",
          ]
        # 'tox -e update_docker_platforms' updates above
      tox_packages_factors:
        description: 'Stringified JSON object listing tox packages factors'
        type: string
        default: >-
          ["minimal",
           "standard",
           ]
      extra_sage_packages:
        description: 'Extra Sage packages to install as system packages'
        type: string
        default: ""
      max_parallel:
        type: number
        default: 30
      free_disk_space:
        default: false
        type: boolean
      timeout:
        description: 'Elapsed time (seconds) at which to kill the build'
        default: 20000
        type: number
      #
      # Publishing to GitHub Packages
      #
      docker_push_repository:
        required: false
        type: string
      #
      # Incremental builds
      #
      docker_targets:
        default: "with-system-packages configured with-targets-pre with-targets with-targets-optional"
        type: string
      incremental:
        default: false
        type: boolean
      from_docker_repository:
        required: false
        type: string
      from_docker_target:
        required: false
        type: string
      from_docker_tag:
        required: false
        default: "$BUILD_TAG"
        type: string
      #
      # For use in upstream CIs. sage_trac_* are now ignored and will be removed later.
      #
      upstream_artifact:
        required: false
        type: string
      sage_repo:
        required: false
        type: string
      sage_trac_git:
        required: false
        type: string
      sage_trac_ticket:
        required: false
        type: string
      sage_ref:
        required: false
        type: string

jobs:
  linux:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      max-parallel: ${{ inputs.max_parallel }}
      matrix:
        tox_system_factor:   ${{ fromJson(inputs.tox_system_factors) }}
        tox_packages_factor: ${{ fromJson(inputs.tox_packages_factors) }}
    env:
      TOX_ENV:            "docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}${{ inputs.incremental && '-incremental' || '' }}"
      LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
      DOCKER_TARGETS:     ${{ inputs.docker_targets }}
      TARGETS_PRE:        ${{ inputs.targets_pre }}
      TARGETS:            ${{ inputs.targets }}
      TARGETS_OPTIONAL:   ${{ inputs.targets_optional }}
      FROM_DOCKER_REPOSITORY: ${{ inputs.from_docker_repository }}
      FROM_DOCKER_TARGET: ${{ inputs.from_docker_target }}
      FROM_DOCKER_TAG:    ${{ inputs.from_docker_tag }}
      EXTRA_CONFIGURE_ARGS: --enable-fat-binary
      EXTRA_SAGE_PACKAGES:  ${{ inputs.extra_sage_packages }}
    steps:
      - name: Maximize build disk space
        uses: easimon/maximize-build-space@v10
        with:
          # need space in /var for Docker images
          root-reserve-mb:      30000
          remove-dotnet:        true
          remove-android:       true
          remove-haskell:       true
          remove-codeql:        true
          remove-docker-images: true
        continue-on-error: true
        if: inputs.free_disk_space
      - name: Check out SageMath
        uses: actions/checkout@v4
        with:
          repository: ${{ inputs.sage_repo }}
          ref: ${{ inputs.sage_ref }}
          fetch-depth: 10000
      - name: Download upstream artifact
        uses: actions/download-artifact@v3
        with:
          path: upstream
          name: ${{ inputs.upstream_artifact }}
        if: inputs.upstream_artifact
      - name: Install test prerequisites
        run: |
          sudo DEBIAN_FRONTEND=noninteractive apt-get update
          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
          sudo apt-get clean
          df -h
      - name: Update Sage packages from upstream artifact
        run: |
          (export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore && echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - build/bin/write-dockerfile.sh && git diff)
        if: inputs.upstream_artifact

      - name: Try to login to ghcr.io
        if: inputs.docker_push_repository != ''
        # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
        run: |
            TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
            if [ -z "$TOKEN" ]; then
              TOKEN="${{ secrets.GITHUB_TOKEN }}"
            fi
            if echo "$TOKEN" | docker login ghcr.io -u ${{ github.actor }} --password-stdin; then
              echo "DOCKER_PUSH_REPOSITORY=$(echo ${{ inputs.docker_push_repository }} | tr "[:upper:]" "[:lower:]")" >> $GITHUB_ENV
              echo "DOCKER_CONFIG_FILE=$HOME/.docker/config.json" >> $GITHUB_ENV
            fi

      - name: Determine Docker tags to use
        run: |
            # This line needs to be run before the step "Merge CI fixes from sagemath/sage".
            DOCKER_TAG="$(git describe --dirty --always)"
            echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
            # From the docker documentation via .ci/update-env.sh:
            # "A tag name must be valid ASCII and may
            # contain lowercase and uppercase letters, digits, underscores, periods and
            # dashes. A tag name may not start with a period or a dash and may contain a
            # maximum of 128 characters."
            EXTRA_DOCKER_TAGS=`echo $GITHUB_REF_NAME | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '-' | sed 's/^[-.]*//' | cut -c1-128`
            shopt -s extglob
            case "$GITHUB_REF_NAME" in
              +([0-9]).+([0-9])?(.+([0-9])) )
                EXTRA_DOCKER_TAGS="latest dev $EXTRA_DOCKER_TAGS";;
              +([0-9]).+([0-9])?(.+([0-9]))?(.)@(a|alpha|b|beta|rc)+([0-9]) )
                EXTRA_DOCKER_TAGS="dev $EXTRA_DOCKER_TAGS";;
            esac
            echo "EXTRA_DOCKER_TAGS=$EXTRA_DOCKER_TAGS" >> $GITHUB_ENV

      - name: Merge CI fixes from sagemath/sage
        # This step needs to happen after the commit sha is put in DOCKER_TAG
        # so that multi-stage builds can work correctly.
        run: |
          .ci/merge-fixes.sh
        env:
          GH_TOKEN: ${{ github.token }}
          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}

      - name: Show disk space
        run: |
          df -h
        if: inputs.free_disk_space
      - name: Configure and build Sage distribution within a Docker container
        # The first command below is a self-destruct sequence,
        # which preempts the GitHub Actions 6-hour job cancellation.
        #
        # Using "docker exec", we enter the temporary containers used by
        # "docker build" and kill the "make" processes of the Sage distribution.
        #
        # The arcane "find" command is a replacement for "pkill make",
        # which we use because pkill is not installed in the "minimal" package
        # configuration on many platforms.
        run: |
          (sleep ${{ inputs.timeout }}; for id in $(docker ps -q); do docker exec $id find /proc -maxdepth 2 -name cmdline -exec bash -c "grep -l [m][a][k][e] {} | cut -d/ -f3 | xargs --no-run-if-empty kill" \;; done) &
          set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg NUMPROC=4 --build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
      - name: Copy logs from the Docker image or build container
        run: |
          mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
          cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
          if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
          if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
        if: always()
      - uses: actions/upload-artifact@v3
        with:
          path: artifacts
          name: ${{ env.LOGS_ARTIFACT_NAME }}
        if: always()
      - name: Print out logs for immediate inspection
        # and markup the output with GitHub Actions logging commands
        run: |
          .github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
        if: always()
      - name: List docker images
        run: |
          if [ -f .tox/$TOX_ENV/Dockertags ]; then
             cat .tox/$TOX_ENV/Dockertags
          fi
        if: always()