diff --git a/.ci/describe-system.sh b/.ci/describe-system.sh deleted file mode 100755 index 6bd3b0efd4a..00000000000 --- a/.ci/describe-system.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -# **************************************************************************** -# Copyright (C) 2018 Julian Rüth -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# (at your option) any later version. -# http://www.gnu.org/licenses/ -# **************************************************************************** - -set +e -x - -docker info -docker run docker sh -c " - set -x - uname -a - df -h - cat /proc/cpuinfo - cat /proc/meminfo - cat /proc/sys/vm/overcommit_memory - cat /proc/sys/vm/overcommit_ratio" diff --git a/.ci/protect-secrets.sh b/.ci/protect-secrets.sh deleted file mode 100755 index 527604106ca..00000000000 --- a/.ci/protect-secrets.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh - -# This script protects all environment variables that start with "SECRET_". -# It puts them in a temporary file. The name of the variable contains the path -# of that file. This filename can then safely be used in `cat` even if `set -# -x` has been turned on. Also you can run "export" to understand the -# environment without danger. -# Be careful, however, not to use this like the following: -# docker login $DOCKER_USER $(cat $SECRET_DOCKER_PASS) -# as this would expose the password if `set -x` has been turned on. - -# **************************************************************************** -# Copyright (C) 2018 Julian Rüth -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 2 of the License, or -# (at your option) any later version. -# http://www.gnu.org/licenses/ -# **************************************************************************** - -set -eo pipefail -set +x - -function encrypt { - RET=`mktemp` - eval " echo \$$1" > "$RET" - echo $RET -} - -for name in `awk 'END { for (name in ENVIRON) { print name; } }' < /dev/null`; do -case "$name" in - SECRET_*) - export $name="$(encrypt $name)" - echo "Protected $name" - ;; -esac -done - -unset encrypt diff --git a/.github/workflows/docker_hub.yml b/.github/workflows/docker_hub.yml new file mode 100644 index 00000000000..694cd51762f --- /dev/null +++ b/.github/workflows/docker_hub.yml @@ -0,0 +1,91 @@ +name: Reusable workflow for Docker Hub images + +on: + workflow_call: + inputs: + dockerhub_repository: + default: sagemath-dev + type: string + dockerfile_target: + default: make-build + type: string + +jobs: + build-and-push: + name: Build Docker image and push to DockerHub + runs-on: ubuntu-latest + steps: + - name: Maximize build disk space + uses: easimon/maximize-build-space@v8 + with: + # need space in /var for Docker images + root-reserve-mb: 40000 + remove-dotnet: true + remove-android: true + remove-haskell: true + remove-codeql: true + remove-docker-images: true + + - name: Checkout + uses: actions/checkout@v4 + + - name: Set tag + # docker/metadata-action@v4 is not used since we need to distinguish + # between latest and develop tags + id: set_tag + run: | + git fetch --depth=1 origin +refs/tags/*:refs/tags/* + TAG_NAME=$(git tag --sort=creatordate | tail -1) + REPO=${{ inputs.dockerhub_repository }} + # see if the tag has already been pushed + # if yes then skip following steps + URL="https://registry.hub.docker.com/v2/repositories/sagemath/$REPO/tags?page_size=32" + LATEST_TAGS=$(curl -L -s $URL | jq '."results"[]["name"]') + JOB_DONE=false + for i in $LATEST_TAGS; do if [[ $i == \"$TAG_NAME\" ]]; then JOB_DONE=true; break; fi done + echo "JOB_DONE=$JOB_DONE" >> $GITHUB_ENV + + if [[ $JOB_DONE == 'false' ]] + then + TAG="sagemath/$REPO:$TAG_NAME" + TAG_LIST="$TAG, sagemath/$REPO:develop" + BASE="sagemath/sagemath-dev:$TAG_NAME" + echo "TAG_NAME=$TAG_NAME" >> $GITHUB_ENV + echo "TAG=$TAG" >> $GITHUB_ENV + echo "TAG_LIST=$TAG_LIST" >> $GITHUB_ENV + echo "BASE=$BASE" >> $GITHUB_ENV + fi + df -h + + - name: Update Tag List + id: upd_tag_list + run: | + REPO=${{ inputs.dockerhub_repository }} + TAG_LIST="${{ env.TAG_LIST }}, sagemath/$REPO:latest" + echo "TAG_LIST=$TAG_LIST" >> $GITHUB_ENV + if: "env.JOB_DONE == 'false' && !contains(env.TAG_NAME, 'beta') && !contains(env.TAG_NAME, 'rc')" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + if: env.JOB_DONE == 'false' + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + if: env.JOB_DONE == 'false' + + - name: Build and push make-build + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + target: ${{ inputs.dockerfile_target }} + build-args: | + MAKE_BUILD=${{ env.BASE }} + push: true + tags: ${{ env.TAG_LIST }} + cache-from: type=gha + cache-to: type=gha,mode=max + if: env.JOB_DONE == 'false' diff --git a/.github/workflows/push_to_docker_hub.yml b/.github/workflows/push_to_docker_hub.yml index 77a9f819d70..cd289f6a88b 100644 --- a/.github/workflows/push_to_docker_hub.yml +++ b/.github/workflows/push_to_docker_hub.yml @@ -7,106 +7,32 @@ on: - 'develop' push: tags: - # Just create image on pushing a tag - - '*' + # Match all release tags including beta, rc + - '[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.beta[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+.beta[0-9]+' + - '[0-9]+.[0-9]+.rc[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+.rc[0-9]+' + + schedule: + # Recover failed runs each Tuesday and Thursday at one o'clock + - cron: '0 1 * * 2,4' jobs: sagemath-dev: - name: Build Docker image on target make-build and push to DockerHub sagemath-dev - # target make-build replaces former sagemath-dev, see https://github.com/sagemath/sage/pull/36047 - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set tag - # docker/metadata-action@v4 is not used since we need to distinguish - # between latest and develop tags - id: set_tag - run: | - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - TAG_NAME=$(git tag --sort=creatordate | tail -1) - TAG="sagemath/sagemath-dev:$TAG_NAME" - TAG_LIST="$TAG, sagemath/sagemath-dev:develop" - echo "TAG_NAME=$TAG_NAME" >> $GITHUB_ENV - echo "TAG=$TAG" >> $GITHUB_ENV - echo "TAG_LIST=$TAG_LIST" >> $GITHUB_ENV - - - name: Update Tag List - id: upd_tag_list - run: | - TAG_LIST="${{ env.TAG_LIST }}, sagemath/sagemath-dev:latest" - echo "TAG_LIST=$TAG_LIST" >> $GITHUB_ENV - if: "!contains(env.TAG_NAME, 'beta') && !contains(env.TAG_NAME, 'rc')" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push make-build - uses: docker/build-push-action@v5 - with: - context: . - file: docker/Dockerfile - target: make-build # see the corresponding header-note - push: true - tags: ${{ env.TAG_LIST }} - cache-from: type=gha - cache-to: type=gha,mode=max + uses: ./.github/workflows/docker_hub.yml + with: + # Build from scratch + dockerhub_repository: sagemath-dev + dockerfile_target: make-build + secrets: inherit sagemath: needs: sagemath-dev - name: Build Docker image on target sagemath and push to DockerHub sagemath - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set tag - # docker/metadata-action@v4 is not used since we need to distinguish - # between latest and develop tags - id: set_tag - run: | - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - TAG_NAME=$(git tag --sort=creatordate | tail -1) - TAG="sagemath/sagemath:$TAG_NAME" - TAG_LIST="$TAG, sagemath/sagemath:develop" - BASE="sagemath/sagemath-dev:$TAG_NAME" - echo "TAG_NAME=$TAG_NAME" >> $GITHUB_ENV - echo "TAG=$TAG" >> $GITHUB_ENV - echo "TAG_LIST=$TAG_LIST" >> $GITHUB_ENV - echo "BASE=$BASE" >> $GITHUB_ENV - - - name: Update Tag List - id: upd_tag_list - run: | - TAG_LIST="${{ env.TAG_LIST }}, sagemath/sagemath:latest" - echo "TAG_LIST=$TAG_LIST" >> $GITHUB_ENV - if: "!contains(env.TAG_NAME, 'beta') && !contains(env.TAG_NAME, 'rc')" - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push sagemath - uses: docker/build-push-action@v5 - with: - context: . - file: docker/Dockerfile - build-args: | - MAKE_BUILD=${{ env.BASE }} - target: sagemath - push: true - tags: ${{ env.TAG_LIST }} - cache-from: type=gha - cache-to: type=gha,mode=max + uses: ./.github/workflows/docker_hub.yml + with: + # Build from sagemath-dev + dockerhub_repository: sagemath + dockerfile_target: sagemath + secrets: inherit diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index e80a04b422b..00000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,195 +0,0 @@ -# This file configures automatic builds of Sage on [GitLab](https://gitlab.com). -# To make the build time not too excessive, we seed the build cache with -# sagemath/sagemath-dev:develop. When basic SPKGs changed, this does not help -# much and the full build might exceed the set time limit in GitLab. You can -# increase that limit in Settings → CI/CD. -# You can also provision your own private more powerful runner in the same -# place -# https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker-executor; -# or set up your favourite cloud service to provide an on-demand autoscale -# runner. More details below. - -# As of early 2018 a run on GitLab CI takes about 45 minutes. We could probably -# save 10 minutes by not building/pushing/testing dev images for branches other -# than master/develop. - -# Note that most of the time during CI is spent with pulling and pushing of -# docker images and copying files locally as part of the docker build. At the -# moment there is no reliable way of passing the docker images to the following -# stages without explicit pushing/pulling or similar: -# https://gitlab.com/gitlab-org/gitlab-runner/issues/1107 - -# The timings mentioned above are typical values. The shared runners provided -# on gitlab.com are sometimes much slower depending on the runner you are -# scheduled on. Sometimes it's slower for no apparent reason, probably just an -# overcommittment of virtual machines on hardware. - -# GitLab provides several flavours of shared runners (as of early 2018): -# * runners tagged as "do" (digitalocean.com) provide about 60GB of HDD, two -# cores, but only 2GB of RAM. The RAM is sometimes not sufficient to build -# the documentation. -# * runners tagged as "gce" (Google Compute Engine) provide about 22GB of HDD, -# a single core, 4GB of RAM. Since we are relying on OverlayFS, the disk -# space is not sufficient to build sage from scratch. -# The shared runners are terminated after three hours. Currently, this is often -# insufficient to build sage from scratch. - -# If you want to provide your own runners, make sure to tag them as follows: -# * "big" (60GB of disk space are available) to make build-from-clean pass. - -image: docker:stable - -stages: - - build - - test - - release - -variables: - DOCKER_TAG: $CI_COMMIT_REF_NAME - # Builds are very I/O intensive; make sure we have a fast file system. - DOCKER_DRIVER: overlay2 - DEFAULT_ARTIFACT_BASE: sagemath/sagemath-dev:develop - -before_script: - # GitLab has no mechanism yet to hide secret variables: https://gitlab.com/gitlab-org/gitlab-ce/issues/13784 - # So we roll our own which protects all variables that start with SECRET_ - - . .ci/protect-secrets.sh - # Collect debug infos about the system we are running on - - .ci/describe-system.sh - # Set DOCKER_TAG according to the current branch/tag - - . .ci/update-env.sh - # Set MAKEFLAGS and SAGE_NUM_THREADS according to the machine we are running on - - . .ci/setup-make-parallelity.sh - -# We use docker-in-docker to build our docker images, i.e., we run a -# docker:dind "service" container and link to it from the container running the -# actual scripts below. -# Our scripts automatically connect to this service (unless you override it by -# setting DOCKER_HOST.) For example, each RUN statement in the Dockerfile -# spawns a docker container inside the docker:dind container to perform the RUN -# command there. -# It can be faster to expose your outer docker daemon by mounting -# /var/run/docker.sock to /var/run/docker.sock and setting DOCKER_HOST in -# Settings -> CI/CD -> Secret variable to unix:///var/run/docker.sock. (The -# speedup is mostly due to sharing layers of intermediate images.) However, -# this is only possible if you provision your own runners. Shared gitlab -# runners, do not bind mount /var/run/docker.sock. Also, docker:dind provides -# better isolation. If you expect many builds to run simultaneously on a host, -# conflicting tags can cause issues with a mounted DOCKER_HOST. -services: -- docker:stable-dind - -# Build Sage and its documentation. -# The build starts from the build artifacts of DEFAULT_ARTIFACT_BASE which is -# usually much faster than building from a clean checkout of Sage. -build-from-latest: - stage: build - artifacts: - when: always - paths: - - gitlab-build-docker.log - - html - expire_in: 1 month - script: - - apk --update add coreutils rsync - # The output of the build can get larger than gitlab.com's limit; only - # print the first 1MB (and the last 80 lines.) GitLab's limit is 4MB, - # however, the list of all branches and tags that shows up in the initial - # checkout takes already 1.5 MB: - # https://gitlab.com/gitlab-org/gitlab-runner/issues/4142 - - .ci/build-docker.sh | tee gitlab-build-docker.log | .ci/head-tail.sh 1048576 - - .ci/push-gitlab.sh sagemath-dev - - .ci/push-gitlab.sh sagemath - - DOCKER_TAG=$CI_COMMIT_SHA .ci/push-gitlab.sh sagemath - except: - - master - - develop - - tags - - web - -# Build Sage and its documentation from a clean checkout of Sage. -# Note that this takes several hours. You probably want to run this on your own -# gitlab-runner and increase the standard GitLab time limit for CI runs. -# Some of the shared runners provided by GitLab for free do not have enough -# disk space for this to work. If a build fails with "no space left on device", -# you could just retry it and hope to be scheduled on a machine with more disk -# space, or provision your own runner. -build-from-clean: - extends: - - build-from-latest - artifacts: - when: always - paths: - - gitlab-build-docker.log - - html - expire_in: 99 years - variables: - ARTIFACT_BASE: "source-clean" - only: - - master - - develop - - tags - # Run build-from-clean for a pipeline that has been explicitly created - # through GitLab's web interface. - - web - except: [] - tags: - # 60 GB of HDD are available - - big - # This build takes several CPU hours. It is very unlikely that there are any - # actual build errors for a tagged release but the (discounted) cloud - # machines this is running on might be preempted during the long build time. - # So let's try three times before we give up. - retry: 2 - -test-dev: - stage: test - dependencies: [] - script: - - . .ci/pull-gitlab.sh sagemath-dev - - sh .ci/test-dev.sh "$DOCKER_IMAGE" - -test-cli: - stage: test - dependencies: [] - script: - - . .ci/pull-gitlab.sh sagemath - - sh .ci/test-cli.sh "$DOCKER_IMAGE" - -test-jupyter: - stage: test - dependencies: [] - script: - - . .ci/pull-gitlab.sh sagemath - - sh .ci/test-jupyter.sh "$DOCKER_IMAGE" docker - -# Pushes the built images to Docker Hub if the Settings -> CI/CD -> Secret -# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. -push-dockerhub: - stage: release - dependencies: [] - only: - refs: - - branches - - tags - variables: - - $SECRET_DOCKER_PASS - script: - - . .ci/pull-gitlab.sh sagemath - - sh .ci/push-dockerhub.sh sagemath - -# Pushes the built dev images to Docker Hub if the Settings -> CI/CD -> Secret -# variables DOCKER_USER and SECRET_DOCKER_PASS have been set up. -push-dockerhub-dev: - stage: release - dependencies: [] - only: - refs: - - master - - develop - - tags - variables: - - $SECRET_DOCKER_PASS - script: - - . .ci/pull-gitlab.sh sagemath-dev - - sh .ci/push-dockerhub.sh sagemath-dev