Merge branch 'develop' into t/28143/minimal_kernel_bases

.ci/build-docker.sh

@@ -26,7 +26,7 @@ docker_build() {
     # Docker's --cache-from does not really work with multi-stage builds: https://github.com/moby/moby/issues/34715
     # So we just have to rely on the local cache.
     time docker build -f docker/Dockerfile \
---build-arg "MAKEOPTS=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg "MAKEOPTS_DOCBUILD=${MAKEOPTS}" --build-arg "SAGE_NUM_THREADS_DOCBUILD=${SAGE_NUM_THREADS_DOCBUILD}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@
+--build-arg "MAKEFLAGS=${MAKEFLAGS}" --build-arg "SAGE_NUM_THREADS=${SAGE_NUM_THREADS}" --build-arg "MAKEFLAGS_DOCBUILD=${MAKEFLAGS}" --build-arg "SAGE_NUM_THREADS_DOCBUILD=${SAGE_NUM_THREADS_DOCBUILD}" --build-arg ARTIFACT_BASE=$ARTIFACT_BASE $@
 }
 
 # We use a multi-stage build /docker/Dockerfile. For the caching to be
@@ -39,12 +39,21 @@ docker_build --target run-time-dependencies --tag run-time-dependencies:$DOCKER_
 docker_build --target build-time-dependencies --tag build-time-dependencies:$DOCKER_TAG .
 docker_build --target make-all --tag make-all:$DOCKER_TAG .
 
+# Copy docs out of the docker image to save them into browseable GitLab artifacts
+container=$(docker create make-all:$DOCKER_TAG)
+docker cp $container:/home/sage/sage/local/share/doc/sage/html html_
+# GitLab's browser does not like symlinks, so we flatten them
+rsync html_/ html/ -a --copy-links
+
 # Build the release image without build artifacts.
 docker_build --target sagemath --tag "$DOCKER_IMAGE_CLI" .
+# Tag the sagemath:$DOCKER_TAG image that CI has just built as
+# sagemath:$COMMIT_HASH so we can refer to it uniquely later.
+docker tag "$DOCKER_IMAGE_CLI" "$DOCKER_IMAGE_BINDER"
 # Display the layers of this image
 docker history "$DOCKER_IMAGE_CLI"
 # Build the developer image with the build artifacts intact.
-# Note: It's important to build the dev image last because it might be tagged as ARTIFACT_BASE.
+# Note: It is important to build the dev image last because it might be tagged as ARTIFACT_BASE.
 docker_build --target sagemath-dev --tag "$DOCKER_IMAGE_DEV" .
 # Display the layers of this image
 docker history "$DOCKER_IMAGE_DEV"

.ci/head-tail.sh

@@ -33,6 +33,9 @@ stdbuf -i0 -o0 -e0 awk -v limit=$1 -v firstMissingNR=-1 -v offset=$OFFSET -v byt
     a[NR] = $0;
     delete a[NR-offset];
     printf "." > "/dev/stderr"
+    if (/^sage-logger/){
+      print > "/dev/stderr"
+    }
   }
 }
 END {

.ci/push-dockerhub.sh

@@ -26,4 +26,8 @@ if [ -z "$DOCKER_USER" -o -z "$SECRET_DOCKER_PASS" ]; then
 else
   cat "$SECRET_DOCKER_PASS" | docker login -u $DOCKER_USER --password-stdin
   docker push ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG
+
+  # For historical reasons, we also provide a -py3 tag. It's identical to the non-py3 tag.
+  docker tag ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG-py3
+  docker push ${DOCKER_NAMESPACE:-sagemath}/$1:$DOCKER_TAG-py3
 fi

.ci/setup-make-parallelity.sh

@@ -3,10 +3,10 @@
 # Source this to set CPUTHREADS (the number of apparent cores) and RAMTHREADS
 # (free RAM divided by the maximum amount needed per thread typically)
 # From this this script infers reasonable defaults for SAGE_NUM_THREADS and
-# MAKEOPTS.
+# MAKEFLAGS.
 
 # We do exactly the same for CPUTHREADS_DOCBUILD, RAMTHREADS_DOCBUILD,
-# SAGE_NUM_THREADS_DOCBUILD, MAKEOPTS_DOCBUILD. As the docbuild needs
+# SAGE_NUM_THREADS_DOCBUILD, MAKEFLAGS_DOCBUILD. As the docbuild needs
 # substantially more RAM as of May 2018.
 
 # ****************************************************************************
@@ -61,5 +61,5 @@ else
     export SAGE_NUM_THREADS_DOCBUILD=$RAMTHREADS_DOCBUILD
 fi
 # Set -j and -l for make (though -l is probably ignored by Sage)
-export MAKEOPTS="-j $SAGE_NUM_THREADS -l $((CPUTHREADS - 1)).8"
-export MAKEOPTS_DOCBUILD="-j $SAGE_NUM_THREADS_DOCBUILD -l $((CPUTHREADS_DOCBUILD - 1)).8"
+export MAKEFLAGS="-j $SAGE_NUM_THREADS -l $((CPUTHREADS - 1)).8"
+export MAKEFLAGS_DOCBUILD="-j $SAGE_NUM_THREADS_DOCBUILD -l $((CPUTHREADS_DOCBUILD - 1)).8"

.ci/test-dev.sh

@@ -26,7 +26,7 @@ IMAGE="$1"
 # Runs $IMAGE with args and check that it terminates with a zero exit code in at most limit seconds.
 timed_run() {
     START=`date +%s`
-    docker run -e MAKEOPTS="$MAKEOPTS_DOCBUILD" -e SAGE_NUM_THREADS="$SAGE_NUM_THREADS_DOCBUILD" "$IMAGE" "$2"
+    docker run -e MAKEFLAGS="$MAKEFLAGS_DOCBUILD" -e SAGE_NUM_THREADS="$SAGE_NUM_THREADS_DOCBUILD" "$IMAGE" "$2"
     END=`date +%s`
     TOTAL=$((END-START))
     echo "Checking whether running \"$2\" was fast…"
@@ -42,4 +42,5 @@ timed_run 120 true # runs make build
 # The parser in Sphinx fails to parse some .py files and adds the (more
 # recently modified) .pyc files as dependencies instead. (Have a look the
 # changeset that introduced this comment for more details.)
-timed_run $(( 1200/$SAGE_NUM_THREADS_DOCBUILD )) make # runs make build and then make
+# Parts of the docbuild do not seem to be run in parallel at the moment.
+timed_run $(( 180 + 1200/$SAGE_NUM_THREADS_DOCBUILD )) make # runs make build and then make

.ci/update-env.sh

@@ -39,6 +39,8 @@ export DOCKER_TAG=`echo $DOCKER_TAG | tr -d '[:space:]' | tr -c '[:alnum:]_.-' '
 export DOCKER_IMAGE_CLI=${DOCKER_NAMESPACE:-sagemath}/sagemath:$DOCKER_TAG
 export DOCKER_IMAGE_DEV=${DOCKER_NAMESPACE:-sagemath}/sagemath-dev:$DOCKER_TAG
 
+export DOCKER_IMAGE_BINDER="${DOCKER_NAMESPACE:-sagemath}/sagemath:${CI_COMMIT_SHA}"
+
 # Seed the build cache with this image (set to source-clean to build from
 # scratch.)
 export ARTIFACT_BASE=${ARTIFACT_BASE:-$DEFAULT_ARTIFACT_BASE}

.circleci/before-script.sh

@@ -18,7 +18,7 @@
 . .ci/protect-secrets.sh
 # Collect debug infos about the system we are running on
 .ci/describe-system.sh
-# Set MAKEOPTS and SAGE_NUM_THREADS 
+# Set MAKEFLAGS and SAGE_NUM_THREADS
 . .ci/setup-make-parallelity.sh
 
 # Set DOCKER_TAG according to the current branch/tag

.gitattributes

@@ -0,0 +1,6 @@
+# Force LF normalization
+* text=auto eol=lf
+# except for Windows batch files
+*.{cmd,[cC][mM][dD]} text eol=crlf
+*.{bat,[bB][aA][tT]} text eol=crlf
+*.diff_bin binary