Merge branch 'develop' of https://github.com/sagemath/sage into publi…

…c/documentation/add_traceback

.github/workflows/ci-wsl.yml

@@ -9,6 +9,7 @@ on:
     branches:
       - 'public/build/**wsl**'
   workflow_dispatch:
+    # Allow to run manually
 
 jobs:
   windows:

.github/workflows/extract-sage-local.sh

@@ -1,26 +1,27 @@
-#!/usr/bin/env bash
+#!/bin/dash
 # to be run from $SAGE_ROOT, with arguments sage-local-${{ env.PREVIOUS_STAGES }}.tar
 
+if [ -z "$SAGE_LOCAL" ]; then
+    SAGE_LOCAL=$(pwd)/local
+fi
+
 # Show all tar files
 ls -l $*
 
 # We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
 for a in $*; do
     echo Extracting $a
-    tar xf $a
+    (cd / && tar xf -) < $a
     rm -f $a
 done
 
-# Also get rid of the stages that were not extracted
-rm -f sage-local-*.tar
-
 # We set the installation records to the same mtime so that no rebuilds due to dependencies
 # among these packages are triggered.
-(cd local/var/lib/sage/installed/ && touch .dummy && touch --reference=.dummy *)
+(cd "$SAGE_LOCAL"/var/lib/sage/installed/ && touch .dummy && touch --reference=.dummy *)
 
 # Show what has been built already.
-ls -l local local/var/lib/sage/installed/
+ls -l "$SAGE_LOCAL" "$SAGE_LOCAL"/var/lib/sage/installed/
 df -h
 
 # Rebase!
-src/bin/sage-rebase.sh local
+exec src/bin/sage-rebase.sh --all "$SAGE_LOCAL"

.github/workflows/lint.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Install pycodestyle
       run: pip install tox pycodestyle
     - name: Lint using pycodestyle
-      run: tox -e pycodestyle
+      run: tox -e pycodestyle-minimal
   lint-relint:
     name: Code style check with relint
     runs-on: ubuntu-latest
@@ -27,21 +27,7 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
-    - name: Install pycodestyle
+    - name: Install relint
       run: pip install tox relint
     - name: Lint using relint
-      run: tox -e relint src/
-  lint-pyright:
-    name: Static type check with pyright
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Set up node to install pyright
-      uses: actions/setup-node@v1
-      with:
-        node-version: '12'
-    - name: Install pyright
-      run: npm install -g pyright
-    - name: Lint using pyright
-      run: pyright
+      run: tox -e relint src/sage/

.github/workflows/sdist.yml

@@ -0,0 +1,45 @@
+name: Prepare source distributions
+
+on:
+  push:
+    tags:
+      # Match all release tags including beta, rc
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.beta[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+.beta[0-9]+'
+      - '[0-9]+.[0-9]+.rc[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+.rc[0-9]+'
+
+  workflow_dispatch:
+    # Allow to run manually
+
+jobs:
+
+  release_dist:
+
+    # This job, in contrast to "dist" in tox.yml,
+    # does not use "configure --enable-download-from-upstream-url".
+    #
+    # In this way, we check that all necessary package tarballs
+    # have already been uploaded to the Sage server at the time
+    # of pushing a release tag.
+    #
+    # It also uses "bootstrap -D", thus checking that the "configure"
+    # tarball has been uploaded to the Sage server at the time
+    # of pushing a release tag.
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install bootstrap prerequisites
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian _bootstrap)
+      - name: make dist
+        run: |
+          ./bootstrap -D && ./configure && make dist
+      - uses: actions/upload-artifact@v2
+        with:
+          path: "dist/*.tar.gz"
+          name: release_dist

.github/workflows/tox-experimental.yml

@@ -23,6 +23,8 @@ on:
   push:
     tags:
       - '*'
+  workflow_dispatch:
+    # Allow to run manually
 
 env:
   TARGETS_PRE: build/make/Makefile
@@ -36,14 +38,19 @@ jobs:
       fail-fast: false
       max-parallel: 6
       matrix:
-        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, centos-7, centos-8, gentoo, archlinux-latest, slackware-14.2, conda-forge, ubuntu-bionic-i386, ubuntu-focal-i386, debian-buster-i386, centos-7-i386]
+        # This list is different from the one in tox.yml:
+        # Trac #31526 switches gcc 4.x-based distributions to using the gcc_spkg configuration factor
+        # Trac #32281 removes gcc 4.x-based distributions whose binutils are unusable
+        tox_system_factor: [ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, debian-jessie-gcc_spkg, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, centos-7-gcc_spkg, centos-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-gcc_spkg]
         tox_packages_factor: [maximal]
         targets_pattern: [0-g, h-o, p, q-z]
     env:
       TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
       DOCKER_TARGETS: configured with-targets with-targets-optional
-      TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :experimental: | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep '^[${{ matrix.targets_pattern }}]' ) )"
+      # Test all non-dummy experimental packages, but do not test huge packages
+      # and do not test packages that require external software
+      TARGETS_OPTIONAL: "$( echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file spkg-install.in && sage-package list :experimental: --has-file spkg-install && sage-package list :experimental: --has-file requirements.txt | grep -v ^_ | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )"
     steps:
       - uses: actions/checkout@v2
         with:
@@ -52,15 +59,21 @@ jobs:
         run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
       - name: free disk space
         run: |
+          df -h
           sudo swapoff -a
           sudo rm -f /swapfile
-          sudo apt clean
+          sudo apt-get clean
           docker rmi $(docker image ls -aq)
+          echo "Largest packages:"
+          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 50
+          sudo apt-get --fix-broken --yes remove $(dpkg-query -f '${Package}\n' -W | grep -E '^(ghc-|google-cloud-sdk|google-chrome|firefox|mysql-server|dotnet-sdk|hhvm|mono)') || echo "(error ignored)"
           df -h
       - name: Install test prerequisites
         run: |
           sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
+          sudo apt-get clean
+          df -h
       - name: Try to login to docker.pkg.github.com
         # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
         run: |
@@ -104,15 +117,41 @@ jobs:
       fail-fast: false
       max-parallel: 3
       matrix:
-        os: [ macos-10.15, macos-11.0 ]
-        tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, conda-forge-macos]
+        tox_system_factor: [homebrew-macos, conda-forge-macos, homebrew-macos-python3_xcode]
         tox_packages_factor: [maximal]
+        # As of 2021-03, default xcode is 12.4
+        # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md#xcode
+        xcode_version_factor: [default]
         targets_pattern: [0-g, h-o, p, q-z]
-        xcode_version_factor: [11.7, default, 12.3]
+        os: [ macos-10.15, macos-11.0 ]
+        include:
+          # Test xcode 11.7 only on macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: 0-g
+            os: macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: h-o
+            os: macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: p
+            os: macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: q-z
+            os: macos-10.15
     env:
       TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
-      TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :experimental: | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep '^[${{ matrix.targets_pattern }}]' ) )"
+      # Test all non-dummy experimental packages, but do not test huge packages
+      # and do not test packages that require external software
+      TARGETS_OPTIONAL: "$( echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file spkg-install.in && sage-package list :experimental: --has-file spkg-install && sage-package list :experimental: --has-file requirements.txt | grep -v ^_ | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )"
     steps:
       - uses: actions/checkout@v2
       - name: Select Xcode version

.github/workflows/tox-gcc_spkg.yml

@@ -23,9 +23,11 @@ on:
   push:
     tags:
       - '*'
+  workflow_dispatch:
+    # Allow to run manually
 
 env:
-  TARGETS_PRE: sagelib-build-deps
+  TARGETS_PRE: all-sage-local
   TARGETS: build doc-html
   TARGETS_OPTIONAL: ptest
 
@@ -50,15 +52,21 @@ jobs:
         run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
       - name: free disk space
         run: |
+          df -h
           sudo swapoff -a
           sudo rm -f /swapfile
-          sudo apt clean
+          sudo apt-get clean
           docker rmi $(docker image ls -aq)
+          echo "Largest packages:"
+          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 50
+          sudo apt-get --fix-broken --yes remove $(dpkg-query -f '${Package}\n' -W | grep -E '^(ghc-|google-cloud-sdk|google-chrome|firefox|mysql-server|dotnet-sdk|hhvm|mono)') || echo "(error ignored)"
           df -h
       - name: Install test prerequisites
         run: |
           sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
+          sudo apt-get clean
+          df -h
       - name: Try to login to docker.pkg.github.com
         # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
         run: |
@@ -112,7 +120,7 @@ jobs:
       - name: Install test prerequisites
         run: |
           sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
       - name: Build and test with tox
         # We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
         # For doctesting, we use a lower parallelization to avoid timeouts.

.github/workflows/tox-optional.yml

@@ -23,6 +23,8 @@ on:
   push:
     tags:
       - '*'
+  workflow_dispatch:
+    # Allow to run manually
 
 env:
   TARGETS_PRE: build/make/Makefile
@@ -36,16 +38,19 @@ jobs:
       fail-fast: false
       max-parallel: 6
       matrix:
-        tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, centos-7, centos-8, gentoo, archlinux-latest, slackware-14.2, conda-forge, ubuntu-bionic-i386, ubuntu-focal-i386, debian-buster-i386, centos-7-i386]
+        # This list is different from the one in tox.yml:
+        # Trac #31526 switches gcc 4.x-based distributions to using the gcc_spkg configuration factor
+        # Trac #32281 removes gcc 4.x-based distributions whose binutils are unusable
+        tox_system_factor: [ubuntu-xenial, ubuntu-bionic, ubuntu-focal, ubuntu-groovy, ubuntu-hirsute, debian-jessie-gcc_spkg, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-18, linuxmint-19, linuxmint-19.3, linuxmint-20.1, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, centos-7-gcc_spkg, centos-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15, opensuse-15.3, opensuse-tumbleweed, slackware-14.2, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-gcc_spkg]
         tox_packages_factor: [maximal]
         targets_pattern: [0-g, h-o, p, q-z]
     env:
       TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
       DOCKER_TARGETS: configured with-targets with-targets-optional
-      # Test all optional packages, but do not test huge packages,
+      # Test all non-dummy optional packages, but do not test huge packages
       # and do not test packages that require external software
-      TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :optional: | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )"
+      TARGETS_OPTIONAL: "$( echo $(export PATH=build/bin:$PATH && (sage-package list :optional: --has-file spkg-install.in && sage-package list :optional: --has-file spkg-install && sage-package list :optional: --has-file requirements.txt) | grep -v ^_ | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )"
     steps:
       - uses: actions/checkout@v2
         with:
@@ -54,15 +59,21 @@ jobs:
         run: git fetch --depth=1 origin +refs/tags/*:refs/tags/*
       - name: free disk space
         run: |
+          df -h
           sudo swapoff -a
           sudo rm -f /swapfile
-          sudo apt clean
+          sudo apt-get clean
           docker rmi $(docker image ls -aq)
+          echo "Largest packages:"
+          dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 50
+          sudo apt-get --fix-broken --yes remove $(dpkg-query -f '${Package}\n' -W | grep -E '^(ghc-|google-cloud-sdk|google-chrome|firefox|mysql-server|dotnet-sdk|hhvm|mono)') || echo "(error ignored)"
           df -h
       - name: Install test prerequisites
         run: |
           sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
+          sudo apt-get clean
+          df -h
       - name: Try to login to docker.pkg.github.com
         # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
         run: |
@@ -106,17 +117,41 @@ jobs:
       fail-fast: false
       max-parallel: 3
       matrix:
-        os: [ macos-10.15, macos-11.0 ]
-        tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, conda-forge-macos]
+        tox_system_factor: [homebrew-macos, conda-forge-macos, homebrew-macos-python3_xcode]
         tox_packages_factor: [maximal]
+        # As of 2021-03, default xcode is 12.4
+        # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md#xcode
+        xcode_version_factor: [default]
         targets_pattern: [0-g, h-o, p, q-z]
-        xcode_version_factor: [11.7, default, 12.3]
+        os: [ macos-10.15, macos-11.0 ]
+        include:
+          # Test xcode 11.7 only on macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: 0-g
+            os: macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: h-o
+            os: macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: p
+            os: macos-10.15
+          - tox_system_factor: homebrew-macos
+            tox_packages_factor: maximal
+            xcode_version_factor: 11.7
+            targets_pattern: q-z
+            os: macos-10.15
     env:
       TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
       LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
-      # Test all optional packages, but do not test huge packages
+      # Test all non-dummy optional packages, but do not test huge packages
       # and do not test packages that require external software
-      TARGETS_OPTIONAL: "$( echo $(PATH=build/bin:$PATH build/bin/sage-package list :optional: | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )"
+      TARGETS_OPTIONAL: "$( echo $(export PATH=build/bin:$PATH && (sage-package list :optional: --has-file spkg-install.in && sage-package list :optional: --has-file spkg-install && sage-package list :optional: --has-file requirements.txt) | grep -v ^_ | grep -v  database_stein_watkins\\$ | grep -v polytopes_db_4d | grep -v cplex | grep -v gurobi | grep '^[${{ matrix.targets_pattern }}]' ) )"
     steps:
       - uses: actions/checkout@v2
       - name: Select Xcode version