Merge tag '9.3.beta2' into t/30947/src_doc_bootstrap__simplify_by_usi…

…ng_new_options_of__sage__package_list_

SageMath version 9.3.beta2, Release Date: 2020-11-24

.github/workflows/lint.yml

@@ -0,0 +1,47 @@
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  lint-pycodestyle:
+    name: Code style check with pycodestyle
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install pycodestyle
+      run: pip install tox pycodestyle
+    - name: Lint using pycodestyle
+      run: tox -e pycodestyle
+  lint-relint:
+    name: Code style check with relint
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install pycodestyle
+      run: pip install tox relint
+    - name: Lint using relint
+      run: tox -e relint src/
+  lint-pyright:
+    name: Static type check with pyright
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Set up node to install pyright
+      uses: actions/setup-node@v1
+      with:
+        node-version: '12'
+    - name: Install pyright
+      run: npm install -g pyright
+    - name: Lint using pyright
+      run: pyright

.github/workflows/tox-experimental.yml

@@ -69,7 +69,7 @@ jobs:
               TOKEN="${{ secrets.GITHUB_TOKEN }}"
             fi
             if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then
-              echo "::set-env name=DOCKER_PUSH_REPOSITORY::docker.pkg.github.com/${{ github.repository }}/"
+              echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
             fi
       - run: |
           set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"

.github/workflows/tox-gcc_spkg.yml

@@ -67,7 +67,7 @@ jobs:
               TOKEN="${{ secrets.GITHUB_TOKEN }}"
             fi
             if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then
-              echo "::set-env name=DOCKER_PUSH_REPOSITORY::docker.pkg.github.com/${{ github.repository }}/"
+              echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
             fi
       - run: |
           set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"

.github/workflows/tox-optional.yml

@@ -71,7 +71,7 @@ jobs:
               TOKEN="${{ secrets.GITHUB_TOKEN }}"
             fi
             if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then
-              echo "::set-env name=DOCKER_PUSH_REPOSITORY::docker.pkg.github.com/${{ github.repository }}/"
+              echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
             fi
       - run: |
           set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"

.github/workflows/tox.yml

@@ -67,7 +67,7 @@ jobs:
               TOKEN="${{ secrets.GITHUB_TOKEN }}"
             fi
             if echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin; then
-              echo "::set-env name=DOCKER_PUSH_REPOSITORY::docker.pkg.github.com/${{ github.repository }}/"
+              echo "DOCKER_PUSH_REPOSITORY=docker.pkg.github.com/${{ github.repository }}/" >> $GITHUB_ENV
             fi
       - run: |
           set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"

.zenodo.json

@@ -1,10 +1,10 @@
 {
     "description": "Mirror of the Sage https://sagemath.org/ source tree", 
     "license": "other-open", 
-    "title": "sagemath/sage: 9.3.beta1", 
-    "version": "9.3.beta1", 
+    "title": "sagemath/sage: 9.3.beta2", 
+    "version": "9.3.beta2", 
     "upload_type": "software", 
-    "publication_date": "2020-11-07", 
+    "publication_date": "2020-11-24", 
     "creators": [
         {
             "affiliation": "SageMath.org", 
@@ -15,7 +15,7 @@
     "related_identifiers": [
         {
             "scheme": "url", 
-            "identifier": "https://github.com/sagemath/sage/tree/9.3.beta1", 
+            "identifier": "https://github.com/sagemath/sage/tree/9.3.beta2", 
             "relation": "isSupplementTo"
         }, 
         {

VERSION.txt

@@ -1 +1 @@
-SageMath version 9.3.beta1, Release Date: 2020-11-07
+SageMath version 9.3.beta2, Release Date: 2020-11-24

build/bin/sage-system-python → build/bin/sage-bootstrap-python

@@ -2,11 +2,10 @@
 
 # Run the system python.
 #
-# This is primarily for use by the build toolchain so that it can continue
-# using the system Python rather than Sage's Python, preventing conflicts
-# that might otherwise occur, particularly in parallel builds.
+# This is only for use by the build toolchain.
 #
-# See https://trac.sagemath.org/ticket/18438
+# (It has nothing to do with the python3 that configure --with-system-python3
+# is looking for in build/pkgs/python3/spkg-configure.m4.)
 
 if [ -z "$SAGE_ORIG_PATH" ]; then
     # If not we're running from within sage-env just set the existing path

build/bin/sage-build-num-threads

@@ -0,0 +1,187 @@
+#!/usr/bin/env sage-system-python
+#
+# Determine the number of threads to be used by Sage.
+#
+# Outputs three space-separated numbers:
+# 1) The number of threads to use for Sage, based on MAKE, MAKEFLAGS
+#    and SAGE_NUM_THREADS
+# 2) The number of threads to use when parallel execution is explicitly
+#    asked for (e.g. sage -tp)
+# 3) The number of CPU cores in the system, as determined by
+#    multiprocessing.cpu_count()
+#
+# AUTHOR: Jeroen Demeyer (2011-12-08): Trac ticket #12016
+#
+
+from __future__ import print_function
+
+import os
+import multiprocessing
+import re
+import math
+
+
+def number_of_cores():
+    """
+    Try to determine the number of CPU cores in this system.
+    If successful return that number. Otherwise return 1.
+    """
+    # If the environment variable SAGE_NUM_CORES exists, use that value.
+    # This is useful for testing purposes.
+    try:
+        n = int(os.environ["SAGE_NUM_CORES"])
+        if n > 0:
+            return n
+    except (ValueError, KeyError):
+        pass
+
+    try:
+        n = multiprocessing.cpu_count()
+        if n > 0:
+            return n
+    except NotImplementedError:
+        pass
+
+    try:  # Solaris fix
+        from subprocess import Popen, PIPE
+        p = Popen(['sysctl', '-n', 'hw.ncpu'],
+                  stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
+        n = int(p.stdout.read().strip())
+        if n > 0:
+            return n
+    except (ValueError, OSError):
+        pass
+
+    return 1
+
+
+def parse_jobs_from_MAKE(MAKE, unlimited=999999):
+    """
+    Parse an environment variable like :envvar:`MAKE` for the number of
+    jobs specified. This looks at arguments ``-j``, ``--jobs``, ``-l``,
+    ``--load-average``.
+
+    INPUT:
+
+    - ``MAKE`` -- The value of :envvar:`MAKE` or :envvar:`MAKEFLAGS`.
+
+    - ``unlimited`` -- The value to return when ``MAKE`` contains ``-j``
+      without argument and no ``-l`` option.  Normally this is interpreted
+      as "unlimited".
+
+    OUTPUT:
+
+    The number of jobs specified by that variable.  Raise ``KeyError``
+    if no number of jobs is specified in ``MAKE``.
+    """
+    # First, find value of -j
+    # Since this is doing a greedy match on the left and non-greedy on the right,
+    # we find the last -j or --jobs
+    (j, num) = re.subn(r'^(.* )?(-j *|--jobs(=(?=[0-9])| +))([0-9]*)( .*?)?$', r'\4', MAKE, count=1)
+    if num < 1:
+        # No replacement done, i.e. no -j option found
+        raise KeyError("No number of jobs specified")
+    elif j == "":
+        # j is empty: unlimited number of jobs! :-)
+        j = unlimited
+    else:
+        j = int(j)
+        if j <= 0:
+            raise ValueError("Non-positive value specified for -j")
+
+    # Next, find the value of -l
+    # If it is specified, use this as an upper bound on j
+    (l, num) = re.subn(r'^(.* )?(-l *|--(load-average|max-load)(=(?=[0-9])| +))([0-9.]*)( .*?)?$', r'\5', MAKE, count=1)
+    if num < 1:
+        # No replacement done, i.e. no -l option found
+        pass
+    elif not l:
+        # No load limit specified
+        pass
+    else:
+        l = int(math.ceil(float(l)))
+        # A load limit will never prevent starting at least one job
+        if l <= 1:
+            l = 1
+        j = min(j, l)
+
+    return j
+
+
+def num_threads():
+    """
+    Determine the number of threads from the environment variables
+    (in decreasing priority) :envvar:`SAGE_NUM_THREADS`, :envvar:`MAKE`,
+    :envvar:`MAKEFLAGS` and :envvar:`MFLAGS`.
+
+    If :envvar:`SAGE_NUM_THREADS` is 0 and neither :envvar:`MAKE` nor
+    :envvar:`MAKEFLAGS` specifies a number of jobs, the use a default
+    of ``min(8, number_of_cores)``.
+
+    OUTPUT:
+
+    a tuple (num_threads, num_threads_parallel, num_cores)
+    """
+    num_cores = number_of_cores()
+
+    num_threads = None
+    # Handle MFLAGS only for backwards compatibility
+    try:
+        num_threads = parse_jobs_from_MAKE(os.environ["MFLAGS"], unlimited=2)
+    except (ValueError, KeyError):
+        pass
+
+    try:
+        # Prepend hyphen to MAKEFLAGS if it does not start with one
+        MAKEFLAGS = os.environ["MAKEFLAGS"]
+        if MAKEFLAGS[0] != '-':
+            MAKEFLAGS = '-' + MAKEFLAGS
+        # In MAKEFLAGS, "-j" does not mean unlimited.  It probably
+        # means an inherited number of jobs, let us use 2 for safety.
+        num_threads = parse_jobs_from_MAKE(MAKEFLAGS, unlimited=2)
+    except (ValueError, KeyError, IndexError):
+        pass
+
+    try:
+        num_threads = parse_jobs_from_MAKE(os.environ["MAKE"])
+    except (ValueError, KeyError):
+        pass
+
+    # Number of threads to use when parallel execution is explicitly
+    # asked for
+    num_threads_parallel = num_threads
+    if num_threads_parallel is None:
+        num_threads_parallel = max(min(8, num_cores), 2)
+
+    try:
+        sage_num_threads = int(os.environ["SAGE_NUM_THREADS"])
+        if sage_num_threads == 0:
+            # If SAGE_NUM_THREADS is 0, use the default only
+            # if none of the above variables specified anything.
+            if num_threads is None:
+                num_threads = min(8, num_cores)
+        elif sage_num_threads > 0:
+            # SAGE_NUM_THREADS overrides everything
+            num_threads = sage_num_threads
+            num_threads_parallel = sage_num_threads
+    except (ValueError, KeyError):
+        pass
+
+    # If we still don't know, use 1 thread
+    if num_threads is None:
+        num_threads = 1
+
+    # Finally, use SAGE_NUM_THREADS_PARALLEL if set.  A user isn't
+    # likely to set this, but it ensures that sage-env is idempotent
+    # if called more than once.
+    try:
+        sage_num_threads = int(os.environ["SAGE_NUM_THREADS_PARALLEL"])
+        if sage_num_threads > 0:
+            num_threads_parallel = sage_num_threads
+    except (ValueError, KeyError):
+        pass
+
+    return (num_threads, num_threads_parallel, num_cores)
+
+
+print(*num_threads())

build/bin/sage-download-file

@@ -1,4 +1,4 @@
-#!/usr/bin/env sage-system-python
+#!/usr/bin/env sage-bootstrap-python
 
 # USAGE:
 #

build/bin/sage-flock

@@ -1,4 +1,4 @@
-#!/usr/bin/env sage-system-python
+#!/usr/bin/env sage-bootstrap-python
 # vim: set filetype=python:
 
 # USAGE:

build/bin/sage-package

@@ -1,4 +1,4 @@
-#!/usr/bin/env sage-system-python
+#!/usr/bin/env sage-bootstrap-python
 
 # Script to manage third-party tarballs.
 #

build/bin/sage-print-system-package-command

@@ -21,6 +21,12 @@ do
         --prompt)
             PROMPT='  $ '
             ;;
+        --no-install-recommends)
+            NO_INSTALL_RECOMMENDS=yes
+            ;;
+        --yes)
+            YES=yes
+            ;;
         -*)
             echo >&2 "$0: unknown option $1"
             exit 1
@@ -37,6 +43,7 @@ if [ -z "$system" -o -z "$command" ]; then
     exit 1
 fi
 system_packages="$*"
+options=
 shopt -s extglob
 case $system:$command in
     homebrew*:setup-build-env)
@@ -56,22 +63,33 @@ case $system:$command in
         echo "${PROMPT}${SUDO}apt-get $command $system_packages"
         ;;
     @(debian*|ubuntu*):*)
-        [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}apt-get $command $system_packages"
+        [ "$NO_INSTALL_RECOMMENDS" = yes ] && options="$options --no-install-recommends"
+        [ "$YES" = yes ] && options="$options --yes"
+        [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}apt-get $command $options $system_packages"
         ;;
     @(fedora*|redhat*|centos*):install)
-        [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}yum install $system_packages"
+        [ "$YES" = yes ] && options="$options -y"
+        [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}yum install $options $system_packages"
         ;;
     gentoo*:install)
         [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}emerge $system_packages"
         ;;
+    arch*:update)
+        echo "${PROMPT}${SUDO}pacman -Sy"
+        ;;
     arch*:install)
-        [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}pacman -S $system_packages"
+        [ "$YES" = yes ] && options="$options --noconfirm"
+        [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}pacman -S $options $system_packages"
+        ;;
+    void*:update)
+        echo "${PROMPT}${SUDO}xbps-install -Su"
         ;;
-
     void*:install)
-        [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}xbps-install $system_packages"
+        [ "$YES" = yes ] && options="$options --yes"
+        [ -n "$system_packages" ] && echo "${PROMPT}${SUDO}xbps-install $options $system_packages"
         ;;
     *conda*:install)
+        [ "$YES" = yes ] && options="$options --yes"
         [ -n "$system_packages" ] && echo "${PROMPT}conda install $system_packages"
         ;;
     homebrew*:install)

build/bin/sage-spkg-uninstall

@@ -1,4 +1,4 @@
-#!/usr/bin/env sage-system-python
+#!/usr/bin/env sage-bootstrap-python
 
 # usage: sage-spkg-uninstall [-h] PKG [SAGE_LOCAL]
 #