Merge #33851

.github/workflows/ci-conda.yml

@@ -43,15 +43,8 @@ jobs:
           bash ~/miniconda.sh -b -p $HOME/miniconda
           echo "CONDA=$HOME/miniconda" >> $GITHUB_ENV
 
-      - name: Install bootstrap prerequisites
-        run: |
-          export PATH="$(pwd)/build/bin:$PATH"
-          SYSTEM=$(sage-guess-package-system)
-          eval $(sage-print-system-package-command $SYSTEM --sudo install $(sage-get-system-packages $SYSTEM _bootstrap))
-
-      # Create conda environment file
-      - name: Bootstrap
-        run: ./bootstrap
+      - name: Create conda environment files
+        run: ./bootstrap-conda
 
       - name: Cache conda packages
         uses: actions/cache@v2
@@ -80,6 +73,7 @@ jobs:
         shell: bash -l {0}
         continue-on-error: true
         run: |
+          ./bootstrap
           echo "::add-matcher::.github/workflows/configure-systempackage-problem-matcher.json"
           ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt); do echo --with-system-$pkg=force; done)
           echo "::remove-matcher owner=configure-system-package-warning::"
@@ -88,8 +82,8 @@ jobs:
       - name: Build
         shell: bash -l {0}
         run: |
-          pip install --no-build-isolation -v -v -e pkgs/sage-conf pkgs/sage-setup 
-          pip install --no-build-isolation -v -v -e src
+          pip install --no-build-isolation -v -v -e ./pkgs/sage-conf ./pkgs/sage-setup
+          pip install --no-build-isolation -v -v -e ./src
         env:
           SAGE_NUM_THREADS: 2
 

.gitignore

@@ -23,6 +23,7 @@
 /environment.yml
 /environment-optional.yml
 /src/environment.yml
+/src/environment-dev.yml
 /src/environment-optional.yml
 
 /src/setup.cfg

.vscode/settings.json

@@ -20,6 +20,8 @@
     ],
     "python.testing.unittestEnabled": false,
     "cSpell.words": [
+        "Conda",
+        "Cython",
         "sagemath"
     ],
 }

Makefile

@@ -171,6 +171,7 @@ bootstrap-clean:
 	rm -f src/doc/en/reference/repl/*.txt
 	rm -f environment.yml
 	rm -f src/environment.yml
+	rm -f src/environment-dev.yml
 	rm -f environment-optional.yml
 	rm -f src/environment-optional.yml
 	rm -f src/Pipfile

bootstrap

@@ -158,6 +158,7 @@ SAGE_SPKG_FINALIZE([$pkgname], [$pkgtype], [$SPKG_SOURCE], [$SPKG_TREE_VAR])"
     # ONLY stderr, and to re-output the results back to stderr leaving
     # stdout alone. Basically we swap the two descriptors using a
     # third, filter, and then swap them back.
+    ./bootstrap-conda && \
     src/doc/bootstrap && \
     install_config_rpath && \
     aclocal -I m4 && \

bootstrap-conda

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+########################################################################
+# Generate auto-generated conda environment files
+#########################################################################
+
+STRIP_COMMENTS="sed s/#.*//;"
+
+shopt -s extglob
+
+DEVELOP_SPKG_PATTERN="@(_develop$(for a in $(head -n 1 build/pkgs/_develop/dependencies); do echo -n "|"$a; done))"
+
+BOOTSTRAP_PACKAGES=$(echo $(${STRIP_COMMENTS} build/pkgs/_bootstrap/distros/conda.txt))
+SYSTEM_PACKAGES=
+OPTIONAL_SYSTEM_PACKAGES=
+SAGELIB_SYSTEM_PACKAGES=
+SAGELIB_OPTIONAL_SYSTEM_PACKAGES=
+DEVELOP_SYSTEM_PACKAGES=
+for PKG_BASE in $(./sage --package list --has-file distros/conda.txt); do
+    PKG_SCRIPTS=build/pkgs/$PKG_BASE
+    SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/conda.txt
+    PKG_TYPE=$(cat $PKG_SCRIPTS/type)
+    PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE))
+    if [ -n "PKG_SYSTEM_PACKAGES" ]; then
+        if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
+            case "$PKG_BASE:$PKG_TYPE" in
+                *:standard)
+                    SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    ;;
+                $DEVELOP_SPKG_PATTERN:*)
+                    DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    ;;
+                *)
+                    OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    ;;
+            esac
+        else
+            case "$PKG_TYPE" in
+                standard)
+                    SAGELIB_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    ;;
+                *)
+                    SAGELIB_OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES"
+                    ;;
+            esac
+        fi
+    fi
+done
+echo >&2 $0:$LINENO: generate conda enviroment files
+(
+    echo "name: sage-build"
+    echo "channels:"
+    echo "  - conda-forge"
+    echo "  - nodefaults"
+    echo "dependencies:"
+    for pkg in $SYSTEM_PACKAGES; do
+        echo "  - $pkg"
+    done
+    echo "  # Packages needed for ./bootstrap"
+    for pkg in $BOOTSTRAP_PACKAGES; do
+        echo "  - $pkg"
+    done
+) > environment.yml
+
+(
+    sed 's/name: sage-build/name: sage/' environment.yml
+    echo "  # Additional packages providing all dependencies for the Sage library"
+    for pkg in $SAGELIB_SYSTEM_PACKAGES; do
+        echo "  - $pkg"
+    done
+) > src/environment.yml
+
+(
+    sed 's/name: sage/name: sage-dev/' src/environment.yml
+    echo "  # Additional dev tools"
+    for pkg in $DEVELOP_SYSTEM_PACKAGES; do
+        echo "  - $pkg"
+    done
+) > src/environment-dev.yml
+
+(
+    cat environment.yml
+    echo "  # optional packages"
+    for pkg in $OPTIONAL_SYSTEM_PACKAGES; do
+        echo "  - $pkg"
+    done
+) > environment-optional.yml
+
+(
+    cat src/environment.yml
+    echo "  # optional packages"
+    for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do
+        echo "  - $pkg"
+    done
+) > src/environment-optional.yml

build/pkgs/_develop/SPKG.rst

@@ -0,0 +1,7 @@
+\_develop: Represents system packages recommended for development
+=================================================================
+
+Description
+-----------
+
+Script package representing a list of system packages recommended for developers.

build/pkgs/_develop/dependencies

@@ -0,0 +1 @@
+git pytest pytest_xdist

build/pkgs/_develop/distros/conda.txt

@@ -0,0 +1,2 @@
+openssh
+pycodestyle

build/pkgs/_develop/distros/debian.txt

@@ -0,0 +1,3 @@
+# Needed for devcontainer support in VS code
+gpgconf
+openssh-client

build/pkgs/_develop/spkg-configure.m4

@@ -0,0 +1,3 @@
+SAGE_SPKG_CONFIGURE([_develop], [
+  sage_spkg_install__develop=yes
+])

build/pkgs/_develop/spkg-install

@@ -0,0 +1,2 @@
+#! /usr/bin/env bash
+# Nothing to do

build/pkgs/_develop/type

@@ -0,0 +1 @@
+optional

build/pkgs/jupyter_sphinx/distros/conda.txt

@@ -0,0 +1 @@
+jupyter_sphinx

build/pkgs/pytest/dependencies

@@ -1,4 +1,4 @@
-$(PYTHON) pluggy packaging attrs py pyparsing importlib_metadata | $(PYTHON_TOOLCHAIN)
+$(PYTHON) pluggy packaging attrs py pyparsing importlib_metadata tomli | $(PYTHON_TOOLCHAIN)
 
 ----------
 All lines of this file are ignored except the first.

build/pkgs/pytest_xdist/SPKG.rst

@@ -0,0 +1,18 @@
+pytest_xdist: pytest xdist plugin for distributed testing and loop-on-failing modes
+===================================================================================
+
+Description
+-----------
+
+pytest xdist plugin for distributed testing and loop-on-failing modes
+
+License
+-------
+
+MIT
+
+Upstream Contact
+----------------
+
+https://pypi.org/project/pytest-xdist/
+

build/pkgs/pytest_xdist/dependencies

@@ -0,0 +1,4 @@
+$(PYTHON) pytest | $(PYTHON_TOOLCHAIN)
+
+----------
+All lines of this file are ignored except the first.

build/pkgs/pytest_xdist/distros/conda.txt

@@ -0,0 +1 @@
+pytest-xdist

build/pkgs/pytest_xdist/requirements.txt

@@ -0,0 +1 @@
+pytest-xdist

build/pkgs/pytest_xdist/type

@@ -0,0 +1 @@
+optional

src/doc/bootstrap

@@ -27,7 +27,7 @@ shopt -s extglob
 
 RECOMMENDED_SPKG_PATTERN="@(_recommended$(for a in $(head -n 1 build/pkgs/_recommended/dependencies); do echo -n "|"$a; done))"
 
-for SYSTEM in arch debian fedora cygwin homebrew conda; do
+for SYSTEM in arch debian fedora cygwin homebrew; do
     SYSTEM_PACKAGES=
     OPTIONAL_SYSTEM_PACKAGES=
     SAGELIB_SYSTEM_PACKAGES=
@@ -63,40 +63,13 @@ for SYSTEM in arch debian fedora cygwin homebrew conda; do
                     fi
                 fi
     done
-    if [ "${SYSTEM}" = "conda" ]; then
-        if [ "${BOOTSTRAP_QUIET}" = "no" ]; then
-            echo >&2 $0:$LINENO: installing environment.yml, environment-optional.yml, src/environment.yml and src/environment-optional.yml
-        fi
-        echo "name: sage-build" >  environment.yml
-        echo "channels:"        >> environment.yml
-        echo "  - conda-forge"  >> environment.yml
-        echo "  - nodefaults"   >> environment.yml
-        echo "dependencies:"    >> environment.yml
-        for pkg in $SYSTEM_PACKAGES; do
-            echo "  - $pkg" >> environment.yml
-        done
-        sed 's/name: sage-build/name: sage/' environment.yml > src/environment.yml
-        for pkg in $SAGELIB_SYSTEM_PACKAGES; do
-            echo "  - $pkg" >> src/environment.yml
-        done
-        cp environment.yml environment-optional.yml
-            echo "  # optional packages" >> environment-optional.yml
-        for pkg in $OPTIONAL_SYSTEM_PACKAGES; do
-            echo "  - $pkg" >> environment-optional.yml
-            done
-        cp src/environment.yml src/environment-optional.yml
-            echo "  # optional packages" >> src/environment-optional.yml
-        for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do
-            echo "  - $pkg" >> src/environment-optional.yml
-        done
-    else
-        if [ "${BOOTSTRAP_QUIET}" = "no" ]; then
-            echo >&2 $0:$LINENO: installing "$OUTPUT_DIR"/$SYSTEM"*.txt"
-        fi
-        echo "$(sage-print-system-package-command $SYSTEM --prompt --sudo install $(echo $(echo $SYSTEM_PACKAGES | xargs -n 1 echo | sort | uniq)))" > "$OUTPUT_DIR"/$SYSTEM.txt
-        echo "$(sage-print-system-package-command $SYSTEM --prompt --sudo install $(echo $(echo $OPTIONAL_SYSTEM_PACKAGES | xargs -n 1 echo | sort | uniq)))" > "$OUTPUT_DIR"/$SYSTEM-optional.txt
-        echo "$(sage-print-system-package-command $SYSTEM --prompt --sudo install $(echo $(echo $RECOMMENDED_SYSTEM_PACKAGES | xargs -n 1 echo | sort | uniq)))" > "$OUTPUT_DIR"/$SYSTEM-recommended.txt
+
+    if [ "${BOOTSTRAP_QUIET}" = "no" ]; then
+        echo >&2 $0:$LINENO: installing "$OUTPUT_DIR"/$SYSTEM"*.txt"
     fi
+    echo "$(sage-print-system-package-command $SYSTEM --prompt --sudo install $(echo $(echo $SYSTEM_PACKAGES | xargs -n 1 echo | sort | uniq)))" > "$OUTPUT_DIR"/$SYSTEM.txt
+    echo "$(sage-print-system-package-command $SYSTEM --prompt --sudo install $(echo $(echo $OPTIONAL_SYSTEM_PACKAGES | xargs -n 1 echo | sort | uniq)))" > "$OUTPUT_DIR"/$SYSTEM-optional.txt
+    echo "$(sage-print-system-package-command $SYSTEM --prompt --sudo install $(echo $(echo $RECOMMENDED_SYSTEM_PACKAGES | xargs -n 1 echo | sort | uniq)))" > "$OUTPUT_DIR"/$SYSTEM-recommended.txt
 done
 
 OUTPUT_DIR="src/doc/en/reference/spkg"

src/doc/en/developer/tools.rst

@@ -252,13 +252,18 @@ package :mod:`sage.numerical.backends` and some modules in
 
 *Installation:*
 
-- ``./sage -i pytest``.
+- ``./sage -i pytest pytest_xdist``.
 
 *Usage:*
 
 - Tox, Sage doctester: At the end of ``./sage -t`` (or ``./sage --tox -e doctest``), Pytest is automatically invoked.
 
-- Manual: Run ``./sage -pytest path/to/the/test_file.py`` or ``./sage -pytest`` to run all tests.
+- Manual: Run ``./sage -pytest path/to/the/test_file.py`` or ``./sage -pytest``
+  to run all tests. The additional argument ``-n`` can be used to
+  distribute tests across multiple CPUs to speed up test execution.
+  For example, ``./sage -pytest -n 4`` will run 4 tests in parallel, while
+  ``./sage -pytest -n auto`` will spawn a number of workers processes equal
+  to the number of available CPUs.
 
 - VS Code: Install the `Python extension <https://marketplace.visualstudio.com/items?itemName=ms-python.python>`_ and follow the `offical VS Code documentation <https://code.visualstudio.com/docs/python/testing>`__.