Merge branch 'develop' into ticket/23851-memory_leak

COPYING.txt

@@ -54,6 +54,7 @@ extcode                     GPLv2+
 fflas_ffpack                LGPLv2.1+
 flint                       GPLv2+
 flintqs                     GPLv2+
+fplll                       LGPLv2.1+
 freetype                    FreeType License (similar to BSD; see below)
 gap                         GPLv2+
 gcc                         GPLv3+
@@ -72,7 +73,6 @@ jinja2                      Modified BSD
 jmol                        LGPLv2.1+
 jsonschema                  MIT License
 lcalc                       GPLv2+
-libfplll                    LGPLv2.1+
 libgap                      GPLv3+
 libpng                      Custom, very similar to zlib
 linbox                      LGPLv2.1+

Makefile

@@ -9,12 +9,27 @@
 
 default: all
 
-build: all-build
+all: base-toolchain
+	$(MAKE) all-start
+
+build: base-toolchain
+	$(MAKE) all-build
+
+start: base-toolchain
+	$(MAKE) build-start
+
+sageruntime: base-toolchain
+	$(MAKE) all-sageruntime
+
+
+# The --stop flag below is just a random flag to induce graceful
+# breakage with non-GNU versions of make.
+# See https://trac.sagemath.org/ticket/24617
 
 # Defer unknown targets to build/make/Makefile
 %::
 	@if [ -x relocate-once.py ]; then ./relocate-once.py; fi
-	$(MAKE) build/make/Makefile
+	$(MAKE) build/make/Makefile --stop
 	+build/bin/sage-logger \
 		"cd build/make && ./install '$@'" logs/install.log
 
@@ -37,6 +52,13 @@ build/make/Makefile: configure build/make/deps build/pkgs/*/*
 			echo "Since 'SAGE_PORT' is set, we will try to build anyway."; \
 		fi; )
 
+# This is used to monitor progress towards Python 3 and prevent
+# regressions. The target "build" should be upgraded to reflect the
+# level of Python 3 support that is known to work.
+buildbot-python3: configure
+	./configure --with-python=3
+	$(MAKE) build
+
 # Preemptively download all standard upstream source tarballs.
 download:
 	export SAGE_ROOT=$$(pwd) && \
@@ -139,7 +161,12 @@ install: all
 	@echo "from https://github.com/sagemath/binary-pkg"
 	@echo "******************************************************************"
 
+list:
+	@$(MAKE) --silent build/make/Makefile >&2
+	@$(MAKE) --silent -f build/make/Makefile SAGE_SPKG_INST=local $@
+
 .PHONY: default build install micro_release \
 	misc-clean bdist-clean distclean bootstrap-clean maintainer-clean \
 	test check testoptional testall testlong testoptionallong testallong \
-	ptest ptestoptional ptestall ptestlong ptestoptionallong ptestallong
+	ptest ptestoptional ptestall ptestlong ptestoptionallong ptestallong \
+	buildbot-python3 list

README.md

@@ -344,6 +344,62 @@ For more details, see:
     http://doc.sagemath.org/html/en/developer/coding_basics.html#files-and-directory-structure
 
 
+Build System
+------------
+
+This is a brief summary of the Sage software distribution's build system.
+There are two components to the full Sage system--the Sage Python library
+and its associated user interfaces, and the larger software distribution of
+Sage's main dependencies (for those dependencies not supplied by the user's
+system).
+
+Sage's Python library is built and installed using a `setup.py` script as is
+standard for Python packages (Sage's `setup.py` is non-trivial, but not
+unusual).
+
+Most of the rest of the build system is concerned with building all of Sage's
+dependencies in the correct order in relation to each other.  The dependencies
+included by Sage are referred to as SPKGs (i.e. "Sage Packages") and are listed
+under `build/pkgs`.
+
+The main entrypoint to Sage's build system is the top-level `Makefile` at the
+root of the source tree.  Unlike most normal projects that use autoconf (Sage
+does as well, as described below), this `Makefile` is not generated.  Instead,
+it contains a few high-level targets and targets related to bootstrapping the
+system.  Nonetheless, we still run `make <target>` from the root of the source
+tree--targets not explicitly defined in the top-level `Makefile` are passed
+through to another Makefile under `build/make/Makefile`.
+
+The latter `build/make/Makefile` *is* generated by an autoconf-generated
+`configure` script, using the template in `build/make/Makefile.in`.  This
+includes rules for building the Sage library itself (`make sagelib`), and for
+building and installing each of Sage's dependencies (e.g. `make python2`).
+
+Although it's possible to manually run Sage's `configure` script if one wants
+to provide some customizations (e.g. it is possible to select which BLAS
+implementation to use), the top-level `Makefile` will run `configure` for you,
+in order to build `build/make/Makefile` since it's a prerequisite for most of
+Sage's make targets.
+
+The `configure` script itself, if it is not already built, can be generated by
+running the `bootstrap` script.  The top-level `Makefile` also takes care of
+this automatically.
+
+To summarize, running a command like `make python2` at the top-level of the
+source tree goes something like this:
+
+1. `make python2`
+2. run `./bootstrap` if `configure` does not exist
+3. run `./configure` if `build/make/Makefile` doe not exist
+4. `cd` into `build/make` and run the `install` script--this is little more
+   than a front-end to running `make -f build/make/Makefile python2`, which
+   sets some necessary environment variables and logs some information
+5. `build/make/Makefile` contains the actual rule for building `python2`; this
+   includes building all of `python2`'s dependencies first (and their
+   dependencies, recursively); the actual package installation is performed
+   with the `sage-spkg` program
+
+
 Relocation
 ----------
 

VERSION.txt

@@ -1 +1 @@
-SageMath version 8.1.beta5, Release Date: 2017-09-11
+SageMath version 8.3.beta1, Release Date: 2018-05-14

build/bin/sage-flock

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# vim: set filetype=python:
+
+# USAGE:
+#
+#     sage-flock [-s|-x] LOCKFILE COMMAND [...]
+#
+# Obtain an exclusive (-x, default) or shared (-s) lock on LOCKFILE and then
+# exec COMMAND, with remaining arguments passed to COMMAND.
+#
+# This is originally motivated by pip, but has since been generalized.  We
+# should avoid running pip while uninstalling a package because that is prone
+# to race conditions. This script can be used to run pip under a lock.  For
+# details, see https://trac.sagemath.org/ticket/21672
+
+try:
+    import sage_bootstrap
+except ImportError:
+    import os, sys
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+    import sage_bootstrap
+
+from sage_bootstrap.flock import run
+run()

build/bin/sage-pip-install

@@ -60,10 +60,11 @@ fi
 # We should avoid running pip2/3 while uninstalling a package because that
 # is prone to race conditions. Therefore, we use a lockfile while
 # running pip. This is implemented in the Python script pip2/3-lock.
+LOCK="$SAGE_LOCAL/var/lock/$PIP.lock"
 
 # Keep uninstalling as long as it succeeds
 while true; do
-    out=$($PIP-lock uninstall --disable-pip-version-check -y "$name" 2>&1)
+    out=$(sage-flock -x $LOCK $PIP uninstall --disable-pip-version-check -y "$name" 2>&1)
     if [ $? -ne 0 ]; then
         break
     fi
@@ -83,9 +84,8 @@ fi
 # to apply a shared lock)
 echo "Installing package $name using $PIP"
 
-$PIP-lock SHARED install $pip_install_flags .
+sage-flock -s $LOCK $PIP install $pip_install_flags .
 if [ $? -ne 0 ]; then
     echo >&2 "Error: installing with $PIP failed"
     exit 3
 fi
-

build/bin/sage-spkg

@@ -21,6 +21,7 @@
 #      SAGE_ROOT      -- root directory of sage install
 #      SAGE_LOCAL     -- $SAGE_ROOT/local
 #      SAGE_DISTFILES -- directory that stores upstream tarballs
+#      SAGE_DESTDIR   -- temporary root the package will be installed to
 #      PKG_BASE       -- the base name of the package itself (e.g. 'patch')
 #      PKG_VER        -- the version number of the package
 #      PKG_NAME       -- $PKG_BASE-$PKG_VER
@@ -220,7 +221,7 @@ while true; do
         -s)
             export SAGE_KEEP_BUILT_SPKGS=yes;;
         -c|--check)
-            SAGE_CHECK_PACKAGES=x  # nonempty, so not set to '!python2' later
+            SAGE_CHECK_PACKAGES=x  # nonempty, so not set to default later
             export SAGE_CHECK=yes;;
         -*)
             echo >&2 "Error: unknown option '$1'"
@@ -615,7 +616,7 @@ fi
 if [ "$USE_LOCAL_SCRIPTS" = yes ]; then
     # New-style package
     echo "Setting up build directory for $PKG_NAME"
-    cp -Rp "$PKG_SCRIPTS" "$PKG_NAME"
+    cp -RLp "$PKG_SCRIPTS" "$PKG_NAME"
     cd "$PKG_NAME" || exit $?
 
     sage-uncompress-spkg -d src "$PKG_SRC"
@@ -713,7 +714,7 @@ __EOF__
 }
 
 
-for script in build install check; do
+for script in build install check postinst; do
     script="spkg-$script"
     if [ -f "$script" ]; then
         if [ "$USE_LOCAL_SCRIPTS" = "yes" ]; then
@@ -767,7 +768,7 @@ fi
 # Since Python's self-tests seem to fail on all platforms, we disable
 # its test suite by default.
 if [ -z "$SAGE_CHECK_PACKAGES" ]; then
-    SAGE_CHECK_PACKAGES='!python2'
+    SAGE_CHECK_PACKAGES='!python2,!python3'
 fi
 # Allow spaces, commas, or colons as separator (the documentation suggests commas).
 if echo ",$SAGE_CHECK_PACKAGES," | grep -i "[ ,:]\!$PKG_BASE[ ,:]" > /dev/null ; then
@@ -800,10 +801,41 @@ export https_proxy=$http_proxy
 export ftp_proxy=$http_proxy
 export rsync_proxy=$http_proxy
 
+##################################################################
+# We need to run sage-rebase.sh for each package installed, but it
+# can be dangerous to do this while other packages are installing
+# so we need to use a lock to manage when rebase is allowed to
+# run.  Because of this, if multiple sage-spkg runs are waiting on
+# the rebase lock, we can end up with multiple consecutive rebase
+# calls that are redundant, but I don't see an obvious way around
+# that.  This also unfortunately slows down parallel builds since
+# all packages will eventually need to wait for this lock, but
+# again there's no simple way around that.
+##################################################################
+
+if [ "$UNAME" = "CYGWIN" ]; then
+    if [ ! -d "$SAGE_LOCAL/var/lock" ]; then
+        mkdir -p "$SAGE_LOCAL/var/lock"
+    fi
+    exec 200>"$SAGE_LOCAL/var/lock/rebase.lock"
+    sage-flock -s $lock_type 200
+fi
+
 ##################################################################
 # Actually install
 ##################################################################
 
+# Set the $SAGE_DESTDIR variable to be passed to the spkg-install
+# script (the script itself could set this, but better to standardize
+# this in one place)
+export SAGE_DESTDIR="${SAGE_BUILD_DIR}/${PKG_NAME}/inst"
+
+# The actual prefix where the installation will be staged. This is the
+# directory that you need to work in if you want to change the staged
+# installation tree (before final installation to $SAGE_LOCAL) at the
+# end of spkg-install.
+export SAGE_DESTDIR_LOCAL="${SAGE_DESTDIR}${SAGE_LOCAL}"
+
 
 if [ -f spkg-build ]; then
     # Package has both spkg-build and spkg-install; execute the latter with SAGE_SUDO
@@ -826,14 +858,78 @@ else
     fi
 fi
 
+# All spkgs should eventually support this, but fall back on old behavior in
+# case DESTDIR=$SAGE_DESTDIR installation was not used
+echo "Copying package files from temporary location $SAGE_DESTDIR to $SAGE_LOCAL"
+if [ -d "$SAGE_DESTDIR" ]; then
+    PREFIX="${SAGE_DESTDIR_LOCAL%/}/"
+
+    rm -f "$PREFIX"lib/*.la
+    if [ $? -ne 0 ]; then
+        error_msg "Error deleting unnecessary libtool archive files"
+        exit 1
+    fi
+
+    # Generate installed file manifest
+    FILE_LIST=""
+    FIRST=1
+    IFS=$'\n'
+    for filename in $(find "$PREFIX" -type f -o -type l | sort); do
+        filename="${filename#$PREFIX}"
+        if [ $FIRST -eq 1 ]; then
+            FILE_LIST="\"$filename\""
+            FIRST=0
+        else
+            FILE_LIST="${FILE_LIST},"$'\n        '"\"${filename}\""
+        fi
+        # Copy file from the temp install path into $SAGE_LOCAL
+        if [ ! -d "$SAGE_LOCAL/$(dirname "$filename")" ]; then
+            $SAGE_SUDO mkdir -p "$SAGE_LOCAL/$(dirname "$filename")"
+        fi
+        $SAGE_SUDO mv "$PREFIX$filename" "${SAGE_LOCAL%/}/$filename"
+        if [ $? -ne 0 ]; then
+            error_msg "Error moving files for $PKG_BASE."
+            exit 1
+        fi
+    done
+
+    # Remove the $SAGE_DESTDIR entirely once all files have been moved to their
+    # final location.
+    rm -rf "$SAGE_DESTDIR"
+fi
+
+# At this stage the path in $SAGE_DESTDIR no longer exists, so the variable
+# should be unset
+unset SAGE_DESTDIR
+unset SAGE_DESTDIR_LOCAL
+
+# Run the post-install script, if any
+if [ -f spkg-postinst ]; then
+    time $SAGE_SUDO ./spkg-postinst
+    if [ $? -ne 0 ]; then
+        error_msg "Error running the postinst script for $PKG_BASE."
+        exit 1
+    fi
+fi
+
 if [ "$UNAME" = "CYGWIN" ]; then
+    # Drop our sage-spkg's shared lock, and try to call sage-rebase.sh
+    # under an exclusive lock
+    sage-flock -u 200
+
     # Rebase after installing each package--in case any packages load this
     # package at build time we need to ensure during the build that no binaries
     # have conflicting address spaces
-    sage-rebase.sh "$SAGE_LOCAL" 2>/dev/null
+    echo "Waiting for rebase lock"
+    sage-flock -x "$SAGE_LOCAL/var/lock/rebase.lock" \
+        sage-rebase.sh "$SAGE_LOCAL" 2>/dev/null
 fi
 
-echo "Successfully installed $PKG_NAME"
+
+# Note: spkg-check tests are run after the package has been copied into
+# SAGE_LOCAL.  It might make more sense to run the tests before, but the
+# spkg-check scripts were written before use of DESTDIR installs, and so
+# fail in many cases.  This might be good to change later.
 
 if [ "$SAGE_CHECK" = "yes" ]; then
     if [ -f spkg-check ]; then
@@ -850,11 +946,6 @@ if [ "$SAGE_CHECK" = "yes" ]; then
     fi
 fi
 
-rm -f "$SAGE_LOCAL"/lib/*.la
-if [ $? -ne 0 ]; then
-    error_msg "Error deleting unnecessary libtool archive files"
-    exit 1
-fi
 
 # Mark that the new package has been installed (and tested, if
 # applicable).
@@ -866,11 +957,17 @@ cat > "$PKG_NAME_INSTALLED" << __EOF__
     "install_date": "$(date)",
     "system_uname": "$(uname -a)",
     "sage_version": "$(cat "${SAGE_ROOT}/VERSION.txt")",
-    "test_result": "$TEST_SUITE_RESULT"
+    "test_result": "$TEST_SUITE_RESULT",
+    "files": [
+        $FILE_LIST
+    ]
 }
 __EOF__
 
 
+echo "Successfully installed $PKG_NAME"
+
+
 ##################################################################
 # Delete the temporary build directory if required
 ##################################################################