Trac #15580: Integrate prereq in the new build system

After downloading Sage 6.0 either from git or from the `.tar.gz` sdist
tarball, the prereq tarball `prereq-1.2.tar.gz` is missing. This is a
huge problem, because many checks are skipped because of this.

Integrating this was probably overlooked in the sage-git build system. I
think the best solution is not have prereq as a tarball at all and
simply integrate the ''contents'' of the former prereq tarball in the
git repo.

URL: http://trac.sagemath.org/15580
Reported by: jdemeyer
Ticket author(s): R. Andrew Ohana, Jeroen Demeyer
Reviewer(s): R. Andrew Ohana, Volker Braun

.gitignore

@@ -5,6 +5,18 @@
 /logs
 /upstream
 
+#############################
+# Autotools generated files #
+#############################
+/aclocal.m4
+/autom4te.cache/
+/config/
+/config.log
+/config.status
+/configure
+/build/Makefile-auto
+/build/Makefile-auto.in
+
 ###################
 # Temporary Files #
 ###################

Makefile

@@ -9,6 +9,7 @@
 
 PIPE = build/pipestatus
 
+
 all: start doc  # indirectly depends on build
 
 logs:
@@ -82,13 +83,24 @@ bdist-clean: clean
 	rm -rf logs
 	rm -rf dist
 	rm -rf tmp
-	rm -f build/Makefile
+	rm -f aclocal.m4 config.log config.status confcache
+	rm -rf autom4te.cache
+	rm -f build/Makefile build/Makefile-auto
 	rm -f .BUILDSTART
 
 distclean: clean doc-clean lib-clean bdist-clean
 	@echo "Deleting all remaining output from build system ..."
 	rm -rf local
 
+# Delete all auto-generated files which are distributed as part of the
+# source tarball
+bootstrap-clean:
+	rm -rf config configure build/Makefile-auto.in
+
+# Remove absolutely everything which isn't part of the git repo
+maintainer-clean: distclean bootstrap-clean
+	rm -rf upstream
+
 micro_release: bdist-clean lib-clean
 	@echo "Stripping binaries ..."
 	LC_ALL=C find local/lib local/bin -type f -exec strip '{}' ';' 2>&1 | grep -v "File format not recognized" |  grep -v "File truncated" || true
@@ -140,6 +152,16 @@ ptestoptional: ptestall # just an alias
 
 ptestoptionallong: ptestalllong # just an alias
 
+bootstrap:
+	$(MAKE) configure || \
+		bash -c 'source src/bin/sage-env; sage-download-file $$SAGE_UPSTREAM/configure/configure-`cat build/pkgs/configure/package-version.txt`.tar.gz | tar zxmf -'
+
+configure: configure.ac src/bin/sage-version.sh \
+        m4/ax_c_check_flag.m4 m4/ax_gcc_option.m4 m4/ax_gcc_version.m4 m4/ax_gxx_option.m4 m4/ax_gxx_version.m4 m4/ax_prog_perl_version.m4
+	test -d config || mkdir config
+	aclocal -I m4
+	automake --add-missing --copy build/Makefile-auto
+	autoconf
 
 install:
 	echo "Experimental use only!"
@@ -160,8 +182,8 @@ install:
 	"$(DESTDIR)"/bin/sage -c # Run sage-location
 
 
-.PHONY: all build build-serial start install \
+.PHONY: all build build-serial start install micro_release bootstrap \
 	doc doc-html doc-html-jsmath doc-html-mathjax doc-pdf \
-	doc-clean clean lib-clean bdist-clean distclean micro_release \
+	doc-clean clean lib-clean bdist-clean distclean bootstrap-clean maintainer-clean \
 	test check testoptional testall testlong testoptionallong testallong \
 	ptest ptestoptional ptestall ptestlong ptestoptionallong ptestallong

bootstrap

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+########################################################################
+# Regenerate auto-generated files (e.g. configure)
+#
+# If the -s option is given, save the autogenerated scripts in
+# $SAGE_ROOT/upstream/configure-$CONFVERSION.tar.gz where CONFVERSION
+# is the version number stored in
+# build/pkgs/configure/package-version.txt
+#
+# If optional argument -i is given, then automatically increment the
+# version number
+########################################################################
+
+set -e
+
+# Either run this script from SAGE_ROOT or make sure that SAGE_ROOT
+# is set
+test -z "$SAGE_ROOT" || cd "$SAGE_ROOT"
+
+PKG=build/pkgs/configure
+MAKE="${MAKE:-make}"
+CONFVERSION=`cat $PKG/package-version.txt`
+
+bootstrap () {
+    # Start cleanly
+    $MAKE bootstrap-clean
+
+    # Generate auto-generated files
+    $MAKE configure
+}
+
+
+save () {
+    # Create configure tarball
+    CONFBALL="upstream/configure-$CONFVERSION.tar.gz"
+    echo "Creating $CONFBALL..."
+    mkdir -p upstream
+    tar zcf "$CONFBALL" configure config/* build/Makefile-auto.in
+
+    # Update version number
+    echo "$CONFVERSION" >$PKG/package-version.txt
+
+    # Compute checksum
+    src/bin/sage-fix-pkg-checksums "$CONFBALL"
+}
+
+
+usage () {
+    echo >&2 "Usage: bootstrap [-i] [-s] [-h]"
+}
+
+
+# Parse options
+SAVE=
+while getopts "sih" OPTION
+do
+     case "$OPTION" in
+         s)
+             SAVE=yes
+             ;;
+         i)
+             CONFVERSION=$(( CONFVERSION + 1 ))
+             ;;
+         h)
+             usage
+             exit
+             ;;
+         ?)
+             usage
+             exit 2
+             ;;
+     esac
+done
+
+bootstrap
+if [ -n "$SAVE" ]; then
+    save
+fi

build/Makefile-auto.am

@@ -0,0 +1,2 @@
+# Currently, this file is unused. However, it needs to exist to coerce
+# automake into thinking that this is an Automake project.

build/deps

@@ -143,12 +143,13 @@ SAGERUNTIME = $(SCRIPTS) $(INST)/sage $(INST)/$(SAGENB) $(INST)/$(IPYTHON)
 # an update of these packages will not trigger a rebuild of every other
 # package during an upgrade, see #13415.
 ###############################################################################
-base: $(INST)/$(PREREQ) $(INST)/$(BZIP2) $(INST)/$(PATCH)
+base: $(INST)/prereq $(INST)/$(BZIP2) $(INST)/$(PATCH)
 
-$(INST)/$(PREREQ):
-	$(PIPE) "base/$(PREREQ)-install 2>&1" "tee -a $(SAGE_LOGS)/$(PREREQ).log"
+$(INST)/prereq: prereq.sh
+	$(PIPE) "./prereq.sh 2>&1" "tee -a $(SAGE_LOGS)/prereq.log"
+	touch $@
 
-$(INST)/$(BZIP2): $(INST)/$(PREREQ)
+$(INST)/$(BZIP2): $(INST)/prereq
 	+$(PIPE) "$(SAGE_SPKG) $(BZIP2) 2>&1" "tee -a $(SAGE_LOGS)/$(BZIP2).log"
 
 $(INST)/$(PATCH): $(INST)/$(BZIP2)

build/install

@@ -13,7 +13,7 @@ SAGE_EXTCODE="$SAGE_SHARE/sage/ext"
 SAGE_LOGS="$SAGE_ROOT/logs/pkgs"
 SAGE_SPKG_INST="$SAGE_LOCAL/var/lib/sage/installed"
 SAGE_VERSION=`cat $SAGE_ROOT/VERSION.txt | sed 's+.*\ \(.*\),.*+\1+'`
-PATH="$SAGE_ROOT/src/bin:$SAGE_LOCAL/bin:$PATH"
+PATH="$SAGE_SRC/bin:$PATH"
 PYTHONPATH="$SAGE_LOCAL"
 export SAGE_ROOT SAGE_SRC SAGE_LOCAL SAGE_EXTCODE SAGE_LOGS SAGE_SPKG_INST SAGE_VERSION PATH PYTHONPATH
 
@@ -61,16 +61,6 @@ mkdir -p "$SAGE_LOCAL/lib"
 mkdir -p "$SAGE_SPKG_INST"
 mkdir -p "$SAGE_SHARE"
 
-# If we are upgrading from an old version of Sage where $SAGE_SHARE
-# was stored at $SAGE_ROOT/data, move it to the right location.
-# Check that $SAGE_ROOT/data is an existing directory and not a symlink.
-if [ -d "$SAGE_ROOT/data" ] && [ ! -h "$SAGE_ROOT/data" ]; then
-    mkdir -p "$SAGE_ROOT/devel"
-    mv "$SAGE_ROOT"/data/extcode "$SAGE_ROOT/devel/ext-main"
-    mv "$SAGE_ROOT"/data/* "$SAGE_SHARE"
-    rm -rf "$SAGE_ROOT/data"
-fi
-
 ###############################################################################
 # Determine whether to install GCC (gcc, g++, gfortran).
 ###############################################################################
@@ -345,26 +335,6 @@ if [ "${SAGE_PARALLEL_SPKG_BUILD:-yes}" = no ]; then
     echo "" >&3
 fi
 
-# Usage: newest_version_base $pkg
-# Print version number of latest (according to modification time)
-# base package $pkg
-newest_version_base() {
-    PKG=$1
-    # As a fallback, we also look at the latest installed package.
-    for FILE in `{ ls -1t base/$PKG-*-install; ls -1t installed/$PKG-*; } 2>/dev/null`
-    do
-        ANS=`echo "$FILE" | sed 's|.*/||; s|-install||'`
-        if [ -n "$ANS" ]; then
-            echo "$ANS"
-            return 0
-        fi
-    done
-
-    echo >&2 "Cannot determine latest version of $PKG."
-    echo "$PKG"
-    return 1
-}
-
 # Usage: newest_version $pkg
 # Print version number of latest standard package $pkg
 newest_version() {
@@ -380,9 +350,6 @@ newest_version() {
 }
 
 cat >&3 <<EOF
-# Base packages
-PREREQ=`newest_version_base prereq`
-
 # Standard packages
 ATLAS=`newest_version atlas`
 BOEHM_GC=`newest_version boehm_gc`

build/pkgs/configure/SPKG.txt

@@ -0,0 +1,31 @@
+= Configure =
+
+== Description ==
+
+This package contains a tar archive of auto-generated files. They are
+shipped with Sage in case you do not have a sufficiently recent
+autotools version installed.
+
+== License ==
+
+GPLv3+
+
+== SPKG Maintainers ==
+
+* Jeroen Demeyer
+* Volker Braun
+
+== Upstream Contact ==
+
+
+Automatically generated by Sage, use trac and/or sage-devel for questions.
+
+== Dependencies ==
+
+None
+
+== Special Update/Build Instructions ==
+
+This tarball is automatically generated by Sage whenever you run the
+$SAGE_ROOT/bootstrap or the $SAGE_ROOT/src/bin/sage-update-version
+script.

build/pkgs/configure/checksums.ini

@@ -0,0 +1,4 @@
+tarball=configure-VERSION.tar.gz
+sha1=f964016253fbe518d590635ed6a9e37a1b642272
+md5=364040d5bb74cb9452e80a84056d8785
+cksum=253235983

build/pkgs/configure/package-version.txt

@@ -0,0 +1 @@
+1

build/base/prereq-1.2-install → build/prereq.sh

@@ -3,21 +3,11 @@
 ## Check for prerequisite programs
 ###########################################
 
-
-TARGET=prereq-1.2
-if [ "x$SAGE_BUILD_DIR" = x ]; then
-    SAGE_BUILD_DIR="$SAGE_LOCAL/var/tmp/sage/build"
-fi
-mkdir -p "$SAGE_BUILD_DIR"
-if [ $? -ne 0 ]; then
-    echo >&2 "Error creating directory $SAGE_BUILD_DIR."
-    exit 1
-fi
-
 UNAME=`uname`
 RELEASE=`uname -r`
 
-cd "$SAGE_BUILD_DIR"
+# Run this from SAGE_ROOT
+cd ..
 
 echo "Starting prerequisite check."
 echo "Machine: `uname -a`"
@@ -183,40 +173,23 @@ if [ "x$SAGE_PORT" = x ]; then
     fi
 fi
 
+# Make sure the configure script exists
+$MAKE bootstrap || exit $?
 
-# Only run configure if the tarfile exists.  Unfortunately, sage-update
-# does not update the base packages and we cannot retro-actively change
-# that in earlier versions.  So the only thing to do is to skip this
-# test.
-prereq_tarball="$SAGE_ROOT/upstream/$TARGET.tar.gz"
-if [ -f "$prereq_tarball" ]; then
-    gzip -cd "$prereq_tarball" | tar xf -
-    if [ $? -ne 0 ]; then
-        echo >&2 "Error: failed to extract '$prereq_tarball'"
+# A reasonably sophisticated test is performed in a configure
+# script, which checks compilers exist, their version numbers,
+# the fact GNU and non-GNU compilers are not mixed etc.
+./configure --disable-maintainer-mode $PREREQ_OPTIONS
+if [ $? -ne 0 ]; then
+    echo >&2 "You do not have all of the prerequisites needed to build Sage"
+    echo >&2 "from source. See the errors above."
+    if [ "x$SAGE_PORT" = x ]; then
+        echo >&2 "If you would like to try the build anyway (to help porting)"
+        echo >&2 "export the variable 'SAGE_PORT' to something non-empty."
         exit 1
+    else
+        echo >&2 "However, since 'SAGE_PORT' is set, we will try to build anyway."
     fi
-
-    # A reasonably sophisticated test is performed in a configure
-    # script, which checks compilers exist, their version numbers,
-    # the fact GNU and non-GNU compilers are not mixed etc.
-    cd "$TARGET" && ./configure $PREREQ_OPTIONS
-
-    if [ $? -ne 0 ]; then
-        echo >&2 "You do not have all of the prerequisites needed to build Sage"
-        echo >&2 "from source. See the errors above."
-        if [ "x$SAGE_PORT" = x ]; then
-            echo >&2 "If you would like to try the build anyway (to help porting)"
-            echo >&2 "export the variable 'SAGE_PORT' to something non-empty."
-            exit 1
-        else
-            echo >&2 "However, since 'SAGE_PORT' is set, we will try to build anyway."
-        fi
-    fi
-else
-    echo >&2 "The file '$prereq_tarball' does not exist."
-    echo >&2 "Most likely, this is because you upgraded from an old version of Sage."
-    echo >&2 "Since this package only does some checks, it should be okay to continue"
-    echo >&2 "without it.  Skipping prerequisite checks..."
 fi
 
 if pwd | grep >/dev/null " "; then
@@ -319,14 +292,3 @@ if [ "$UNAME" = "Darwin" ]; then
         fi
     fi
 fi
-
-if [ -z "$SAGE_KEEP_BUILT_SPKGS" ]; then
-    echo "Now cleaning up tmp files."
-    cd "$SAGE_BUILD_DIR"
-    rm -rf "$SAGE_BUILD_DIR/$TARGET"
-else
-    echo "You can safely delete the temporary build directory"
-    echo "$SAGE_BUILD_DIR/$TARGET"
-fi
-
-touch "$SAGE_SPKG_INST/$TARGET"