20676: Merge this ticket with 7.3 beta3.

Makefile

@@ -15,7 +15,7 @@ build: all-build
 %::
 	@if [ -x relocate-once.py ]; then ./relocate-once.py; fi
 	$(MAKE) build/make/Makefile
-	+build/bin/sage-logger -p \
+	+build/bin/sage-logger \
 		"cd build/make && ./install '$@'" logs/install.log
 
 # If configure was run before, rerun it with the old arguments.

VERSION.txt

@@ -1 +1 @@
-SageMath version 7.3.beta2, Release Date: 2016-05-28
+SageMath version 7.3.beta3, Release Date: 2016-06-05

build/bin/sage-logger

@@ -41,13 +41,24 @@ else
     prefix=""
 fi
 
+# Use sed option to reduce buffering, to make the output appear more
+# smoothly. For GNU sed, this is the --unbuffered option.
+# For BSD sed (which is also on OS X), this is the -l option.
+if sed </dev/null 2>/dev/null --unbuffered ""; then
+    SED="sed --unbuffered"
+elif sed </dev/null 2>/dev/null -l ""; then
+    SED="sed -l"
+else
+    SED="sed"
+fi
+
 mkdir -p "$logdir"
 
 # Redirect stdout and stderr to a subprocess running tee.
 # We trap SIGINT such that SIGINT interrupts the main process being
 # run, not the logging.
-( exec 2>&1; eval "$cmd" ) | ( trap '' SIGINT; tee -a "$logfile" ) | \
-    (while read line; do echo "${prefix}${line}"; done)
+( exec 2>&1; eval "$cmd" ) | \
+    ( trap '' SIGINT; tee -a "$logfile" | $SED "s/^/$prefix/" )
 
 pipestatus=(${PIPESTATUS[*]})
 

build/bin/sage-spkg

@@ -528,25 +528,13 @@ else
     ls -l "$PKG_SRC"
 fi
 
-sage-uncompress-spkg "$PKG_SRC"
+sage-uncompress-spkg -d src "$PKG_SRC"
 if [ $? -ne 0 ]; then
     echo >&2 "Error: failed to extract $PKG_SRC"
     exit 1
 fi
 
 if [ "$USE_LOCAL_SCRIPTS" = yes ]; then
-    shopt -s nocaseglob      # see trac:16415
-    # Strip file extension from the archive file and hope (as per
-    # automake standards) that this is the same as the directory name
-    # inside; move the directory to "src".  (This goes wrong for
-    # upstream package archives that have been renamed to appease some
-    # other Sage script, such as "latte-int", whose archive is renamed
-    # to "latte_int".)
-    mv "$(echo "$PKG_NAME_UPSTREAM" | sed 's/\.zip$//g;s/\.tgz$//g;s/\.tar\.gz$//g;s/\.tar\.xz$//g;s/\.tar\.lz$//g;s/\.tar\.bz2$//g;s/\.tar$//g')"* src
-    # Do not check for errors here; the "mv" fails in the situation
-    # mentioned above, and the package-specific script needs to do
-    # this work itself.
-    shopt -u nocaseglob
     echo "Finished set up"
 else
     echo "Finished extraction"

build/bin/sage-uncompress-spkg

@@ -1,74 +1,246 @@
 #!/usr/bin/env python
 
-# USAGE:
-#
-#   sage-uncompress-spkg PKG [FILE]
-#
-# With a single argument, unpack the file PKG to the current directory.
-#
-# If FILE is specified, extract FILE from PKG and send it to
-# stdout. (This option is present only for backwards compatibility:
-# printing the SPKG.txt file from an old-style spkg.)
+"""
+USAGE:
 
+  sage-uncompress-spkg [-d DIR] PKG [FILE]
+
+With a single argument, unpack the file PKG to the current directory.
+
+If a directory is specified with the -d option the contents of
+the archive are extracted into that directory using the following
+rules:
+
+  1. If the archive contains (like most) a top-level directory
+     which contains all other files in the archive, the contents
+     of that directory are extracted into DIR, ignoring the name
+     of the top-level directory in the archive.
+
+  2. If the archive does not contain a single top-level directory
+     then all the contents of the archive are extracted into DIR.
+
+The directory must not already exist.
+
+If FILE is specified, extract FILE from PKG and send it to
+stdout. (This option is present only for backwards compatibility:
+printing the SPKG.txt file from an old-style spkg.)
+"""
+
+from __future__ import print_function
+
+import argparse
 import copy
 import os
 import sys
 import tarfile
 import zipfile
 
 
-class UmaskExtractTarFile(tarfile.TarFile):
+def filter_os_files(filenames):
+    """
+    Given a list of filenames, returns a filtered list with OS-specific
+    special files removed.
+
+    Currently removes OSX .DS_Store files and AppleDouble format ._ files.
+    """
+
+    files_set = set(filenames)
+
+    def is_os_file(path):
+        dirname, name = os.path.split(path)
+
+        if name == '.DS_Store':
+            return True
+
+        if name.startswith('._'):
+            name = os.path.join(dirname, name[2:])
+            # These files store extended attributes on OSX
+            # In principle this could be a false positive but it's
+            # unlikely, and to be really sure we'd have to extract the file
+            # (or at least the first four bytes to check for the magic number
+            # documented in
+            # http://kaiser-edv.de/documents/AppleSingle_AppleDouble.pdf)
+            if name in files_set or os.path.normpath(name) in files_set:
+                return True
+
+        return False
+
+    filenames = filter(lambda f: not is_os_file(f), filenames)
+
+    if sys.version_info[0] == 2:
+        return filenames
+    else:
+        # Make sure to return a list on Python >= 3
+        return list(filenames)
+
+
+class SageTarFile(tarfile.TarFile):
     """
     Sage as tarfile.TarFile, but applies the user's current umask to the
-    permissions of al extracted files and directories.
+    permissions of all extracted files and directories.
 
     This mimics the default behavior of the ``tar`` utility.
 
     See http://trac.sagemath.org/ticket/20218#comment:16 for more background.
     """
 
     def __init__(self, *args, **kwargs):
-        super(UmaskExtractTarFile, self).__init__(*args, **kwargs)
+        super(SageTarFile, self).__init__(*args, **kwargs)
 
         # Unfortunately the only way to get the current umask is to set it
         # and then restore it
-        self.umask = os.umask(0777)
+        self.umask = os.umask(0o777)
         os.umask(self.umask)
 
+    @classmethod
+    def can_read(cls, filename):
+        """
+        Given an archive filename, returns True if this class can read and
+        process the archive format of that file.
+        """
+
+        return tarfile.is_tarfile(filename)
+
+    @property
+    def names(self):
+        """
+        List of filenames in the archive.
+
+        Filters out names of OS-related files that shouldn't be in the
+        archive (.DS_Store, etc.)
+        """
+
+        return filter_os_files(self.getnames())
+
     def chmod(self, tarinfo, target):
         tarinfo = copy.copy(tarinfo)
         tarinfo.mode &= ~self.umask
-        return super(UmaskExtractTarFile, self).chmod(tarinfo, target)
+        return super(SageTarFile, self).chmod(tarinfo, target)
 
+    def extractall(self, path='.', members=None):
+        """
+        Same as tarfile.TarFile.extractall but allows filenames for
+        the members argument (like zipfile.ZipFile).
+        """
 
-if __name__ == '__main__':
-    filename = sys.argv[1]
-    if tarfile.is_tarfile(filename):
-        # tar file, possibly compressed:
-        archive = UmaskExtractTarFile.open(filename, 'r:*')
-        if len(sys.argv) == 2:
-            archive.extractall()
-        else:
-            member = sys.argv[2]
-            if member in archive.getnames():
-                SPKG_TXT = archive.extractfile(member)
-                sys.stdout.write(SPKG_TXT.read())
-                exit(1)
-        archive.close()
-        exit(0)
-    if zipfile.is_zipfile(filename):
-        # zip file:
-        archive = zipfile.ZipFile(filename, 'r')
-        if len(sys.argv) == 2:
-            archive.extractall()
-        else:
-            member = sys.argv[2]
-            if member in archive.namelist():
-                sys.stdout.write(archive.read(member))
-            else:
-                exit(1)
-        archive.close()
-        exit(0)
+        if members:
+            members = [m if isinstance(m, tarfile.TarInfo)
+                         else self.getmember(m)
+                         for m in members]
+
+        return super(SageTarFile, self).extractall(path=path, members=members)
+
+    def extractbytes(self, member):
+        """
+        Return the contents of the specified archive member as bytes.
+
+        If the member does not exist, returns None.
+        """
+
+        if member in self.getnames():
+            reader = self.extractfile(member)
+            return reader.read()
+
+
+class SageZipFile(zipfile.ZipFile):
+    """
+    Wrapper for zipfile.ZipFile to provide better API fidelity with
+    SageTarFile insofar as it's used by this script.
+    """
+
+    @classmethod
+    def can_read(cls, filename):
+        """
+        Given an archive filename, returns True if this class can read and
+        process the archive format of that file.
+        """
+
+        return zipfile.is_zipfile(filename)
+
+    @property
+    def names(self):
+        """
+        List of filenames in the archive.
+
+        Filters out names of OS-related files that shouldn't be in the
+        archive (.DS_Store, etc.)
+        """
+
+        return filter_os_files(self.namelist())
+
+    def extractbytes(self, member):
+        """
+        Return the contents of the specified archive member as bytes.
+
+        If the member does not exist, returns None.
+        """
+
+        if member in self.namelist():
+            return self.read(member)
+
+
+ARCHIVE_TYPES = [SageTarFile, SageZipFile]
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-d', dest='dir', nargs=1, metavar='DIR',
+                        help='directory to extract archive contents into')
+    parser.add_argument('pkg', nargs=1, metavar='PKG',
+                        help='the archive to extract')
+    parser.add_argument('file', nargs='?', metavar='FILE',
+                        help='(deprecated) print the contents of the given '
+                             'archive member to stdout')
+
+    args = parser.parse_args(argv)
+
+    filename = args.pkg[0]
+    dirname = args.dir[0]
+
+    for cls in ARCHIVE_TYPES:
+        if cls.can_read(filename):
+            break
     else:
-        print ('Error: Unknown file type: {}'.format(filename))
-        exit(1)
+        print('Error: Unknown file type: {}'.format(filename),
+              file=sys.stderr)
+        return 1
+
+    # For now ZipFile and TarFile both have default open modes that are
+    # acceptable
+    archive = cls.open(filename)
+
+    if args.file:
+        contents = archive.extractbytes(args.file)
+        if contents:
+            print(contents, end='')
+            return 0
+        else:
+            return 1
+
+    top_level = None
+
+    if dirname:
+        if os.path.exists(dirname):
+            print('Error: Directory {} already exists'.format(dirname),
+                  file=sys.stderr)
+            return 1
+
+        top_levels = set()
+        for member in archive.names:
+            # Zip and tar files all use forward slashes as separators
+            # internally
+            top_levels.add(member.split('/', 1)[0])
+
+        if len(top_levels) == 1:
+            top_level = top_levels.pop()
+
+    archive.extractall(members=archive.names)
+
+    if top_level:
+        os.rename(top_level, dirname)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

build/pkgs/4ti2/dependencies

@@ -0,0 +1,5 @@
+$(MP_LIBRARY) glpk
+
+----------
+All lines of this file are ignored except the first.
+It is copied by SAGE_ROOT/build/make/install into SAGE_ROOT/build/make/Makefile.

build/pkgs/boost/SPKG.txt

@@ -0,0 +1,17 @@
+= boost =
+
+== Description ==
+
+Boost provides free peer-reviewed portable C++ source libraries.
+
+== License ==
+
+Boost software license (GPL compatible)
+
+== Upstream Contact ==
+
+Home page: http://boost.org
+
+== Dependencies ==
+
+None

build/pkgs/boost/checksums.ini

@@ -0,0 +1,4 @@
+tarball=boost_VERSION.tar.bz2
+sha1=f84b1a1ce764108ec3c2b7bd7704cf8dfd3c9d01
+md5=6095876341956f65f9d35939ccea1a9f
+cksum=2563896741

build/pkgs/boost/dependencies

@@ -0,0 +1,5 @@
+iconv zlib
+
+----------
+All lines of this file are ignored except the first.
+It is copied by SAGE_ROOT/build/make/install into SAGE_ROOT/build/make/Makefile.

build/pkgs/boost/package-version.txt

@@ -0,0 +1 @@
+1_61_0