Merge branch 'develop' of git://trac.sagemath.org/sage into develop

* 'develop' of git://trac.sagemath.org/sage: (185 commits)
  Updated SageMath version to 7.3.beta3
  Fixing doctests in KR.py.
  Fixed one line in doctest.
  20772 two more typos
  some minor typos in the docs (duplicated is, etc)
  Added list of basis and finite-dimensionality to finite rank free modules.
  Fixing _repr_diagram and pp for tableau (tuples).
  use flint for euler numbers
  remove setuptools from dependencies
  trac 20748 remove the dep to python in latte/4ti2
  fix two minor grammar/typo issues
  Last little fixes in the doc.
  Fixed docstring
  trac 20748 using MP_LIBRARY also in 4ti2 dependencies
  trac 20748 using MP_LIBRARY
  trac 20748 add missing deps for latte
  Boost: don't delete the old headers until the new version builds.
  trac 20718: fix issues with sorting (or not) of vertices
  Some last little doc fixes.
  Fixing docbuild issue.
  ...

Makefile

@@ -15,7 +15,7 @@ build: all-build
 %::
 	@if [ -x relocate-once.py ]; then ./relocate-once.py; fi
 	$(MAKE) build/make/Makefile
-	+build/bin/sage-logger -p \
+	+build/bin/sage-logger \
 		"cd build/make && ./install '$@'" logs/install.log
 
 # If configure was run before, rerun it with the old arguments.

VERSION.txt

@@ -1 +1 @@
-SageMath version 7.3.beta2, Release Date: 2016-05-28
+SageMath version 7.3.beta3, Release Date: 2016-06-05

build/bin/sage-logger

@@ -41,13 +41,24 @@ else
     prefix=""
 fi
 
+# Use sed option to reduce buffering, to make the output appear more
+# smoothly. For GNU sed, this is the --unbuffered option.
+# For BSD sed (which is also on OS X), this is the -l option.
+if sed </dev/null 2>/dev/null --unbuffered ""; then
+    SED="sed --unbuffered"
+elif sed </dev/null 2>/dev/null -l ""; then
+    SED="sed -l"
+else
+    SED="sed"
+fi
+
 mkdir -p "$logdir"
 
 # Redirect stdout and stderr to a subprocess running tee.
 # We trap SIGINT such that SIGINT interrupts the main process being
 # run, not the logging.
-( exec 2>&1; eval "$cmd" ) | ( trap '' SIGINT; tee -a "$logfile" ) | \
-    (while read line; do echo "${prefix}${line}"; done)
+( exec 2>&1; eval "$cmd" ) | \
+    ( trap '' SIGINT; tee -a "$logfile" | $SED "s/^/$prefix/" )
 
 pipestatus=(${PIPESTATUS[*]})
 

build/bin/sage-spkg

@@ -528,25 +528,13 @@ else
     ls -l "$PKG_SRC"
 fi
 
-sage-uncompress-spkg "$PKG_SRC"
+sage-uncompress-spkg -d src "$PKG_SRC"
 if [ $? -ne 0 ]; then
     echo >&2 "Error: failed to extract $PKG_SRC"
     exit 1
 fi
 
 if [ "$USE_LOCAL_SCRIPTS" = yes ]; then
-    shopt -s nocaseglob      # see trac:16415
-    # Strip file extension from the archive file and hope (as per
-    # automake standards) that this is the same as the directory name
-    # inside; move the directory to "src".  (This goes wrong for
-    # upstream package archives that have been renamed to appease some
-    # other Sage script, such as "latte-int", whose archive is renamed
-    # to "latte_int".)
-    mv "$(echo "$PKG_NAME_UPSTREAM" | sed 's/\.zip$//g;s/\.tgz$//g;s/\.tar\.gz$//g;s/\.tar\.xz$//g;s/\.tar\.lz$//g;s/\.tar\.bz2$//g;s/\.tar$//g')"* src
-    # Do not check for errors here; the "mv" fails in the situation
-    # mentioned above, and the package-specific script needs to do
-    # this work itself.
-    shopt -u nocaseglob
     echo "Finished set up"
 else
     echo "Finished extraction"

build/bin/sage-uncompress-spkg

@@ -1,74 +1,246 @@
 #!/usr/bin/env python
 
-# USAGE:
-#
-#   sage-uncompress-spkg PKG [FILE]
-#
-# With a single argument, unpack the file PKG to the current directory.
-#
-# If FILE is specified, extract FILE from PKG and send it to
-# stdout. (This option is present only for backwards compatibility:
-# printing the SPKG.txt file from an old-style spkg.)
+"""
+USAGE:
 
+  sage-uncompress-spkg [-d DIR] PKG [FILE]
+
+With a single argument, unpack the file PKG to the current directory.
+
+If a directory is specified with the -d option the contents of
+the archive are extracted into that directory using the following
+rules:
+
+  1. If the archive contains (like most) a top-level directory
+     which contains all other files in the archive, the contents
+     of that directory are extracted into DIR, ignoring the name
+     of the top-level directory in the archive.
+
+  2. If the archive does not contain a single top-level directory
+     then all the contents of the archive are extracted into DIR.
+
+The directory must not already exist.
+
+If FILE is specified, extract FILE from PKG and send it to
+stdout. (This option is present only for backwards compatibility:
+printing the SPKG.txt file from an old-style spkg.)
+"""
+
+from __future__ import print_function
+
+import argparse
 import copy
 import os
 import sys
 import tarfile
 import zipfile
 
 
-class UmaskExtractTarFile(tarfile.TarFile):
+def filter_os_files(filenames):
+    """
+    Given a list of filenames, returns a filtered list with OS-specific
+    special files removed.
+
+    Currently removes OSX .DS_Store files and AppleDouble format ._ files.
+    """
+
+    files_set = set(filenames)
+
+    def is_os_file(path):
+        dirname, name = os.path.split(path)
+
+        if name == '.DS_Store':
+            return True
+
+        if name.startswith('._'):
+            name = os.path.join(dirname, name[2:])
+            # These files store extended attributes on OSX
+            # In principle this could be a false positive but it's
+            # unlikely, and to be really sure we'd have to extract the file
+            # (or at least the first four bytes to check for the magic number
+            # documented in
+            # http://kaiser-edv.de/documents/AppleSingle_AppleDouble.pdf)
+            if name in files_set or os.path.normpath(name) in files_set:
+                return True
+
+        return False
+
+    filenames = filter(lambda f: not is_os_file(f), filenames)
+
+    if sys.version_info[0] == 2:
+        return filenames
+    else:
+        # Make sure to return a list on Python >= 3
+        return list(filenames)
+
+
+class SageTarFile(tarfile.TarFile):
     """
     Sage as tarfile.TarFile, but applies the user's current umask to the
-    permissions of al extracted files and directories.
+    permissions of all extracted files and directories.
 
     This mimics the default behavior of the ``tar`` utility.
 
     See http://trac.sagemath.org/ticket/20218#comment:16 for more background.
     """
 
     def __init__(self, *args, **kwargs):
-        super(UmaskExtractTarFile, self).__init__(*args, **kwargs)
+        super(SageTarFile, self).__init__(*args, **kwargs)
 
         # Unfortunately the only way to get the current umask is to set it
         # and then restore it
-        self.umask = os.umask(0777)
+        self.umask = os.umask(0o777)
         os.umask(self.umask)
 
+    @classmethod
+    def can_read(cls, filename):
+        """
+        Given an archive filename, returns True if this class can read and
+        process the archive format of that file.
+        """
+
+        return tarfile.is_tarfile(filename)
+
+    @property
+    def names(self):
+        """
+        List of filenames in the archive.
+
+        Filters out names of OS-related files that shouldn't be in the
+        archive (.DS_Store, etc.)
+        """
+
+        return filter_os_files(self.getnames())
+
     def chmod(self, tarinfo, target):
         tarinfo = copy.copy(tarinfo)
         tarinfo.mode &= ~self.umask
-        return super(UmaskExtractTarFile, self).chmod(tarinfo, target)
+        return super(SageTarFile, self).chmod(tarinfo, target)
 
+    def extractall(self, path='.', members=None):
+        """
+        Same as tarfile.TarFile.extractall but allows filenames for
+        the members argument (like zipfile.ZipFile).
+        """
 
-if __name__ == '__main__':
-    filename = sys.argv[1]
-    if tarfile.is_tarfile(filename):
-        # tar file, possibly compressed:
-        archive = UmaskExtractTarFile.open(filename, 'r:*')
-        if len(sys.argv) == 2:
-            archive.extractall()
-        else:
-            member = sys.argv[2]
-            if member in archive.getnames():
-                SPKG_TXT = archive.extractfile(member)
-                sys.stdout.write(SPKG_TXT.read())
-                exit(1)
-        archive.close()
-        exit(0)
-    if zipfile.is_zipfile(filename):
-        # zip file:
-        archive = zipfile.ZipFile(filename, 'r')
-        if len(sys.argv) == 2:
-            archive.extractall()
-        else:
-            member = sys.argv[2]
-            if member in archive.namelist():
-                sys.stdout.write(archive.read(member))
-            else:
-                exit(1)
-        archive.close()
-        exit(0)
+        if members:
+            members = [m if isinstance(m, tarfile.TarInfo)
+                         else self.getmember(m)
+                         for m in members]
+
+        return super(SageTarFile, self).extractall(path=path, members=members)
+
+    def extractbytes(self, member):
+        """
+        Return the contents of the specified archive member as bytes.
+
+        If the member does not exist, returns None.
+        """
+
+        if member in self.getnames():
+            reader = self.extractfile(member)
+            return reader.read()
+
+
+class SageZipFile(zipfile.ZipFile):
+    """
+    Wrapper for zipfile.ZipFile to provide better API fidelity with
+    SageTarFile insofar as it's used by this script.
+    """
+
+    @classmethod
+    def can_read(cls, filename):
+        """
+        Given an archive filename, returns True if this class can read and
+        process the archive format of that file.
+        """
+
+        return zipfile.is_zipfile(filename)
+
+    @property
+    def names(self):
+        """
+        List of filenames in the archive.
+
+        Filters out names of OS-related files that shouldn't be in the
+        archive (.DS_Store, etc.)
+        """
+
+        return filter_os_files(self.namelist())
+
+    def extractbytes(self, member):
+        """
+        Return the contents of the specified archive member as bytes.
+
+        If the member does not exist, returns None.
+        """
+
+        if member in self.namelist():
+            return self.read(member)
+
+
+ARCHIVE_TYPES = [SageTarFile, SageZipFile]
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-d', dest='dir', nargs=1, metavar='DIR',
+                        help='directory to extract archive contents into')
+    parser.add_argument('pkg', nargs=1, metavar='PKG',
+                        help='the archive to extract')
+    parser.add_argument('file', nargs='?', metavar='FILE',
+                        help='(deprecated) print the contents of the given '
+                             'archive member to stdout')
+
+    args = parser.parse_args(argv)
+
+    filename = args.pkg[0]
+    dirname = args.dir[0]
+
+    for cls in ARCHIVE_TYPES:
+        if cls.can_read(filename):
+            break
     else:
-        print ('Error: Unknown file type: {}'.format(filename))
-        exit(1)
+        print('Error: Unknown file type: {}'.format(filename),
+              file=sys.stderr)
+        return 1
+
+    # For now ZipFile and TarFile both have default open modes that are
+    # acceptable
+    archive = cls.open(filename)
+
+    if args.file:
+        contents = archive.extractbytes(args.file)
+        if contents:
+            print(contents, end='')
+            return 0
+        else:
+            return 1
+
+    top_level = None
+
+    if dirname:
+        if os.path.exists(dirname):
+            print('Error: Directory {} already exists'.format(dirname),
+                  file=sys.stderr)
+            return 1
+
+        top_levels = set()
+        for member in archive.names:
+            # Zip and tar files all use forward slashes as separators
+            # internally
+            top_levels.add(member.split('/', 1)[0])
+
+        if len(top_levels) == 1:
+            top_level = top_levels.pop()
+
+    archive.extractall(members=archive.names)
+
+    if top_level:
+        os.rename(top_level, dirname)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

build/pkgs/4ti2/dependencies

@@ -0,0 +1,5 @@
+$(MP_LIBRARY) glpk
+
+----------
+All lines of this file are ignored except the first.
+It is copied by SAGE_ROOT/build/make/install into SAGE_ROOT/build/make/Makefile.

build/pkgs/boost/SPKG.txt

@@ -0,0 +1,17 @@
+= boost =
+
+== Description ==
+
+Boost provides free peer-reviewed portable C++ source libraries.
+
+== License ==
+
+Boost software license (GPL compatible)
+
+== Upstream Contact ==
+
+Home page: http://boost.org
+
+== Dependencies ==
+
+None

build/pkgs/boost/checksums.ini

@@ -0,0 +1,4 @@
+tarball=boost_VERSION.tar.bz2
+sha1=f84b1a1ce764108ec3c2b7bd7704cf8dfd3c9d01
+md5=6095876341956f65f9d35939ccea1a9f
+cksum=2563896741

build/pkgs/boost/dependencies

@@ -0,0 +1,5 @@
+iconv zlib
+
+----------
+All lines of this file are ignored except the first.
+It is copied by SAGE_ROOT/build/make/install into SAGE_ROOT/build/make/Makefile.

build/pkgs/boost/package-version.txt

@@ -0,0 +1 @@
+1_61_0