Trac #18748: Python library to bootstrap Sage

Move the logic behind package handling and I/O for Python scripts into a
separate Python library for code reuse and automated testing.

URL: http://trac.sagemath.org/18748
Reported by: vbraun
Ticket author(s): Volker Braun
Reviewer(s): John Palmieri

src/bin/sage-env

@@ -258,7 +258,7 @@ if [ -z "${SAGE_ORIG_PATH_SET}" ]; then
     SAGE_ORIG_PATH=$PATH && export SAGE_ORIG_PATH 
     SAGE_ORIG_PATH_SET=True && export SAGE_ORIG_PATH_SET 
 fi 
-export PATH="$SAGE_SRC/bin:$SAGE_LOCAL/bin:$PATH"
+export PATH="$SAGE_SRC/sage_bootstrap/bin:$SAGE_SRC/bin:$SAGE_LOCAL/bin:$PATH"
 
 # We offer a toolchain option, so if $SAGE_LOCAL/toolchain/toolchain-env exists source it.
 # Since the user might do something crazy we do not do any checks, but hope for the best.

src/sage_bootstrap/.gitignore

@@ -0,0 +1,2 @@
+/.tox
+/MANIFEST

src/sage_bootstrap/README

@@ -0,0 +1,24 @@
+Sage-Bootstrap
+
+This is a utility libary for dealing with third-party tarballs and
+building Sage. You should never import anything from the actual Sage
+library here, nor should you import anything from sage_bootstrap into
+Sage (because this would reconfigure logging). They must be kept
+separate.
+
+Everything here must support Python 2.6, 2.7, and 3.3+. Use tox
+(https://testrun.org/tox/latest/) to automatically run the tests with
+all relevant Python versions. Tests are written as unittest, not as
+doctests, because the library is not meant to be used interactively.
+Note that the library comes with a setup.py file so that tox can test
+it, but it is not meant to be installed to SAGE_LOCAL.
+
+Command-line utilities must be able to run as part of a pipe | filter
+chain. So you have to be careful about what you send to stdout. You
+should use:
+
+* print() for anything that is meant to be sent to the output pipe.
+
+* log.info() for human-readable messages about the normal program
+  flow.
+

src/sage_bootstrap/bin/sage-download-file

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+# USAGE:
+#
+#     sage-download-file --print-fastest-mirror
+#
+# Print out the fastest mirror. All further arguments are ignored in
+# that case.
+#
+#     sage-download-file [--quiet] url-or-tarball [destination]
+#
+# The single mandatory argument can be a http:// url or a tarball
+# filename. In the latter case, the tarball is downloaded from the
+# mirror network and its checksum is verified.
+#
+# If the destination is not specified:
+# * a url will be downloaded and the content written to stdout
+# * a tarball will be saved under {SAGE_DISTFILES}
+
+try:
+    import sage_bootstrap
+except ImportError:
+    import os, sys
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+    import sage_bootstrap
+
+from sage_bootstrap.cmdline import SageDownloadFileApplication
+SageDownloadFileApplication().run()

src/sage_bootstrap/bin/sage-package

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+# Script to manage third-party tarballs.
+#
+# Usage:
+#
+# * Print the configuration
+#
+#   $ sage-package config
+#   Configuration:
+#     * log = info
+#     * interactive = True
+#
+# * Print a list of all available packages
+#
+#   $ sage-package list | sort
+#   4ti2
+#   arb
+#   atlas
+#   autotools
+#   [...]
+#   zn_poly
+#
+# * Find the package name given a tarball filename
+#
+#   $ sage-package name pari-2.8-1564-gdeac36e.tar.gz
+#   pari
+#
+# * Find the tarball filename given a package name
+#
+#   $ sage-package tarball pari
+#   pari-2.8-1564-gdeac36e.tar.gz
+
+try:
+    import sage_bootstrap
+except ImportError:
+    import os, sys
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+    import sage_bootstrap
+
+from sage_bootstrap.cmdline import SagePkgApplication
+SagePkgApplication().run()

src/sage_bootstrap/sage_bootstrap/__init__.py

@@ -0,0 +1,9 @@
+
+from sage_bootstrap.config import Configuration
+config = Configuration()
+
+from sage_bootstrap.stdio import init_streams
+init_streams(config)
+
+from sage_bootstrap.logger import init_logger
+init_logger(config)

src/sage_bootstrap/sage_bootstrap/cmdline.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+"""
+Commandline handling
+
+Note that argparse is not part of Python 2.6, so we cannot rely on it here.
+"""
+
+
+#*****************************************************************************
+#       Copyright (C) 2015 Volker Braun <vbraun.name@gmail.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+#*****************************************************************************
+
+import os
+import sys
+from textwrap import dedent
+import logging
+log = logging.getLogger()
+
+from sage_bootstrap.env import SAGE_DISTFILES
+from sage_bootstrap.package import Package
+from sage_bootstrap.download import Download
+from sage_bootstrap.mirror_list import MirrorList
+from sage_bootstrap.tarball import Tarball
+
+
+class CmdlineSubcommands(object):
+
+    def __init__(self, argv=None):
+        if argv is None:
+            argv = sys.argv
+        if len(argv) == 1:
+            self.print_help()
+            sys.exit(0)
+        self.subcommand = argv[1]
+        self.extra_args = argv[2:]
+
+    def print_help(self):
+        print(dedent(self.__doc__).lstrip())
+        print('Usage:')
+        for method in sorted(dir(self)):
+            if not method.startswith('run_'):
+                continue
+            doc = dedent(getattr(self, method).__doc__).lstrip().splitlines()
+            print('')
+            print('* ' + doc[0])
+            for line in doc[1:]:
+                print('  ' + line)
+
+    def run(self):
+        try:
+            method = getattr(self, 'run_{0}'.format(self.subcommand))
+        except AttributeError:
+            log.error('unknown subcommand: {0}'.format(self.subcommand))
+            sys.exit(1)
+        try:
+            method(*self.extra_args)
+        except TypeError as err:
+            log.error('invalid arguments to the {0} subcommand: {1}'
+                      .format(self.subcommand, self.extra_args))
+            sys.exit(1)
+
+
+class SagePkgApplication(CmdlineSubcommands):
+    """
+    sage-package
+    --------
+    
+    The package script is used to manage third-party tarballs.
+    """
+
+    def run_config(self):
+        """
+        Print the configuration
+
+        $ sage-package config
+        Configuration:
+          * log = info
+          * interactive = True
+        """
+        from sage_bootstrap.config import Configuration
+        print(Configuration())
+
+    def run_list(self):
+        """
+        Print a list of all available packages
+
+        $ sage-package list | sort
+        4ti2
+        arb
+        atlas
+        autotools
+        [...]
+        zn_poly
+        """
+        for pkg in Package.all():
+            print(pkg.name)
+
+    def run_name(self, tarball_filename):
+        """
+        Find the package name given a tarball filename
+    
+        $ sage-package name pari-2.8-1564-gdeac36e.tar.gz
+        pari
+        """
+        tarball = Tarball(os.path.basename(tarball_filename))
+        print(tarball.package.name)
+
+    def run_tarball(self, package_name):
+        """
+        Find the tarball filename given a package name
+    
+        $ sage-package tarball pari
+        pari-2.8-1564-gdeac36e.tar.gz
+        """
+        package = Package(package_name)
+        print(package.tarball.filename)
+
+
+class SageDownloadFileApplication(object):    
+    """
+    USAGE:
+    
+        sage-download-file --print-fastest-mirror
+    
+    Print out the fastest mirror. All further arguments are ignored in
+    that case.
+    
+        sage-download-file [--quiet] url-or-tarball [destination]
+    
+    The single mandatory argument can be a http:// url or a tarball
+    filename. In the latter case, the tarball is downloaded from the
+    mirror network and its checksum is verified.
+    
+    If the destination is not specified:
+    * a url will be downloaded and the content written to stdout
+    * a tarball will be saved under {SAGE_DISTFILES}
+    """
+
+    def run(self):
+        progress = True
+        url = None
+        destination = None
+        for arg in sys.argv[1:]:
+            if arg.startswith('--print-fastest-mirror'):
+                print(MirrorList().fastest)
+                sys.exit(0)
+            if arg.startswith('--quiet'):
+                progress = False
+                continue
+            if url is None:
+                url = arg
+                continue
+            if destination is None:
+                destination = arg
+                continue
+            raise ValueError('too many arguments')
+        if url is None:
+            print(dedent(self.__doc__.format(SAGE_DISTFILES=SAGE_DISTFILES)))
+            sys.exit(1)
+        if url.startswith('http://') or url.startswith('https://') or url.startswith('ftp://'):
+            Download(url, destination, progress=progress, ignore_errors=True).run()
+        else:
+            # url is a tarball name
+            tarball = Tarball(url)
+            tarball.download()
+            if destination is not None:
+                tarball.save_as(destination)
+

src/sage_bootstrap/sage_bootstrap/compat.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+"""
+Python 2/3 compatibility utils
+"""
+
+
+#*****************************************************************************
+#       Copyright (C) 2015 Volker Braun <vbraun.name@gmail.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+#*****************************************************************************
+
+
+try:
+    # Python 3
+    import urllib.request as urllib
+    import urllib.parse as urlparse
+except ImportError:
+    import urllib
+    import urlparse
+
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO