move sage-download-file to the sage_bootstrap library

src/sage_bootstrap/bin/sage-download-file

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+try:
+    import sage_bootstrap
+except ImportError:
+    import os, sys
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+    import sage_bootstrap
+
+from sage_bootstrap.cmdline import SageDownloadFileApplication
+SageDownloadFileApplication().run()

src/sage_bootstrap/sage_bootstrap/cmdline.py

@@ -18,9 +18,14 @@
 
 import os
 import sys
+from textwrap import dedent
 import logging
 log = logging.getLogger()
 
+from sage_bootstrap.env import SAGE_DISTFILES
+from sage_bootstrap.download import Download
+from sage_bootstrap.mirror_list import MirrorList
+from sage_bootstrap.tarball import Tarball
 
 
 class CmdlineSubcommands(object):
@@ -35,7 +40,6 @@ def __init__(self, argv=None):
         self.extra_args = argv[2:]
 
     def print_help(self):
-        from textwrap import dedent
         print(dedent(self.__doc__).lstrip())
 
     def run(self):
@@ -85,3 +89,54 @@ def run_tarball(self, package_name):
         """
         pass
 
+
+class SageDownloadFileApplication(object):    
+    """
+    USAGE:
+    
+        sage-download-file --print-fastest-mirror
+    
+    Print out the fastest mirror. All further arguments are ignored in
+    that case.
+    
+        sage-download-file [--quiet] url-or-tarball [destination]
+    
+    The single mandatory argument can be a http:// url or a tarball
+    filename. In the latter case, the tarball is downloaded from the
+    mirror network and its checksum is verified.
+    
+    If the destination is not specified:
+    * a url will be downloaded and the content written to stdout
+    * a tarball will be saved under {SAGE_DISTFILES}
+    """
+
+    def run(self):
+        progress = True
+        url = None
+        destination = None
+        for arg in sys.argv[1:]:
+            if arg.startswith('--print-fastest-mirror'):
+                print(MirrorList().fastest)
+                sys.exit(0)
+            if arg.startswith('--quiet'):
+                progress = False
+                continue
+            if url is None:
+                url = arg
+                continue
+            if destination is None:
+                destination = arg
+                continue
+            raise ValueError('too many arguments')
+        if url is None:
+            print(dedent(self.__doc__.format(SAGE_DISTFILES=SAGE_DISTFILES)))
+            sys.exit(1)
+        if url.startswith('http://') or url.startswith('https://') or url.startswith('ftp://'):
+            Download(url, destination, progress=progress, ignore_errors=True).run()
+        else:
+            # url is a tarball name
+            tarball = Tarball(url)
+            tarball.download()
+            if destination is not None:
+                tarball.save_as(destination)
+

src/sage_bootstrap/sage_bootstrap/compat.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+"""
+Python 2/3 compatibility utils
+"""
+
+
+#*****************************************************************************
+#       Copyright (C) 2015 Volker Braun <vbraun.name@gmail.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+#*****************************************************************************
+
+
+try:
+    # Python 3
+    import urllib.request as urllib
+    import urllib.parse as urlparse
+except ImportError:
+    import urllib
+    import urlparse
+
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO

src/sage_bootstrap/sage_bootstrap/download.py

@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+"""
+Download files from the internet
+"""
+
+
+#*****************************************************************************
+#       Copyright (C) 2015 Volker Braun <vbraun.name@gmail.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#                  http://www.gnu.org/licenses/
+#*****************************************************************************
+
+
+import sys
+import logging
+log = logging.getLogger()
+
+from sage_bootstrap.stdio import flush
+from sage_bootstrap.compat import urllib
+
+
+class ProgressBar(object):
+    """
+    Progress bar as urllib reporthook
+    """
+
+    def __init__(self, stream, length=70):
+        self.length = length
+        self.progress = 0
+        self.stream = stream
+
+    def start(self):
+        flush()    # make sure to not interleave stdout/stderr
+        self.stream.write('[')
+        self.stream.flush()
+
+    def __call__(self, chunks_so_far, chunk_size, total_size):
+        if total_size == -1:  # we do not know size
+            n = 0 if chunks_so_far == 0 else self.length // 2
+        else:
+            n = chunks_so_far * chunk_size * self.length // total_size
+        if n > self.length: 
+            # If there is a Content-Length, this will be sent as the last progress
+            return
+        # n ranges from 0 to length*total (exclude), so we'll print at most length dots
+        if n >= self.progress:
+            self.stream.write('.' * (n-self.progress))
+            self.stream.flush()
+        self.progress = n
+
+    def stop(self):
+        missing = '.' * (self.length - self.progress)
+        self.stream.write(missing + ']\n')
+        self.stream.flush()
+
+    def error_stop(self):
+        missing = 'x' * (self.length - self.progress)
+        self.stream.write(missing + ']\n')
+        self.stream.flush()
+
+
+
+
+class Download(object):
+    """
+    Download URL
+    
+    Right now, only via HTTP
+
+    This should work for FTP as well but, in fact, hangs on python <
+    3.4, see http://bugs.python.org/issue16270
+
+    INPUT:
+
+    - ``url`` -- string. The URL to download.
+
+    - ``destination`` -- string or ``None`` (default). The destination
+      file name to save to. If not specified, the file is written to
+      stdout.
+
+    - ``progress`` -- boolean (default: ``True``). Whether to print a
+      progress bar to stderr. For testing, this can also be a stream
+      to which the progress bar is being sent.
+
+    - ``ignore_errors`` -- boolean (default: ``False``). Catch network
+      errors (a message is still being logged).
+    """
+
+    def __init__(self, url, destination=None, progress=True, ignore_errors=False):
+        self.url = url
+        self.destination = destination or '/dev/stdout'
+        self.progress = (progress is not False)
+        self.progress_stream = sys.stderr if isinstance(progress, bool) else progress
+        self.ignore_errors = ignore_errors
+
+    def http_error_default(self, url, fp, errcode, errmsg, headers):
+        """
+        Callback for the URLopener to raise an exception on HTTP errors
+        """
+        fp.close()
+        raise IOError(errcode, errmsg, url)
+
+    def start_progress_bar(self):
+        if self.progress:
+            self.progress_bar = ProgressBar(self.progress_stream)
+            self.progress_bar.start()
+
+    def success_progress_bar(self):
+        if self.progress:
+            self.progress_bar.stop()
+
+    def error_progress_bar(self):
+        if self.progress:
+            self.progress_bar.error_stop()
+
+    def run(self):
+        opener = urllib.FancyURLopener()
+        opener.http_error_default = self.http_error_default
+        self.start_progress_bar()
+        try:
+            if self.progress:
+                filename, info = opener.retrieve(
+                    self.url, self.destination, self.progress_bar)
+            else:
+                filename, info = opener.retrieve(
+                    self.url, self.destination)
+        except IOError as err:
+            self.error_progress_bar()
+            log.error(err)
+            if not self.ignore_errors:
+                raise
+        self.success_progress_bar()

src/sage_bootstrap/sage_bootstrap/env.py

@@ -7,6 +7,7 @@
 
 * ``SAGE_ROOT``
 * ``SAGE_SRC``
+* ``SAGE_DISTFILES``
 """
 
 
@@ -23,17 +24,25 @@
 import os
 
 
-
-try:
-    SAGE_SRC = os.environ['SAGE_SRC']
-except KeyError:
-    SAGE_SRC = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
-
 try:
     SAGE_ROOT = os.environ['SAGE_ROOT']
 except KeyError:
-    SAGE_ROOT = os.path.dirname(SAGE_SRC)
+    SAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(
+        os.path.abspath(__file__)))))
 
+SAGE_SRC = os.environ.get('SAGE_SRC',
+    os.path.join(SAGE_ROOT, 'src'))
+SAGE_DISTFILES = os.environ.get('SAGE_DISTFILES',
+    os.path.join(SAGE_ROOT, 'upstream'))
+
+
+assert os.path.isfile(os.path.join(SAGE_ROOT, 'configure.ac')), SAGE_ROOT
+assert os.path.isfile(os.path.join(SAGE_SRC, 'sage_bootstrap', 'setup.py')), SAGE_SRC
+
+try:
+    # SAGE_DISTFILES does not exist in a fresh git clone
+    os.mkdir(SAGE_DISTFILES)
+except OSError:
+    pass
 
-assert os.path.isfile(os.path.join(SAGE_ROOT, 'configure.ac'))
-assert os.path.isfile(os.path.join(SAGE_SRC, 'sage_bootstrap', 'setup.py'))
+assert os.path.isdir(SAGE_DISTFILES)