diff --git a/src/bin/sage-download-file b/src/bin/sage-download-file deleted file mode 100755 index 6f9cdc92946..00000000000 --- a/src/bin/sage-download-file +++ /dev/null @@ -1,459 +0,0 @@ -#!/usr/bin/env python - -#***************************************************************************** -# Copyright (C) 2013 Volker Braun -# Distributed under the terms of the GNU General Public License (GPL) -# as published by the Free Software Foundation; either version 2 of -# the License, or (at your option) any later version. -# http://www.gnu.org/licenses/ -#***************************************************************************** - -import contextlib -import os -import sys -import re - -try: - # Python 3 - import urllib.request as urllib - import urllib.parse as urlparse -except ImportError: - import urllib - import urlparse - -try: - from sage.env import SAGE_ROOT, SAGE_DISTFILES -except ImportError: - # Sage is not yet installed - SAGE_ROOT = os.environ['SAGE_ROOT'] - SAGE_DISTFILES = os.environ.get('SAGE_DISTFILES', - os.path.join(SAGE_ROOT, 'upstream')) - -try: - # we want to assume that SAGE_DISTFILES is an actual - # directory for the remainder of this script - os.mkdir(SAGE_DISTFILES) -except OSError: - pass - - -def printflush(x): - """Print ``x`` and flush output""" - print(x) - sys.stdout.flush() - - -def http_error_default(url, fp, errcode, errmsg, headers): - """ - Callback for the URLopener to raise an exception on HTTP errors - """ - fp.close() - raise IOError(errcode, errmsg, url) - - -class ProgressBar(object): - """ - Progress bar as urllib reporthook - """ - - def __init__(self, length=70): - self.length = length - self.progress = 0 - self.stream = sys.stderr - - def start(self): - sys.stdout.flush() # make sure to not interleave stdout/stderr - self.stream.write('[') - self.stream.flush() - - def __call__(self, chunks_so_far, chunk_size, total_size): - if total_size == -1: # we do not know size - n = 0 if chunks_so_far == 0 else self.length // 2 - else: - n = chunks_so_far * chunk_size * self.length // total_size - if n > self.length: - # If there is a Content-Length, this will be sent as the last progress - return - # n ranges from 0 to length*total (exclude), so we'll print at most length dots - if n >= self.progress: - self.stream.write('.' * (n-self.progress)) - self.stream.flush() - self.progress = n - - def stop(self): - missing = '.' * (self.length - self.progress) - self.stream.write(missing + ']\n') - self.stream.flush() - - def error_stop(self): - missing = 'x' * (self.length - self.progress) - self.stream.write(missing + ']\n') - self.stream.flush() - - -def http_download(url, destination=None, progress=True, ignore_errors=False): - """ - Download via HTTP - - This should work for FTP as well but, in fact, hangs on python < - 3.4, see http://bugs.python.org/issue16270 - - INPUT: - - - ``url`` -- string. The URL to download. - - - ``destination`` -- string or ``None`` (default). The destination - file name to save to. If not specified, the file is written to - stdout. - - - ``progress`` -- boolean (default: ``True``). Whether to print a - progress bar to stderr. - - - ``ignore_errors`` -- boolean (default: ``False``). Catch network - errors (a message is still printed to stdout). - """ - if destination is None: - destination = '/dev/stdout' - opener = urllib.FancyURLopener() - opener.http_error_default = http_error_default - if progress: - progress_bar = ProgressBar() - progress_bar.start() - try: - filename, info = opener.retrieve(url, destination, progress_bar) - except IOError as err: - progress_bar.error_stop() - printflush(err) - if not ignore_errors: - raise - else: - progress_bar.stop() - else: - filename, info = opener.retrieve(url, destination) - - -class MirrorList(object): - - URL = 'http://www.sagemath.org/mirror_list' - - MAXAGE = 24*60*60 # seconds - - def __init__(self, verbose=True): - """ - If `verbose` is False, don't print messages along the way. - This is needed to produce the appropriate output for - `sage-download-file --print-fastest-mirror`. - """ - self.filename = os.path.join(SAGE_DISTFILES, 'mirror_list') - self.verbose = verbose - if self.must_refresh(): - if self.verbose: - printflush('Downloading the Sage mirror list') - with contextlib.closing(urllib.urlopen(self.URL)) as f: - mirror_list = f.read().decode("ascii") - self.mirrors = self._load(mirror_list) - self._rank_mirrors() - self._save() - else: - self.mirrors = self._load() - - def _load(self, mirror_list = None): - """ - Load and return `mirror_list` (defaults to the one on disk) as - a list of strings - """ - if mirror_list is None: - with open(self.filename, 'rt') as f: - mirror_list = f.read() - import ast - return ast.literal_eval(mirror_list) - - def _save(self): - """ - Save the mirror list for (short-term) future use. - """ - with open(self.filename, 'wt') as f: - f.write(repr(self.mirrors)) - - def _port_of_mirror(self, mirror): - if mirror.startswith('http://'): - return 80 - if mirror.startswith('https://'): - return 443 - if mirror.startswith('ftp://'): - return 21 - - def _rank_mirrors(self): - """ - Sort the mirrors by speed, fastest being first - - This method is used by the YUM fastestmirror plugin - """ - timed_mirrors = [] - import time, socket - if self.verbose: - printflush('Searching fastest mirror') - timeout = 1 - for mirror in self.mirrors: - if not mirror.startswith('http'): - # we currently cannot handle ftp:// - continue - port = self._port_of_mirror(mirror) - mirror_hostname = urlparse.urlsplit(mirror).netloc - time_before = time.time() - try: - sock = socket.create_connection((mirror_hostname, port), timeout) - sock.close() - except (IOError, socket.error, socket.timeout): - continue - result = time.time() - time_before - result_ms = int(1000 * result) - if self.verbose: - printflush(str(result_ms).rjust(5) + 'ms: ' + mirror) - timed_mirrors.append((result, mirror)) - timed_mirrors.sort() - self.mirrors = [m[1] for m in timed_mirrors] - if self.verbose: - printflush('Fastest mirror: ' + self.fastest) - - @property - def fastest(self): - return self.mirrors[0] - - def age(self): - """ - Return the age of the cached mirror list in seconds - """ - import time - mtime = os.path.getmtime(self.filename) - now = time.mktime(time.localtime()) - return now - mtime - - def must_refresh(self): - """ - Return whether we must download the mirror list. - - If and only if this method returns ``False`` is it admissible - to use the cached mirror list. - """ - if not os.path.exists(self.filename): - return True - return self.age() > self.MAXAGE - - def __iter__(self): - """ - Iterate through the list of mirrors. - - This is the main entry point into the mirror list. Every - script should just use this function to try mirrors in order - of preference. This will not just yield the official mirrors, - but also urls for packages that are currently being tested. - """ - try: - yield os.environ['SAGE_SERVER'] - except KeyError: - pass - for mirror in self.mirrors: - yield mirror - # If all else fails: Try the packages we host ourselves - yield 'http://sagepad.org/' - - - -class ChecksumError(Exception): - pass - -class FileNotMirroredError(Exception): - pass - - -class Package(object): - - def __init__(self, package_name): - self.name = package_name - self._init_checksum() - self._init_version() - - @classmethod - def all(cls): - base = os.path.join(SAGE_ROOT, 'build', 'pkgs') - for subdir in os.listdir(base): - path = os.path.join(base, subdir) - if not os.path.isdir(path): - continue - yield cls(subdir) - - @property - def path(self): - return os.path.join(SAGE_ROOT, 'build', 'pkgs', self.name) - - def _init_checksum(self): - """ - Load the checksums from the appropriate ``checksums.ini`` file - """ - checksums_ini = os.path.join(self.path, 'checksums.ini') - assignment = re.compile('(?P[a-zA-Z0-9]*)=(?P.*)') - result = dict() - with open(checksums_ini, 'rt') as f: - for line in f.readlines(): - match = assignment.match(line) - if match is None: - continue - var, value = match.groups() - result[var] = value - self.md5 = result['md5'] - self.sha1 = result['sha1'] - self.cksum = result['cksum'] - self.sha1 = result['sha1'] - self.tarball_pattern = result['tarball'] - - VERSION_PATCHLEVEL = re.compile('(?P.*)\.p(?P[0-9]+)') - - def _init_version(self): - with open(os.path.join(self.path, 'package-version.txt')) as f: - package_version = f.read().strip() - match = self.VERSION_PATCHLEVEL.match(package_version) - if match is None: - self.version = package_version - self.patchlevel = -1 - else: - self.version = match.group('version') - self.patchlevel = match.group('patchlevel') - self.tarball = self.tarball_pattern.replace('VERSION', self.version) - - -class Tarball(object): - - def __init__(self, tarball_name): - """ - A (third-party downloadable) tarball - - INPUT: - - - ``name`` - string. The full filename (``foo-1.3.tar.bz2``) - of a tarball on the Sage mirror network. - """ - self.filename = tarball_name - self.package = None - for pkg in Package.all(): - if pkg.tarball == tarball_name: - self.package = pkg - if self.package is None: - raise ValueError('tarball {0} is not referenced by any Sage package' - .format(tarball_name)) - - @property - def upstream_fqn(self): - """ - The fully-qualified (including directory) file name in the upstream directory. - """ - return os.path.join(SAGE_DISTFILES, self.filename) - - def _compute_hash(self, algorithm): - with open(self.upstream_fqn, 'rb') as f: - while True: - buf = f.read(0x100000) - if not buf: - break - algorithm.update(buf) - return algorithm.hexdigest() - - def _compute_sha1(self): - import hashlib - return self._compute_hash(hashlib.sha1()) - - def _compute_md5(self): - import hashlib - return self._compute_md5(hashlib.md5()) - - def checksum_verifies(self): - """ - Test whether the checksum of the downloaded file is correct. - """ - sha1 = self._compute_sha1() - return sha1 == self.package.sha1 - - def download(self): - """ - Download the tarball to the upstream directory. - """ - destination = os.path.join(SAGE_DISTFILES, self.filename) - if os.path.isfile(destination): - if self.checksum_verifies(): - print('Using cached file {destination}'.format(destination=destination)) - return - else: - # Garbage in the upstream directory? Delete and re-download - print('Invalid checksum for cached file {destination}, deleting' - .format(destination=destination)) - os.remove(destination) - successful_download = False - print('Attempting to download package {0} from mirrors'.format(self.filename)) - for mirror in MirrorList(): - url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename]) - printflush(url) - try: - http_download(url, self.upstream_fqn) - successful_download = True - break - except IOError: - pass # mirror doesn't have file for whatever reason... - if not successful_download: - raise FileNotMirroredError('tarball does not exist on mirror') - if not self.checksum_verifies(): - raise ChecksumError('checksum does not match') - - def save_as(self, destination): - import shutil - shutil.copy(self.upstream_fqn, destination) - - -usage = \ -""" -USAGE: - - sage-download-file --print-fastest-mirror - -Print out the fastest mirror. All further arguments are ignored in -that case. - - sage-download-file [--quiet] url-or-tarball [destination] - -The single mandatory argument can be a http:// url or a tarball -filename. In the latter case, the tarball is downloaded from the -mirror network and its checksum is verified. - -If the destination is not specified: -* a url will be downloaded and the content written to stdout -* a tarball will be saved under {SAGE_DISTFILES} -""".format(SAGE_DISTFILES=SAGE_DISTFILES) - -if __name__ == '__main__': - progress = True - url = None - destination = None - for arg in sys.argv[1:]: - if arg.startswith('--print-fastest-mirror'): - print(MirrorList(verbose=False).fastest) - sys.exit(0) - if arg.startswith('--quiet'): - progress = False - continue - if url is None: - url = arg - continue - if destination is None: - destination = arg - continue - raise ValueError('too many arguments') - if url is None: - print(usage) - sys.exit(1) - if url.startswith('http://') or url.startswith('https://') or url.startswith('ftp://'): - http_download(url, destination, progress=progress, ignore_errors=True) - else: - # url is a tarball name - tarball = Tarball(url) - tarball.download() - if destination is not None: - tarball.save_as(destination) diff --git a/src/sage_bootstrap/bin/sage-download-file b/src/sage_bootstrap/bin/sage-download-file new file mode 100755 index 00000000000..450b8843593 --- /dev/null +++ b/src/sage_bootstrap/bin/sage-download-file @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +try: + import sage_bootstrap +except ImportError: + import os, sys + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + import sage_bootstrap + +from sage_bootstrap.cmdline import SageDownloadFileApplication +SageDownloadFileApplication().run() diff --git a/src/sage_bootstrap/sage_bootstrap/cmdline.py b/src/sage_bootstrap/sage_bootstrap/cmdline.py index 98a4db6bf37..4d12d193d0a 100644 --- a/src/sage_bootstrap/sage_bootstrap/cmdline.py +++ b/src/sage_bootstrap/sage_bootstrap/cmdline.py @@ -18,9 +18,14 @@ import os import sys +from textwrap import dedent import logging log = logging.getLogger() +from sage_bootstrap.env import SAGE_DISTFILES +from sage_bootstrap.download import Download +from sage_bootstrap.mirror_list import MirrorList +from sage_bootstrap.tarball import Tarball class CmdlineSubcommands(object): @@ -35,7 +40,6 @@ def __init__(self, argv=None): self.extra_args = argv[2:] def print_help(self): - from textwrap import dedent print(dedent(self.__doc__).lstrip()) def run(self): @@ -85,3 +89,54 @@ def run_tarball(self, package_name): """ pass + +class SageDownloadFileApplication(object): + """ + USAGE: + + sage-download-file --print-fastest-mirror + + Print out the fastest mirror. All further arguments are ignored in + that case. + + sage-download-file [--quiet] url-or-tarball [destination] + + The single mandatory argument can be a http:// url or a tarball + filename. In the latter case, the tarball is downloaded from the + mirror network and its checksum is verified. + + If the destination is not specified: + * a url will be downloaded and the content written to stdout + * a tarball will be saved under {SAGE_DISTFILES} + """ + + def run(self): + progress = True + url = None + destination = None + for arg in sys.argv[1:]: + if arg.startswith('--print-fastest-mirror'): + print(MirrorList().fastest) + sys.exit(0) + if arg.startswith('--quiet'): + progress = False + continue + if url is None: + url = arg + continue + if destination is None: + destination = arg + continue + raise ValueError('too many arguments') + if url is None: + print(dedent(self.__doc__.format(SAGE_DISTFILES=SAGE_DISTFILES))) + sys.exit(1) + if url.startswith('http://') or url.startswith('https://') or url.startswith('ftp://'): + Download(url, destination, progress=progress, ignore_errors=True).run() + else: + # url is a tarball name + tarball = Tarball(url) + tarball.download() + if destination is not None: + tarball.save_as(destination) + diff --git a/src/sage_bootstrap/sage_bootstrap/compat.py b/src/sage_bootstrap/sage_bootstrap/compat.py new file mode 100644 index 00000000000..e5c485a1bd7 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/compat.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" +Python 2/3 compatibility utils +""" + + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +try: + # Python 3 + import urllib.request as urllib + import urllib.parse as urlparse +except ImportError: + import urllib + import urlparse + + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO diff --git a/src/sage_bootstrap/sage_bootstrap/download.py b/src/sage_bootstrap/sage_bootstrap/download.py new file mode 100644 index 00000000000..0ccd8d6f1d3 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/download.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +""" +Download files from the internet +""" + + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import sys +import logging +log = logging.getLogger() + +from sage_bootstrap.stdio import flush +from sage_bootstrap.compat import urllib + + +class ProgressBar(object): + """ + Progress bar as urllib reporthook + """ + + def __init__(self, stream, length=70): + self.length = length + self.progress = 0 + self.stream = stream + + def start(self): + flush() # make sure to not interleave stdout/stderr + self.stream.write('[') + self.stream.flush() + + def __call__(self, chunks_so_far, chunk_size, total_size): + if total_size == -1: # we do not know size + n = 0 if chunks_so_far == 0 else self.length // 2 + else: + n = chunks_so_far * chunk_size * self.length // total_size + if n > self.length: + # If there is a Content-Length, this will be sent as the last progress + return + # n ranges from 0 to length*total (exclude), so we'll print at most length dots + if n >= self.progress: + self.stream.write('.' * (n-self.progress)) + self.stream.flush() + self.progress = n + + def stop(self): + missing = '.' * (self.length - self.progress) + self.stream.write(missing + ']\n') + self.stream.flush() + + def error_stop(self): + missing = 'x' * (self.length - self.progress) + self.stream.write(missing + ']\n') + self.stream.flush() + + + + +class Download(object): + """ + Download URL + + Right now, only via HTTP + + This should work for FTP as well but, in fact, hangs on python < + 3.4, see http://bugs.python.org/issue16270 + + INPUT: + + - ``url`` -- string. The URL to download. + + - ``destination`` -- string or ``None`` (default). The destination + file name to save to. If not specified, the file is written to + stdout. + + - ``progress`` -- boolean (default: ``True``). Whether to print a + progress bar to stderr. For testing, this can also be a stream + to which the progress bar is being sent. + + - ``ignore_errors`` -- boolean (default: ``False``). Catch network + errors (a message is still being logged). + """ + + def __init__(self, url, destination=None, progress=True, ignore_errors=False): + self.url = url + self.destination = destination or '/dev/stdout' + self.progress = (progress is not False) + self.progress_stream = sys.stderr if isinstance(progress, bool) else progress + self.ignore_errors = ignore_errors + + def http_error_default(self, url, fp, errcode, errmsg, headers): + """ + Callback for the URLopener to raise an exception on HTTP errors + """ + fp.close() + raise IOError(errcode, errmsg, url) + + def start_progress_bar(self): + if self.progress: + self.progress_bar = ProgressBar(self.progress_stream) + self.progress_bar.start() + + def success_progress_bar(self): + if self.progress: + self.progress_bar.stop() + + def error_progress_bar(self): + if self.progress: + self.progress_bar.error_stop() + + def run(self): + opener = urllib.FancyURLopener() + opener.http_error_default = self.http_error_default + self.start_progress_bar() + try: + if self.progress: + filename, info = opener.retrieve( + self.url, self.destination, self.progress_bar) + else: + filename, info = opener.retrieve( + self.url, self.destination) + except IOError as err: + self.error_progress_bar() + log.error(err) + if not self.ignore_errors: + raise + self.success_progress_bar() diff --git a/src/sage_bootstrap/sage_bootstrap/env.py b/src/sage_bootstrap/sage_bootstrap/env.py index ae9c9417e16..b0fc3e2679a 100644 --- a/src/sage_bootstrap/sage_bootstrap/env.py +++ b/src/sage_bootstrap/sage_bootstrap/env.py @@ -7,6 +7,7 @@ * ``SAGE_ROOT`` * ``SAGE_SRC`` +* ``SAGE_DISTFILES`` """ @@ -23,17 +24,25 @@ import os - -try: - SAGE_SRC = os.environ['SAGE_SRC'] -except KeyError: - SAGE_SRC = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - try: SAGE_ROOT = os.environ['SAGE_ROOT'] except KeyError: - SAGE_ROOT = os.path.dirname(SAGE_SRC) + SAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))) +SAGE_SRC = os.environ.get('SAGE_SRC', + os.path.join(SAGE_ROOT, 'src')) +SAGE_DISTFILES = os.environ.get('SAGE_DISTFILES', + os.path.join(SAGE_ROOT, 'upstream')) + + +assert os.path.isfile(os.path.join(SAGE_ROOT, 'configure.ac')), SAGE_ROOT +assert os.path.isfile(os.path.join(SAGE_SRC, 'sage_bootstrap', 'setup.py')), SAGE_SRC + +try: + # SAGE_DISTFILES does not exist in a fresh git clone + os.mkdir(SAGE_DISTFILES) +except OSError: + pass -assert os.path.isfile(os.path.join(SAGE_ROOT, 'configure.ac')) -assert os.path.isfile(os.path.join(SAGE_SRC, 'sage_bootstrap', 'setup.py')) +assert os.path.isdir(SAGE_DISTFILES) diff --git a/src/sage_bootstrap/sage_bootstrap/mirror_list.py b/src/sage_bootstrap/sage_bootstrap/mirror_list.py new file mode 100644 index 00000000000..5d38dc02e7d --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/mirror_list.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +""" +Access the List of Sage Download Mirrors +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import os +import contextlib +import logging +log = logging.getLogger() + +from sage_bootstrap.compat import urllib, urlparse +from sage_bootstrap.env import SAGE_DISTFILES + + +class MirrorList(object): + + URL = 'http://www.sagemath.org/mirror_list' + + MAXAGE = 24*60*60 # seconds + + def __init__(self): + self.filename = os.path.join(SAGE_DISTFILES, 'mirror_list') + if self.must_refresh(): + log.info('Downloading the Sage mirror list') + try: + with contextlib.closing(urllib.urlopen(self.URL)) as f: + mirror_list = f.read().decode("ascii") + except IOError: + log.critical('Downloading the mirror list failed') + self.mirrors = self._load() + else: + self.mirrors = self._load(mirror_list) + self._rank_mirrors() + self._save() + else: + self.mirrors = self._load() + + def _load(self, mirror_list=None): + """ + Load and return `mirror_list` (defaults to the one on disk) as + a list of strings + """ + if mirror_list is None: + try: + with open(self.filename, 'rt') as f: + mirror_list = f.read() + except IOError: + log.critical('Failed to load the cached mirror list') + return [] + import ast + return ast.literal_eval(mirror_list) + + def _save(self): + """ + Save the mirror list for (short-term) future use. + """ + with open(self.filename, 'wt') as f: + f.write(repr(self.mirrors)) + + def _port_of_mirror(self, mirror): + if mirror.startswith('http://'): + return 80 + if mirror.startswith('https://'): + return 443 + if mirror.startswith('ftp://'): + return 21 + + def _rank_mirrors(self): + """ + Sort the mirrors by speed, fastest being first + + This method is used by the YUM fastestmirror plugin + """ + timed_mirrors = [] + import time, socket + log.info('Searching fastest mirror') + timeout = 1 + for mirror in self.mirrors: + if not mirror.startswith('http'): + log.debug('we currently can only handle http, got %s', mirror) + continue + port = self._port_of_mirror(mirror) + mirror_hostname = urlparse.urlsplit(mirror).netloc + time_before = time.time() + try: + sock = socket.create_connection((mirror_hostname, port), timeout) + sock.close() + except (IOError, socket.error, socket.timeout) as err: + log.warning(str(err).strip() + ': ' + mirror) + continue + result = time.time() - time_before + result_ms = int(1000 * result) + log.info(str(result_ms).rjust(5) + 'ms: ' + mirror) + timed_mirrors.append((result, mirror)) + if len(timed_mirrors) == 0: + # We cannot reach any mirror directly, most likely firewall issue + if 'http_proxy' not in os.environ: + log.error('Could not reach any mirror directly and no proxy set') + raise RuntimeError('no internet connection') + log.info('Cannot time mirrors via proxy, using default order') + else: + timed_mirrors.sort() + self.mirrors = [m[1] for m in timed_mirrors] + log.info('Fastest mirror: ' + self.fastest) + + @property + def fastest(self): + return next(iter(self)) + + def age(self): + """ + Return the age of the cached mirror list in seconds + """ + import time + mtime = os.path.getmtime(self.filename) + now = time.mktime(time.localtime()) + return now - mtime + + def must_refresh(self): + """ + Return whether we must download the mirror list. + + If and only if this method returns ``False`` is it admissible + to use the cached mirror list. + """ + if not os.path.exists(self.filename): + return True + return self.age() > self.MAXAGE + + def __iter__(self): + """ + Iterate through the list of mirrors. + + This is the main entry point into the mirror list. Every + script should just use this function to try mirrors in order + of preference. This will not just yield the official mirrors, + but also urls for packages that are currently being tested. + """ + try: + yield os.environ['SAGE_SERVER'] + except KeyError: + pass + for mirror in self.mirrors: + yield mirror + # If all else fails: Try the packages we host ourselves + yield 'http://sagepad.org/' + + diff --git a/src/sage_bootstrap/sage_bootstrap/package.py b/src/sage_bootstrap/sage_bootstrap/package.py new file mode 100644 index 00000000000..3ab80f98dd3 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/package.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" +Sage Packages +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import re +import os + +import logging +log = logging.getLogger() + +from sage_bootstrap.env import SAGE_ROOT + + + +class Package(object): + + def __init__(self, package_name): + self.name = package_name + self._init_checksum() + self._init_version() + + def __eq__(self, other): + return self.tarball == other.tarball + + @classmethod + def all(cls): + base = os.path.join(SAGE_ROOT, 'build', 'pkgs') + for subdir in os.listdir(base): + path = os.path.join(base, subdir) + if not os.path.isdir(path): + continue + yield cls(subdir) + + @property + def path(self): + return os.path.join(SAGE_ROOT, 'build', 'pkgs', self.name) + + def _init_checksum(self): + """ + Load the checksums from the appropriate ``checksums.ini`` file + """ + checksums_ini = os.path.join(self.path, 'checksums.ini') + assignment = re.compile('(?P[a-zA-Z0-9]*)=(?P.*)') + result = dict() + with open(checksums_ini, 'rt') as f: + for line in f.readlines(): + match = assignment.match(line) + if match is None: + continue + var, value = match.groups() + result[var] = value + self.md5 = result['md5'] + self.sha1 = result['sha1'] + self.cksum = result['cksum'] + self.sha1 = result['sha1'] + self.tarball_pattern = result['tarball'] + + VERSION_PATCHLEVEL = re.compile('(?P.*)\.p(?P[0-9]+)') + + def _init_version(self): + with open(os.path.join(self.path, 'package-version.txt')) as f: + package_version = f.read().strip() + match = self.VERSION_PATCHLEVEL.match(package_version) + if match is None: + self.version = package_version + self.patchlevel = -1 + else: + self.version = match.group('version') + self.patchlevel = match.group('patchlevel') + self.tarball = self.tarball_pattern.replace('VERSION', self.version) + + diff --git a/src/sage_bootstrap/sage_bootstrap/stdio.py b/src/sage_bootstrap/sage_bootstrap/stdio.py index e1a863573e6..322043e0aeb 100644 --- a/src/sage_bootstrap/sage_bootstrap/stdio.py +++ b/src/sage_bootstrap/sage_bootstrap/stdio.py @@ -41,3 +41,8 @@ def __getattr__(self, attr): def init_streams(config): if not config.interactive: sys.stdout = UnbufferedStream(REAL_STDOUT) + + +def flush(): + REAL_STDOUT.flush() + REAL_STDERR.flush() diff --git a/src/sage_bootstrap/sage_bootstrap/tarball.py b/src/sage_bootstrap/sage_bootstrap/tarball.py new file mode 100644 index 00000000000..04f4be7d45e --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/tarball.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +""" +Third-Party Tarballs +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import os +import logging +log = logging.getLogger() + +from sage_bootstrap.env import SAGE_DISTFILES +from sage_bootstrap.download import Download +from sage_bootstrap.package import Package +from sage_bootstrap.mirror_list import MirrorList + + +class ChecksumError(Exception): + pass + + +class FileNotMirroredError(Exception): + pass + + +class Tarball(object): + + def __init__(self, tarball_name): + """ + A (third-party downloadable) tarball + + INPUT: + + - ``name`` - string. The full filename (``foo-1.3.tar.bz2``) + of a tarball on the Sage mirror network. + """ + self.filename = tarball_name + self.package = None + for pkg in Package.all(): + if pkg.tarball == tarball_name: + self.package = pkg + if self.package is None: + error = 'tarball {0} is not referenced by any Sage package'.format(tarball_name) + log.error(error) + raise ValueError(error) + + @property + def upstream_fqn(self): + """ + The fully-qualified (including directory) file name in the upstream directory. + """ + return os.path.join(SAGE_DISTFILES, self.filename) + + def _compute_hash(self, algorithm): + with open(self.upstream_fqn, 'rb') as f: + while True: + buf = f.read(0x100000) + if not buf: + break + algorithm.update(buf) + return algorithm.hexdigest() + + def _compute_sha1(self): + import hashlib + return self._compute_hash(hashlib.sha1()) + + def _compute_md5(self): + import hashlib + return self._compute_md5(hashlib.md5()) + + def checksum_verifies(self): + """ + Test whether the checksum of the downloaded file is correct. + """ + sha1 = self._compute_sha1() + return sha1 == self.package.sha1 + + def download(self): + """ + Download the tarball to the upstream directory. + """ + destination = os.path.join(SAGE_DISTFILES, self.filename) + if os.path.isfile(destination): + if self.checksum_verifies(): + log.info('Using cached file {destination}'.format(destination=destination)) + return + else: + # Garbage in the upstream directory? Delete and re-download + log.info('Invalid checksum for cached file {destination}, deleting' + .format(destination=destination)) + os.remove(destination) + successful_download = False + log.info('Attempting to download package {0} from mirrors'.format(self.filename)) + for mirror in MirrorList(): + url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename]) + log.info(url) + try: + Download(url, self.upstream_fqn).run() + successful_download = True + break + except IOError: + log.debug('File not on mirror') + if not successful_download: + raise FileNotMirroredError('tarball does not exist on mirror network') + if not self.checksum_verifies(): + raise ChecksumError('checksum does not match') + + def save_as(self, destination): + import shutil + shutil.copy(self.upstream_fqn, destination) + diff --git a/src/sage_bootstrap/setup.py b/src/sage_bootstrap/setup.py index ce8727c98f5..a783496b951 100755 --- a/src/sage_bootstrap/setup.py +++ b/src/sage_bootstrap/setup.py @@ -8,7 +8,7 @@ author='Volker Braun', author_email='vbraun.name@gmail.com', packages=['sage_bootstrap'], - scripts=['sage-pkg'], + scripts=['bin/sage-pkg'], version='1.0', url='https://www.sagemath.org', ) diff --git a/src/sage_bootstrap/test/capture.py b/src/sage_bootstrap/test/capture.py new file mode 100644 index 00000000000..4b9c4e17f72 --- /dev/null +++ b/src/sage_bootstrap/test/capture.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +""" +Capture output for testing purposes +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import sys +import contextlib +import logging +log = logging.getLogger() + +from sage_bootstrap.compat import StringIO + + +class LogCaptureHandler(logging.Handler): + + def __init__(self, log_capture): + self.records = log_capture.records + logging.Handler.__init__(self) + + def emit(self, record): + self.records.append(record) + + +class CapturedLog(object): + + def __init__(self): + self.records = [] + + def __enter__(self): + self.old_level = log.level + self.old_handlers = log.handlers + log.level = logging.INFO + log.handlers = [LogCaptureHandler(self)] + return self + + def __exit__(self, type, value, traceback): + log.level = self.old_level + log.handlers = self.old_handlers + + def messages(self): + return tuple((rec.levelname, rec.getMessage()) for rec in self.records) + + +@contextlib.contextmanager +def CapturedOutput(): + new_out, new_err = StringIO(), StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = new_out, new_err + yield sys.stdout, sys.stderr + finally: + sys.stdout, sys.stderr = old_out, old_err diff --git a/src/sage_bootstrap/test/test_download.py b/src/sage_bootstrap/test/test_download.py new file mode 100644 index 00000000000..135583a89d8 --- /dev/null +++ b/src/sage_bootstrap/test/test_download.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" +Test downloading files +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import unittest +import tempfile +from textwrap import dedent + + +from .capture import CapturedLog +from sage_bootstrap.download import Download +from sage_bootstrap.mirror_list import MirrorList +from sage_bootstrap.compat import StringIO + + +class DownloadTestCase(unittest.TestCase): + + def test_download_mirror_list(self): + tmp = tempfile.NamedTemporaryFile() + tmp.close() + progress = StringIO() + Download(MirrorList.URL, tmp.name, progress=progress).run() + self.assertEqual(progress.getvalue(), + '[......................................................................]\n') + with open(tmp.name, 'r') as f: + content = f.read() + self.assertTrue(content.startswith('# Sage Mirror List')) + + def test_error(self): + URL = 'http://www.sagemath.org/sage_bootstrap/this_url_does_not_exist' + progress = StringIO() + download = Download(URL, progress=progress) + log = CapturedLog() + def action(): + with log: + download.run() + self.assertRaises(IOError, action) + self.assertIsNotFoundError(log.messages()) + self.assertEqual(progress.getvalue(), + '[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]\n') + + def test_ignore_errors(self): + URL = 'http://www.sagemath.org/sage_bootstrap/this_url_does_not_exist' + with CapturedLog() as log: + Download(URL, progress=False, ignore_errors=True).run() + self.assertIsNotFoundError(log.messages()) + + def assertIsNotFoundError(self, messages): + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0][0], 'ERROR') + self.assertTrue(messages[0][1].startswith('[Errno')) + self.assertTrue(messages[0][1].endswith( + "Not Found: '//www.sagemath.org/sage_bootstrap/this_url_does_not_exist'")) + + diff --git a/src/sage_bootstrap/test/test_mirror_list.py b/src/sage_bootstrap/test/test_mirror_list.py new file mode 100644 index 00000000000..ddd8e21ca79 --- /dev/null +++ b/src/sage_bootstrap/test/test_mirror_list.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +""" +Test downloading files +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import unittest +import tempfile + +from .capture import CapturedLog +from sage_bootstrap.mirror_list import MirrorList + + + +class MirrorListTestCase(unittest.TestCase): + + def test_mirror_list(self): + with CapturedLog() as log: + ml = MirrorList() + msg = log.messages() + if len(msg) > 0: + self.assertEqual(msg[0], ('INFO', 'Downloading the Sage mirror list')) + self.assertTrue(len(ml.mirrors) >= 0) + self.assertTrue(ml.fastest.startswith('http://')) diff --git a/src/sage_bootstrap/test/test_package.py b/src/sage_bootstrap/test/test_package.py new file mode 100644 index 00000000000..bcbd9cea53b --- /dev/null +++ b/src/sage_bootstrap/test/test_package.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +Test Sage Package Handling +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import unittest +from sage_bootstrap.package import Package + + +class PackageTestCase(unittest.TestCase): + + def test_package(self): + pkg = Package('pari') + self.assertTrue(pkg.name, 'pari') + self.assertTrue(pkg.path.endswith('build/pkgs/pari')) + self.assertEqual(pkg.tarball_pattern, 'pari-VERSION.tar.gz') + self.assertTrue(pkg.tarball.startswith('pari-') and + pkg.tarball.endswith('.tar.gz')) + + + def test_all(self): + pari = Package('pari') + self.assertTrue(pari in Package.all()) + diff --git a/src/sage_bootstrap/test/test_tarball.py b/src/sage_bootstrap/test/test_tarball.py new file mode 100644 index 00000000000..663bef8f486 --- /dev/null +++ b/src/sage_bootstrap/test/test_tarball.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +""" +Test Sage Third-Party Tarball Handling +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import unittest + +from sage_bootstrap.package import Package +from sage_bootstrap.tarball import Tarball + +from .capture import CapturedLog, CapturedOutput + + + +class TarballTestCase(unittest.TestCase): + + def test_tarball(self): + pkg = Package('configure') + tarball = Tarball(pkg.tarball) + self.assertEqual(pkg, tarball.package) + with CapturedOutput() as (stdout, stderr): + with CapturedLog() as log: + tarball.download() + self.assertEqual(stdout.getvalue(), '') + self.assertTrue(tarball.checksum_verifies()) + + def test_checksum(self): + pkg = Package('configure') + tarball = Tarball(pkg.tarball) + self.assertTrue(tarball.checksum_verifies()) + with open(tarball.upstream_fqn, 'w') as f: + f.write('foobar') + self.assertFalse(tarball.checksum_verifies()) + with CapturedOutput() as (stdout, stderr): + with CapturedLog() as log: + tarball.download() + msg = log.messages() + self.assertTrue( + ('INFO', 'Attempting to download package {0} from mirrors'.format(pkg.tarball)) in msg) + self.assertEqual(stdout.getvalue(), '') + self.assertEqual(stderr.getvalue(), + '[......................................................................]\n') + self.assertTrue(tarball.checksum_verifies())