From f034a447d1cde413b93b56a429a2740c317feb25 Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Sat, 20 Jun 2015 22:05:51 +0200 Subject: [PATCH 01/13] set up print and logging for sage_bootstrap --- src/sage_bootstrap/.gitignore | 2 + src/sage_bootstrap/sage-bootstrap | 9 ++ src/sage_bootstrap/sage_bootstrap/__init__.py | 9 ++ src/sage_bootstrap/sage_bootstrap/config.py | 91 +++++++++++++++++++ src/sage_bootstrap/sage_bootstrap/logger.py | 59 ++++++++++++ src/sage_bootstrap/sage_bootstrap/stdio.py | 43 +++++++++ src/sage_bootstrap/setup.py | 14 +++ src/sage_bootstrap/test/__init__.py | 0 src/sage_bootstrap/test/runnable.py | 83 +++++++++++++++++ src/sage_bootstrap/test/test_config.py | 64 +++++++++++++ src/sage_bootstrap/test/test_logger.py | 78 ++++++++++++++++ src/sage_bootstrap/tox.ini | 22 +++++ 12 files changed, 474 insertions(+) create mode 100644 src/sage_bootstrap/.gitignore create mode 100755 src/sage_bootstrap/sage-bootstrap create mode 100644 src/sage_bootstrap/sage_bootstrap/__init__.py create mode 100644 src/sage_bootstrap/sage_bootstrap/config.py create mode 100644 src/sage_bootstrap/sage_bootstrap/logger.py create mode 100644 src/sage_bootstrap/sage_bootstrap/stdio.py create mode 100755 src/sage_bootstrap/setup.py create mode 100644 src/sage_bootstrap/test/__init__.py create mode 100755 src/sage_bootstrap/test/runnable.py create mode 100644 src/sage_bootstrap/test/test_config.py create mode 100644 src/sage_bootstrap/test/test_logger.py create mode 100644 src/sage_bootstrap/tox.ini diff --git a/src/sage_bootstrap/.gitignore b/src/sage_bootstrap/.gitignore new file mode 100644 index 00000000000..d7cf999bf8f --- /dev/null +++ b/src/sage_bootstrap/.gitignore @@ -0,0 +1,2 @@ +/.tox +/MANIFEST diff --git a/src/sage_bootstrap/sage-bootstrap b/src/sage_bootstrap/sage-bootstrap new file mode 100755 index 00000000000..92cb7d0b635 --- /dev/null +++ b/src/sage_bootstrap/sage-bootstrap @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +import sage_bootstrap + +print('done') + +from sage_bootstrap.config import Configuration + +print(Configuration()) diff --git a/src/sage_bootstrap/sage_bootstrap/__init__.py b/src/sage_bootstrap/sage_bootstrap/__init__.py new file mode 100644 index 00000000000..3e74d02db70 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/__init__.py @@ -0,0 +1,9 @@ + +from sage_bootstrap.config import Configuration +config = Configuration() + +from sage_bootstrap.stdio import init_streams +init_streams(config) + +from sage_bootstrap.logger import init_logger +init_logger(config) diff --git a/src/sage_bootstrap/sage_bootstrap/config.py b/src/sage_bootstrap/sage_bootstrap/config.py new file mode 100644 index 00000000000..666411e28ff --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/config.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +""" +Set Up Logging + +Logging can be customized using the ``SAGE_BOOTSTRAP`` environment +variable. It is a comma-separated list of ``key:value`` pairs. They +are not case sensitive. Valid pairs are: + +* ``log:[level]``, where ``[level]`` is one of + + * ``debug`` + * ``info`` + * ``warning`` + * ``critical`` + * ``error`` + +* ``interactive:true`` or ``interactive:false``, to override isatty detection. +""" + + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import sys +import os +import logging + + +LOG_LEVELS = ( + 'debug', + 'info', + 'warning', + 'critical', + 'error' +) + + + + +class Configuration(object): + + _initialized = False + + log = 'info' + + interactive = os.isatty(sys.stdout.fileno()) + + def __init__(self): + if not Configuration._initialized: + Configuration._init_from_environ() + if self.log not in LOG_LEVELS: + raise ValueError('invalid log level: {0}'.format(self.log)) + assert isinstance(self.interactive, bool) + + @classmethod + def _init_from_environ(cls): + env = os.environ.get('SAGE_BOOTSTRAP', '').lower() + for pair in env.split(','): + if not pair.strip(): + continue + key, value = pair.split(':', 1) + key = key.strip() + value = value.strip() + if key == 'log': + cls.log = value + elif key == 'interactive': + if value == 'true': + cls.interactive = True + elif value == 'false': + cls.interactive = False + else: + raise ValueError('interactive value must be "true" or "false", got "{0}"' + .format(value)) + else: + raise ValueError('unknown key: "{0}"'.format(key)) + cls._initialized = True + + def __repr__(self): + return '\n'.join([ + 'Configuration:', + ' * log = {0}'.format(self.log), + ' * interactive = {0}'.format(self.interactive) + ]) diff --git a/src/sage_bootstrap/sage_bootstrap/logger.py b/src/sage_bootstrap/sage_bootstrap/logger.py new file mode 100644 index 00000000000..3b6ac5d56b4 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/logger.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +""" +Set Up Logging + +When using a script interactively, logging should go to stdout and be +human-readable. When using a script as part of a pipe (usually +involving tee), logging should go to stderr. +""" + + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import sys +import os +import logging + +logger = logging.getLogger() + + +default_formatter = logging.Formatter( + '%(levelname)s [%(module)s.%(funcName)s:%(lineno)s]: %(message)s') + +plain_formatter = logging.Formatter('%(message)s') + + +class ExcludeInfoFilter(logging.Filter): + + def filter(self, record): + return record.levelno != logging.INFO + +class OnlyInfoFilter(logging.Filter): + + def filter(self, record): + return record.levelno == logging.INFO + + +def init_logger(config): + level = getattr(logging, config.log.upper()) + logger.setLevel(level) + ch_stderr = logging.StreamHandler(sys.stderr) + ch_stderr.setLevel(logging.DEBUG) + ch_stderr.setFormatter(default_formatter) + if config.interactive: + ch_stderr.addFilter(ExcludeInfoFilter()) + ch_stdout = logging.StreamHandler(sys.stdout) + ch_stdout.setLevel(logging.DEBUG) + ch_stdout.setFormatter(plain_formatter) + ch_stdout.addFilter(OnlyInfoFilter()) + logger.addHandler(ch_stdout) + logger.addHandler(ch_stderr) + diff --git a/src/sage_bootstrap/sage_bootstrap/stdio.py b/src/sage_bootstrap/sage_bootstrap/stdio.py new file mode 100644 index 00000000000..e1a863573e6 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/stdio.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" +Set Up Input/output + +Output should always be unbuffered so that it appears immediately on +the terminal. +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import sys +import os + + +class UnbufferedStream(object): + + def __init__(self, stream): + self.stream = stream + + def write(self, data): + self.stream.write(data) + self.stream.flush() + + def __getattr__(self, attr): + return getattr(self.stream, attr) + + +REAL_STDOUT = sys.stdout +REAL_STDERR = sys.stderr + + +def init_streams(config): + if not config.interactive: + sys.stdout = UnbufferedStream(REAL_STDOUT) diff --git a/src/sage_bootstrap/setup.py b/src/sage_bootstrap/setup.py new file mode 100755 index 00000000000..1509a4c3bc5 --- /dev/null +++ b/src/sage_bootstrap/setup.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +from distutils.core import setup + +setup( + name='sage_bootstrap', + description='', + author='Volker Braun', + author_email='vbraun.name@gmail.com', + packages=['sage_bootstrap'], + scripts=['sage-bootstrap'], + version='1.0', + url='https://www.sagemath.org', +) diff --git a/src/sage_bootstrap/test/__init__.py b/src/sage_bootstrap/test/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/sage_bootstrap/test/runnable.py b/src/sage_bootstrap/test/runnable.py new file mode 100755 index 00000000000..9ea98f717e9 --- /dev/null +++ b/src/sage_bootstrap/test/runnable.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Utility to test running with different values for ``SAGE_BOOTSTRAP`` +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +# This function's line numbers are in unit tests, try to not move it +# up or down. This is why it is up here at the beginning. +def print_log(): + import logging + log = logging.getLogger() + log.debug('This is the debug log level') + log.info('This is the info log level') + log.warning('This is the warning log level') + log.critical('This is the critical log level') + log.error('This is the error log level') + print('This is printed') + +# From here on the line number does not matter + +import sys +import os +import json +import subprocess + + +def run_with(command, SAGE_BOOTSTRAP): + env = dict(os.environ) + env['SAGE_BOOTSTRAP'] = SAGE_BOOTSTRAP + proc = subprocess.Popen( + [sys.executable, __file__, command], + env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + out, err = proc.communicate() + return out.decode('ascii'), err.decode('ascii') + + +def run_config_with(SAGE_BOOTSTRAP): + out, err = run_with('print_config', SAGE_BOOTSTRAP) + assert not err, err + return json.loads(out) + + +def print_config(): + from sage_bootstrap.config import Configuration + from sage_bootstrap.stdio import REAL_STDOUT, REAL_STDERR + config = Configuration() + result = dict( + log=config.log, + interactive=config.interactive, + stdout='default stdout' if sys.stdout == REAL_STDOUT else str(type(sys.stdout)), + stderr='default stderr' if sys.stderr == REAL_STDERR else str(type(sys.stderr)), + ) + print(json.dumps(result)) + + +def run_log_with(SAGE_BOOTSTRAP): + return run_with('print_log', SAGE_BOOTSTRAP) + + +commands = dict( + print_config=print_config, + print_log=print_log, +) + + +if __name__ == '__main__': + sys.path.insert(0, + os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + import sage_bootstrap + commands[sys.argv[1]]() diff --git a/src/sage_bootstrap/test/test_config.py b/src/sage_bootstrap/test/test_config.py new file mode 100644 index 00000000000..03d50deddf8 --- /dev/null +++ b/src/sage_bootstrap/test/test_config.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import unittest +from sage_bootstrap.config import Configuration, LOG_LEVELS +from .runnable import run_config_with + + +class ConfigurationTestCase(unittest.TestCase): + + def test_default(self): + """ + Test the default configuration + """ + config = Configuration() + self.assertEqual(config.log, 'info') + self.assertTrue(config.interactive) + + def test_example(self): + """ + Test all ``SAGE_BOOTSTRAP`` settings + """ + SAGE_BOOTSTRAP=' loG:CrItIcAl, interactive:TRUE' + result = run_config_with(SAGE_BOOTSTRAP) + self.assertEqual(len(result), 4) + self.assertEqual(result['log'], u'critical') + self.assertTrue(result['interactive']) + self.assertEqual(result['stdout'], 'default stdout') + self.assertEqual(result['stderr'], 'default stderr') + + + def test_logging(self): + """ + Test that the different log levels are understood + """ + for level in LOG_LEVELS: + self.assertEqual( + run_config_with('LOG:{0}'.format(level.upper()))['log'], + level) + + def test_logging(self): + """ + Test that overriding the isatty detection works + """ + interactive = run_config_with('interactive:true') + self.assertTrue(interactive['interactive']) + self.assertEqual(interactive['stdout'], 'default stdout') + self.assertEqual(interactive['stderr'], 'default stderr') + in_pipe = run_config_with('interactive:false') + self.assertFalse(in_pipe['interactive']) + self.assertEqual(in_pipe['stdout'], u"") + self.assertEqual(in_pipe['stderr'], 'default stderr') + + diff --git a/src/sage_bootstrap/test/test_logger.py b/src/sage_bootstrap/test/test_logger.py new file mode 100644 index 00000000000..8f5207d1956 --- /dev/null +++ b/src/sage_bootstrap/test/test_logger.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +""" +Test the printing and logging +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import unittest +from textwrap import dedent +from .runnable import run_log_with + + + +class LoggerTestCase(unittest.TestCase): + + def test_interactive(self): + stdout, stderr = run_log_with('interactive:true') + self.assertEqual(stderr.strip(), dedent(""" + WARNING [runnable.print_log:25]: This is the warning log level + CRITICAL [runnable.print_log:26]: This is the critical log level + ERROR [runnable.print_log:27]: This is the error log level + """).strip()) + self.assertEqual(stdout.strip(), dedent(""" + This is the info log level + This is printed + """).strip()) + + def test_noninteractive(self): + stdout, stderr = run_log_with('interactive:false') + self.assertEqual(stderr.strip(), dedent(""" + INFO [runnable.print_log:24]: This is the info log level + WARNING [runnable.print_log:25]: This is the warning log level + CRITICAL [runnable.print_log:26]: This is the critical log level + ERROR [runnable.print_log:27]: This is the error log level + """).strip()) + self.assertEqual(stdout.strip(), dedent(""" + This is printed + """).strip()) + + + def test_debug(self): + """ + The lowest logging level + """ + stdout, stderr = run_log_with('log:debug,interactive:true') + self.assertEqual(stderr.strip(), dedent(""" + DEBUG [runnable.print_log:23]: This is the debug log level + WARNING [runnable.print_log:25]: This is the warning log level + CRITICAL [runnable.print_log:26]: This is the critical log level + ERROR [runnable.print_log:27]: This is the error log level + """).strip()) + self.assertEqual(stdout.strip(), dedent(""" + This is the info log level + This is printed + """).strip()) + + def test_error(self): + """ + The highest logging level + """ + stdout, stderr = run_log_with('log:error,interactive:true') + self.assertEqual(stderr.strip(), dedent(""" + CRITICAL [runnable.print_log:26]: This is the critical log level + ERROR [runnable.print_log:27]: This is the error log level + """).strip()) + self.assertEqual(stdout.strip(), dedent(""" + This is printed + """).strip()) + diff --git a/src/sage_bootstrap/tox.ini b/src/sage_bootstrap/tox.ini new file mode 100644 index 00000000000..6ac2630ae9f --- /dev/null +++ b/src/sage_bootstrap/tox.ini @@ -0,0 +1,22 @@ +[tox] +envlist = py26, py27, py33, py34 +skip_missing_interpreters=true + + +[testenv:py26] +deps = + unittest2 +commands = unit2 discover + +# We make it harder to get the encoding right by using the dumbest default +setenv = + LC_ALL = C + +[testenv:py27] +commands=python2.7 -m unittest discover + +[testenv:py33] +commands=python3.3 -m unittest discover + +[testenv:py34] +commands=python3.4 -m unittest discover From bd61b67c486126af2f90104a3fa8a277facd056a Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Sun, 21 Jun 2015 09:12:36 +0200 Subject: [PATCH 02/13] add the sage-pkg entrypoint --- src/sage_bootstrap/README | 24 ++++++ src/sage_bootstrap/bin/sage-pkg | 11 +++ src/sage_bootstrap/sage-bootstrap | 9 -- src/sage_bootstrap/sage_bootstrap/cmdline.py | 87 ++++++++++++++++++++ src/sage_bootstrap/sage_bootstrap/env.py | 39 +++++++++ src/sage_bootstrap/setup.py | 2 +- 6 files changed, 162 insertions(+), 10 deletions(-) create mode 100644 src/sage_bootstrap/README create mode 100755 src/sage_bootstrap/bin/sage-pkg delete mode 100755 src/sage_bootstrap/sage-bootstrap create mode 100644 src/sage_bootstrap/sage_bootstrap/cmdline.py create mode 100644 src/sage_bootstrap/sage_bootstrap/env.py diff --git a/src/sage_bootstrap/README b/src/sage_bootstrap/README new file mode 100644 index 00000000000..9ca5362d73d --- /dev/null +++ b/src/sage_bootstrap/README @@ -0,0 +1,24 @@ +Sage-Bootstrap + +This is a utility libary for dealing with third-party tarballs and +building Sage. You should never import anything from the actual Sage +library here, nor should you import anything from sage_bootstrap into +Sage (because this would reconfigure logging). They must be kept +separate. + +Everything here must support Python 2.6, 2.7, and 3.3+. Use tox to +automatically run the tests with all relevant Python versions. Tests +are written as unittest, not as doctests, because the library is not +meant to be used interactively. Note that the library comes with a +setup.py file so that tox can test it, but it is not meant to be +installed to SAGE_LOCAL. + +Command-line utilities must be able to run as part of a pipe | filter +chain. So you have to be careful about what you send to stdout. You +should use: + +* print() for anything that is meant to be sent to the output pipe. + +* log.info() for human-readable messages about the normal program + flow. + diff --git a/src/sage_bootstrap/bin/sage-pkg b/src/sage_bootstrap/bin/sage-pkg new file mode 100755 index 00000000000..648a34fbbfb --- /dev/null +++ b/src/sage_bootstrap/bin/sage-pkg @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +try: + import sage_bootstrap +except ImportError: + import os, sys + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + import sage_bootstrap + +from sage_bootstrap.cmdline import SagePkgApplication +SagePkgApplication().run() diff --git a/src/sage_bootstrap/sage-bootstrap b/src/sage_bootstrap/sage-bootstrap deleted file mode 100755 index 92cb7d0b635..00000000000 --- a/src/sage_bootstrap/sage-bootstrap +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env python - -import sage_bootstrap - -print('done') - -from sage_bootstrap.config import Configuration - -print(Configuration()) diff --git a/src/sage_bootstrap/sage_bootstrap/cmdline.py b/src/sage_bootstrap/sage_bootstrap/cmdline.py new file mode 100644 index 00000000000..98a4db6bf37 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/cmdline.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +""" +Commandline handling + +Note that argparse is not part of Python 2.6, so we cannot rely on it here. +""" + + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import os +import sys +import logging +log = logging.getLogger() + + + +class CmdlineSubcommands(object): + + def __init__(self, argv=None): + if argv is None: + argv = sys.argv + if len(argv) == 1: + self.print_help() + sys.exit(0) + self.subcommand = argv[1] + self.extra_args = argv[2:] + + def print_help(self): + from textwrap import dedent + print(dedent(self.__doc__).lstrip()) + + def run(self): + try: + method = getattr(self, 'run_{0}'.format(self.subcommand)) + except AttributeError: + log.error('unknown subcommand: {0}'.format(self.subcommand)) + sys.exit(1) + try: + method(*self.extra_args) + except TypeError as err: + log.error('invalid arguments to the {0} subcommand: {1}' + .format(self.subcommand, self.extra_args)) + sys.exit(1) + + +class SagePkgApplication(CmdlineSubcommands): + """ + sage-pkg + -------- + + The package script is used to manage third-party tarballs. + """ + + def run_config(self): + """ + Print the configuration + """ + from sage_bootstrap.config import Configuration + print(Configuration()) + + def run_list(self): + """ + Print a list of all available packages + """ + pass + + def run_name(self, tarball_filename): + """ + Find the package name given a tarball filename + """ + pass + + def run_tarball(self, package_name): + """ + Find the tarball filename given a package name + """ + pass + diff --git a/src/sage_bootstrap/sage_bootstrap/env.py b/src/sage_bootstrap/sage_bootstrap/env.py new file mode 100644 index 00000000000..ae9c9417e16 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/env.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +""" +Environment Variables + +This module defines the following subset of the Sage environment +variables: + +* ``SAGE_ROOT`` +* ``SAGE_SRC`` +""" + + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import os + + + +try: + SAGE_SRC = os.environ['SAGE_SRC'] +except KeyError: + SAGE_SRC = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + +try: + SAGE_ROOT = os.environ['SAGE_ROOT'] +except KeyError: + SAGE_ROOT = os.path.dirname(SAGE_SRC) + + +assert os.path.isfile(os.path.join(SAGE_ROOT, 'configure.ac')) +assert os.path.isfile(os.path.join(SAGE_SRC, 'sage_bootstrap', 'setup.py')) diff --git a/src/sage_bootstrap/setup.py b/src/sage_bootstrap/setup.py index 1509a4c3bc5..ce8727c98f5 100755 --- a/src/sage_bootstrap/setup.py +++ b/src/sage_bootstrap/setup.py @@ -8,7 +8,7 @@ author='Volker Braun', author_email='vbraun.name@gmail.com', packages=['sage_bootstrap'], - scripts=['sage-bootstrap'], + scripts=['sage-pkg'], version='1.0', url='https://www.sagemath.org', ) From 25022c39f788f5dbfef8aa3329e650f7da659926 Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Sun, 21 Jun 2015 12:27:14 +0200 Subject: [PATCH 03/13] move sage-download-file to the sage_bootstrap library --- src/bin/sage-download-file | 459 ------------------ src/sage_bootstrap/bin/sage-download-file | 11 + src/sage_bootstrap/sage_bootstrap/cmdline.py | 57 ++- src/sage_bootstrap/sage_bootstrap/compat.py | 30 ++ src/sage_bootstrap/sage_bootstrap/download.py | 136 ++++++ src/sage_bootstrap/sage_bootstrap/env.py | 27 +- .../sage_bootstrap/mirror_list.py | 158 ++++++ src/sage_bootstrap/sage_bootstrap/package.py | 83 ++++ src/sage_bootstrap/sage_bootstrap/stdio.py | 5 + src/sage_bootstrap/sage_bootstrap/tarball.py | 119 +++++ src/sage_bootstrap/setup.py | 2 +- src/sage_bootstrap/test/capture.py | 62 +++ src/sage_bootstrap/test/test_download.py | 67 +++ src/sage_bootstrap/test/test_mirror_list.py | 34 ++ src/sage_bootstrap/test/test_package.py | 35 ++ src/sage_bootstrap/test/test_tarball.py | 53 ++ 16 files changed, 868 insertions(+), 470 deletions(-) delete mode 100755 src/bin/sage-download-file create mode 100755 src/sage_bootstrap/bin/sage-download-file create mode 100644 src/sage_bootstrap/sage_bootstrap/compat.py create mode 100644 src/sage_bootstrap/sage_bootstrap/download.py create mode 100644 src/sage_bootstrap/sage_bootstrap/mirror_list.py create mode 100644 src/sage_bootstrap/sage_bootstrap/package.py create mode 100644 src/sage_bootstrap/sage_bootstrap/tarball.py create mode 100644 src/sage_bootstrap/test/capture.py create mode 100644 src/sage_bootstrap/test/test_download.py create mode 100644 src/sage_bootstrap/test/test_mirror_list.py create mode 100644 src/sage_bootstrap/test/test_package.py create mode 100644 src/sage_bootstrap/test/test_tarball.py diff --git a/src/bin/sage-download-file b/src/bin/sage-download-file deleted file mode 100755 index 6f9cdc92946..00000000000 --- a/src/bin/sage-download-file +++ /dev/null @@ -1,459 +0,0 @@ -#!/usr/bin/env python - -#***************************************************************************** -# Copyright (C) 2013 Volker Braun -# Distributed under the terms of the GNU General Public License (GPL) -# as published by the Free Software Foundation; either version 2 of -# the License, or (at your option) any later version. -# http://www.gnu.org/licenses/ -#***************************************************************************** - -import contextlib -import os -import sys -import re - -try: - # Python 3 - import urllib.request as urllib - import urllib.parse as urlparse -except ImportError: - import urllib - import urlparse - -try: - from sage.env import SAGE_ROOT, SAGE_DISTFILES -except ImportError: - # Sage is not yet installed - SAGE_ROOT = os.environ['SAGE_ROOT'] - SAGE_DISTFILES = os.environ.get('SAGE_DISTFILES', - os.path.join(SAGE_ROOT, 'upstream')) - -try: - # we want to assume that SAGE_DISTFILES is an actual - # directory for the remainder of this script - os.mkdir(SAGE_DISTFILES) -except OSError: - pass - - -def printflush(x): - """Print ``x`` and flush output""" - print(x) - sys.stdout.flush() - - -def http_error_default(url, fp, errcode, errmsg, headers): - """ - Callback for the URLopener to raise an exception on HTTP errors - """ - fp.close() - raise IOError(errcode, errmsg, url) - - -class ProgressBar(object): - """ - Progress bar as urllib reporthook - """ - - def __init__(self, length=70): - self.length = length - self.progress = 0 - self.stream = sys.stderr - - def start(self): - sys.stdout.flush() # make sure to not interleave stdout/stderr - self.stream.write('[') - self.stream.flush() - - def __call__(self, chunks_so_far, chunk_size, total_size): - if total_size == -1: # we do not know size - n = 0 if chunks_so_far == 0 else self.length // 2 - else: - n = chunks_so_far * chunk_size * self.length // total_size - if n > self.length: - # If there is a Content-Length, this will be sent as the last progress - return - # n ranges from 0 to length*total (exclude), so we'll print at most length dots - if n >= self.progress: - self.stream.write('.' * (n-self.progress)) - self.stream.flush() - self.progress = n - - def stop(self): - missing = '.' * (self.length - self.progress) - self.stream.write(missing + ']\n') - self.stream.flush() - - def error_stop(self): - missing = 'x' * (self.length - self.progress) - self.stream.write(missing + ']\n') - self.stream.flush() - - -def http_download(url, destination=None, progress=True, ignore_errors=False): - """ - Download via HTTP - - This should work for FTP as well but, in fact, hangs on python < - 3.4, see http://bugs.python.org/issue16270 - - INPUT: - - - ``url`` -- string. The URL to download. - - - ``destination`` -- string or ``None`` (default). The destination - file name to save to. If not specified, the file is written to - stdout. - - - ``progress`` -- boolean (default: ``True``). Whether to print a - progress bar to stderr. - - - ``ignore_errors`` -- boolean (default: ``False``). Catch network - errors (a message is still printed to stdout). - """ - if destination is None: - destination = '/dev/stdout' - opener = urllib.FancyURLopener() - opener.http_error_default = http_error_default - if progress: - progress_bar = ProgressBar() - progress_bar.start() - try: - filename, info = opener.retrieve(url, destination, progress_bar) - except IOError as err: - progress_bar.error_stop() - printflush(err) - if not ignore_errors: - raise - else: - progress_bar.stop() - else: - filename, info = opener.retrieve(url, destination) - - -class MirrorList(object): - - URL = 'http://www.sagemath.org/mirror_list' - - MAXAGE = 24*60*60 # seconds - - def __init__(self, verbose=True): - """ - If `verbose` is False, don't print messages along the way. - This is needed to produce the appropriate output for - `sage-download-file --print-fastest-mirror`. - """ - self.filename = os.path.join(SAGE_DISTFILES, 'mirror_list') - self.verbose = verbose - if self.must_refresh(): - if self.verbose: - printflush('Downloading the Sage mirror list') - with contextlib.closing(urllib.urlopen(self.URL)) as f: - mirror_list = f.read().decode("ascii") - self.mirrors = self._load(mirror_list) - self._rank_mirrors() - self._save() - else: - self.mirrors = self._load() - - def _load(self, mirror_list = None): - """ - Load and return `mirror_list` (defaults to the one on disk) as - a list of strings - """ - if mirror_list is None: - with open(self.filename, 'rt') as f: - mirror_list = f.read() - import ast - return ast.literal_eval(mirror_list) - - def _save(self): - """ - Save the mirror list for (short-term) future use. - """ - with open(self.filename, 'wt') as f: - f.write(repr(self.mirrors)) - - def _port_of_mirror(self, mirror): - if mirror.startswith('http://'): - return 80 - if mirror.startswith('https://'): - return 443 - if mirror.startswith('ftp://'): - return 21 - - def _rank_mirrors(self): - """ - Sort the mirrors by speed, fastest being first - - This method is used by the YUM fastestmirror plugin - """ - timed_mirrors = [] - import time, socket - if self.verbose: - printflush('Searching fastest mirror') - timeout = 1 - for mirror in self.mirrors: - if not mirror.startswith('http'): - # we currently cannot handle ftp:// - continue - port = self._port_of_mirror(mirror) - mirror_hostname = urlparse.urlsplit(mirror).netloc - time_before = time.time() - try: - sock = socket.create_connection((mirror_hostname, port), timeout) - sock.close() - except (IOError, socket.error, socket.timeout): - continue - result = time.time() - time_before - result_ms = int(1000 * result) - if self.verbose: - printflush(str(result_ms).rjust(5) + 'ms: ' + mirror) - timed_mirrors.append((result, mirror)) - timed_mirrors.sort() - self.mirrors = [m[1] for m in timed_mirrors] - if self.verbose: - printflush('Fastest mirror: ' + self.fastest) - - @property - def fastest(self): - return self.mirrors[0] - - def age(self): - """ - Return the age of the cached mirror list in seconds - """ - import time - mtime = os.path.getmtime(self.filename) - now = time.mktime(time.localtime()) - return now - mtime - - def must_refresh(self): - """ - Return whether we must download the mirror list. - - If and only if this method returns ``False`` is it admissible - to use the cached mirror list. - """ - if not os.path.exists(self.filename): - return True - return self.age() > self.MAXAGE - - def __iter__(self): - """ - Iterate through the list of mirrors. - - This is the main entry point into the mirror list. Every - script should just use this function to try mirrors in order - of preference. This will not just yield the official mirrors, - but also urls for packages that are currently being tested. - """ - try: - yield os.environ['SAGE_SERVER'] - except KeyError: - pass - for mirror in self.mirrors: - yield mirror - # If all else fails: Try the packages we host ourselves - yield 'http://sagepad.org/' - - - -class ChecksumError(Exception): - pass - -class FileNotMirroredError(Exception): - pass - - -class Package(object): - - def __init__(self, package_name): - self.name = package_name - self._init_checksum() - self._init_version() - - @classmethod - def all(cls): - base = os.path.join(SAGE_ROOT, 'build', 'pkgs') - for subdir in os.listdir(base): - path = os.path.join(base, subdir) - if not os.path.isdir(path): - continue - yield cls(subdir) - - @property - def path(self): - return os.path.join(SAGE_ROOT, 'build', 'pkgs', self.name) - - def _init_checksum(self): - """ - Load the checksums from the appropriate ``checksums.ini`` file - """ - checksums_ini = os.path.join(self.path, 'checksums.ini') - assignment = re.compile('(?P[a-zA-Z0-9]*)=(?P.*)') - result = dict() - with open(checksums_ini, 'rt') as f: - for line in f.readlines(): - match = assignment.match(line) - if match is None: - continue - var, value = match.groups() - result[var] = value - self.md5 = result['md5'] - self.sha1 = result['sha1'] - self.cksum = result['cksum'] - self.sha1 = result['sha1'] - self.tarball_pattern = result['tarball'] - - VERSION_PATCHLEVEL = re.compile('(?P.*)\.p(?P[0-9]+)') - - def _init_version(self): - with open(os.path.join(self.path, 'package-version.txt')) as f: - package_version = f.read().strip() - match = self.VERSION_PATCHLEVEL.match(package_version) - if match is None: - self.version = package_version - self.patchlevel = -1 - else: - self.version = match.group('version') - self.patchlevel = match.group('patchlevel') - self.tarball = self.tarball_pattern.replace('VERSION', self.version) - - -class Tarball(object): - - def __init__(self, tarball_name): - """ - A (third-party downloadable) tarball - - INPUT: - - - ``name`` - string. The full filename (``foo-1.3.tar.bz2``) - of a tarball on the Sage mirror network. - """ - self.filename = tarball_name - self.package = None - for pkg in Package.all(): - if pkg.tarball == tarball_name: - self.package = pkg - if self.package is None: - raise ValueError('tarball {0} is not referenced by any Sage package' - .format(tarball_name)) - - @property - def upstream_fqn(self): - """ - The fully-qualified (including directory) file name in the upstream directory. - """ - return os.path.join(SAGE_DISTFILES, self.filename) - - def _compute_hash(self, algorithm): - with open(self.upstream_fqn, 'rb') as f: - while True: - buf = f.read(0x100000) - if not buf: - break - algorithm.update(buf) - return algorithm.hexdigest() - - def _compute_sha1(self): - import hashlib - return self._compute_hash(hashlib.sha1()) - - def _compute_md5(self): - import hashlib - return self._compute_md5(hashlib.md5()) - - def checksum_verifies(self): - """ - Test whether the checksum of the downloaded file is correct. - """ - sha1 = self._compute_sha1() - return sha1 == self.package.sha1 - - def download(self): - """ - Download the tarball to the upstream directory. - """ - destination = os.path.join(SAGE_DISTFILES, self.filename) - if os.path.isfile(destination): - if self.checksum_verifies(): - print('Using cached file {destination}'.format(destination=destination)) - return - else: - # Garbage in the upstream directory? Delete and re-download - print('Invalid checksum for cached file {destination}, deleting' - .format(destination=destination)) - os.remove(destination) - successful_download = False - print('Attempting to download package {0} from mirrors'.format(self.filename)) - for mirror in MirrorList(): - url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename]) - printflush(url) - try: - http_download(url, self.upstream_fqn) - successful_download = True - break - except IOError: - pass # mirror doesn't have file for whatever reason... - if not successful_download: - raise FileNotMirroredError('tarball does not exist on mirror') - if not self.checksum_verifies(): - raise ChecksumError('checksum does not match') - - def save_as(self, destination): - import shutil - shutil.copy(self.upstream_fqn, destination) - - -usage = \ -""" -USAGE: - - sage-download-file --print-fastest-mirror - -Print out the fastest mirror. All further arguments are ignored in -that case. - - sage-download-file [--quiet] url-or-tarball [destination] - -The single mandatory argument can be a http:// url or a tarball -filename. In the latter case, the tarball is downloaded from the -mirror network and its checksum is verified. - -If the destination is not specified: -* a url will be downloaded and the content written to stdout -* a tarball will be saved under {SAGE_DISTFILES} -""".format(SAGE_DISTFILES=SAGE_DISTFILES) - -if __name__ == '__main__': - progress = True - url = None - destination = None - for arg in sys.argv[1:]: - if arg.startswith('--print-fastest-mirror'): - print(MirrorList(verbose=False).fastest) - sys.exit(0) - if arg.startswith('--quiet'): - progress = False - continue - if url is None: - url = arg - continue - if destination is None: - destination = arg - continue - raise ValueError('too many arguments') - if url is None: - print(usage) - sys.exit(1) - if url.startswith('http://') or url.startswith('https://') or url.startswith('ftp://'): - http_download(url, destination, progress=progress, ignore_errors=True) - else: - # url is a tarball name - tarball = Tarball(url) - tarball.download() - if destination is not None: - tarball.save_as(destination) diff --git a/src/sage_bootstrap/bin/sage-download-file b/src/sage_bootstrap/bin/sage-download-file new file mode 100755 index 00000000000..450b8843593 --- /dev/null +++ b/src/sage_bootstrap/bin/sage-download-file @@ -0,0 +1,11 @@ +#!/usr/bin/env python + +try: + import sage_bootstrap +except ImportError: + import os, sys + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) + import sage_bootstrap + +from sage_bootstrap.cmdline import SageDownloadFileApplication +SageDownloadFileApplication().run() diff --git a/src/sage_bootstrap/sage_bootstrap/cmdline.py b/src/sage_bootstrap/sage_bootstrap/cmdline.py index 98a4db6bf37..4d12d193d0a 100644 --- a/src/sage_bootstrap/sage_bootstrap/cmdline.py +++ b/src/sage_bootstrap/sage_bootstrap/cmdline.py @@ -18,9 +18,14 @@ import os import sys +from textwrap import dedent import logging log = logging.getLogger() +from sage_bootstrap.env import SAGE_DISTFILES +from sage_bootstrap.download import Download +from sage_bootstrap.mirror_list import MirrorList +from sage_bootstrap.tarball import Tarball class CmdlineSubcommands(object): @@ -35,7 +40,6 @@ def __init__(self, argv=None): self.extra_args = argv[2:] def print_help(self): - from textwrap import dedent print(dedent(self.__doc__).lstrip()) def run(self): @@ -85,3 +89,54 @@ def run_tarball(self, package_name): """ pass + +class SageDownloadFileApplication(object): + """ + USAGE: + + sage-download-file --print-fastest-mirror + + Print out the fastest mirror. All further arguments are ignored in + that case. + + sage-download-file [--quiet] url-or-tarball [destination] + + The single mandatory argument can be a http:// url or a tarball + filename. In the latter case, the tarball is downloaded from the + mirror network and its checksum is verified. + + If the destination is not specified: + * a url will be downloaded and the content written to stdout + * a tarball will be saved under {SAGE_DISTFILES} + """ + + def run(self): + progress = True + url = None + destination = None + for arg in sys.argv[1:]: + if arg.startswith('--print-fastest-mirror'): + print(MirrorList().fastest) + sys.exit(0) + if arg.startswith('--quiet'): + progress = False + continue + if url is None: + url = arg + continue + if destination is None: + destination = arg + continue + raise ValueError('too many arguments') + if url is None: + print(dedent(self.__doc__.format(SAGE_DISTFILES=SAGE_DISTFILES))) + sys.exit(1) + if url.startswith('http://') or url.startswith('https://') or url.startswith('ftp://'): + Download(url, destination, progress=progress, ignore_errors=True).run() + else: + # url is a tarball name + tarball = Tarball(url) + tarball.download() + if destination is not None: + tarball.save_as(destination) + diff --git a/src/sage_bootstrap/sage_bootstrap/compat.py b/src/sage_bootstrap/sage_bootstrap/compat.py new file mode 100644 index 00000000000..e5c485a1bd7 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/compat.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +""" +Python 2/3 compatibility utils +""" + + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +try: + # Python 3 + import urllib.request as urllib + import urllib.parse as urlparse +except ImportError: + import urllib + import urlparse + + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO diff --git a/src/sage_bootstrap/sage_bootstrap/download.py b/src/sage_bootstrap/sage_bootstrap/download.py new file mode 100644 index 00000000000..0ccd8d6f1d3 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/download.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +""" +Download files from the internet +""" + + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import sys +import logging +log = logging.getLogger() + +from sage_bootstrap.stdio import flush +from sage_bootstrap.compat import urllib + + +class ProgressBar(object): + """ + Progress bar as urllib reporthook + """ + + def __init__(self, stream, length=70): + self.length = length + self.progress = 0 + self.stream = stream + + def start(self): + flush() # make sure to not interleave stdout/stderr + self.stream.write('[') + self.stream.flush() + + def __call__(self, chunks_so_far, chunk_size, total_size): + if total_size == -1: # we do not know size + n = 0 if chunks_so_far == 0 else self.length // 2 + else: + n = chunks_so_far * chunk_size * self.length // total_size + if n > self.length: + # If there is a Content-Length, this will be sent as the last progress + return + # n ranges from 0 to length*total (exclude), so we'll print at most length dots + if n >= self.progress: + self.stream.write('.' * (n-self.progress)) + self.stream.flush() + self.progress = n + + def stop(self): + missing = '.' * (self.length - self.progress) + self.stream.write(missing + ']\n') + self.stream.flush() + + def error_stop(self): + missing = 'x' * (self.length - self.progress) + self.stream.write(missing + ']\n') + self.stream.flush() + + + + +class Download(object): + """ + Download URL + + Right now, only via HTTP + + This should work for FTP as well but, in fact, hangs on python < + 3.4, see http://bugs.python.org/issue16270 + + INPUT: + + - ``url`` -- string. The URL to download. + + - ``destination`` -- string or ``None`` (default). The destination + file name to save to. If not specified, the file is written to + stdout. + + - ``progress`` -- boolean (default: ``True``). Whether to print a + progress bar to stderr. For testing, this can also be a stream + to which the progress bar is being sent. + + - ``ignore_errors`` -- boolean (default: ``False``). Catch network + errors (a message is still being logged). + """ + + def __init__(self, url, destination=None, progress=True, ignore_errors=False): + self.url = url + self.destination = destination or '/dev/stdout' + self.progress = (progress is not False) + self.progress_stream = sys.stderr if isinstance(progress, bool) else progress + self.ignore_errors = ignore_errors + + def http_error_default(self, url, fp, errcode, errmsg, headers): + """ + Callback for the URLopener to raise an exception on HTTP errors + """ + fp.close() + raise IOError(errcode, errmsg, url) + + def start_progress_bar(self): + if self.progress: + self.progress_bar = ProgressBar(self.progress_stream) + self.progress_bar.start() + + def success_progress_bar(self): + if self.progress: + self.progress_bar.stop() + + def error_progress_bar(self): + if self.progress: + self.progress_bar.error_stop() + + def run(self): + opener = urllib.FancyURLopener() + opener.http_error_default = self.http_error_default + self.start_progress_bar() + try: + if self.progress: + filename, info = opener.retrieve( + self.url, self.destination, self.progress_bar) + else: + filename, info = opener.retrieve( + self.url, self.destination) + except IOError as err: + self.error_progress_bar() + log.error(err) + if not self.ignore_errors: + raise + self.success_progress_bar() diff --git a/src/sage_bootstrap/sage_bootstrap/env.py b/src/sage_bootstrap/sage_bootstrap/env.py index ae9c9417e16..b0fc3e2679a 100644 --- a/src/sage_bootstrap/sage_bootstrap/env.py +++ b/src/sage_bootstrap/sage_bootstrap/env.py @@ -7,6 +7,7 @@ * ``SAGE_ROOT`` * ``SAGE_SRC`` +* ``SAGE_DISTFILES`` """ @@ -23,17 +24,25 @@ import os - -try: - SAGE_SRC = os.environ['SAGE_SRC'] -except KeyError: - SAGE_SRC = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) - try: SAGE_ROOT = os.environ['SAGE_ROOT'] except KeyError: - SAGE_ROOT = os.path.dirname(SAGE_SRC) + SAGE_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))))) +SAGE_SRC = os.environ.get('SAGE_SRC', + os.path.join(SAGE_ROOT, 'src')) +SAGE_DISTFILES = os.environ.get('SAGE_DISTFILES', + os.path.join(SAGE_ROOT, 'upstream')) + + +assert os.path.isfile(os.path.join(SAGE_ROOT, 'configure.ac')), SAGE_ROOT +assert os.path.isfile(os.path.join(SAGE_SRC, 'sage_bootstrap', 'setup.py')), SAGE_SRC + +try: + # SAGE_DISTFILES does not exist in a fresh git clone + os.mkdir(SAGE_DISTFILES) +except OSError: + pass -assert os.path.isfile(os.path.join(SAGE_ROOT, 'configure.ac')) -assert os.path.isfile(os.path.join(SAGE_SRC, 'sage_bootstrap', 'setup.py')) +assert os.path.isdir(SAGE_DISTFILES) diff --git a/src/sage_bootstrap/sage_bootstrap/mirror_list.py b/src/sage_bootstrap/sage_bootstrap/mirror_list.py new file mode 100644 index 00000000000..5d38dc02e7d --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/mirror_list.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +""" +Access the List of Sage Download Mirrors +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import os +import contextlib +import logging +log = logging.getLogger() + +from sage_bootstrap.compat import urllib, urlparse +from sage_bootstrap.env import SAGE_DISTFILES + + +class MirrorList(object): + + URL = 'http://www.sagemath.org/mirror_list' + + MAXAGE = 24*60*60 # seconds + + def __init__(self): + self.filename = os.path.join(SAGE_DISTFILES, 'mirror_list') + if self.must_refresh(): + log.info('Downloading the Sage mirror list') + try: + with contextlib.closing(urllib.urlopen(self.URL)) as f: + mirror_list = f.read().decode("ascii") + except IOError: + log.critical('Downloading the mirror list failed') + self.mirrors = self._load() + else: + self.mirrors = self._load(mirror_list) + self._rank_mirrors() + self._save() + else: + self.mirrors = self._load() + + def _load(self, mirror_list=None): + """ + Load and return `mirror_list` (defaults to the one on disk) as + a list of strings + """ + if mirror_list is None: + try: + with open(self.filename, 'rt') as f: + mirror_list = f.read() + except IOError: + log.critical('Failed to load the cached mirror list') + return [] + import ast + return ast.literal_eval(mirror_list) + + def _save(self): + """ + Save the mirror list for (short-term) future use. + """ + with open(self.filename, 'wt') as f: + f.write(repr(self.mirrors)) + + def _port_of_mirror(self, mirror): + if mirror.startswith('http://'): + return 80 + if mirror.startswith('https://'): + return 443 + if mirror.startswith('ftp://'): + return 21 + + def _rank_mirrors(self): + """ + Sort the mirrors by speed, fastest being first + + This method is used by the YUM fastestmirror plugin + """ + timed_mirrors = [] + import time, socket + log.info('Searching fastest mirror') + timeout = 1 + for mirror in self.mirrors: + if not mirror.startswith('http'): + log.debug('we currently can only handle http, got %s', mirror) + continue + port = self._port_of_mirror(mirror) + mirror_hostname = urlparse.urlsplit(mirror).netloc + time_before = time.time() + try: + sock = socket.create_connection((mirror_hostname, port), timeout) + sock.close() + except (IOError, socket.error, socket.timeout) as err: + log.warning(str(err).strip() + ': ' + mirror) + continue + result = time.time() - time_before + result_ms = int(1000 * result) + log.info(str(result_ms).rjust(5) + 'ms: ' + mirror) + timed_mirrors.append((result, mirror)) + if len(timed_mirrors) == 0: + # We cannot reach any mirror directly, most likely firewall issue + if 'http_proxy' not in os.environ: + log.error('Could not reach any mirror directly and no proxy set') + raise RuntimeError('no internet connection') + log.info('Cannot time mirrors via proxy, using default order') + else: + timed_mirrors.sort() + self.mirrors = [m[1] for m in timed_mirrors] + log.info('Fastest mirror: ' + self.fastest) + + @property + def fastest(self): + return next(iter(self)) + + def age(self): + """ + Return the age of the cached mirror list in seconds + """ + import time + mtime = os.path.getmtime(self.filename) + now = time.mktime(time.localtime()) + return now - mtime + + def must_refresh(self): + """ + Return whether we must download the mirror list. + + If and only if this method returns ``False`` is it admissible + to use the cached mirror list. + """ + if not os.path.exists(self.filename): + return True + return self.age() > self.MAXAGE + + def __iter__(self): + """ + Iterate through the list of mirrors. + + This is the main entry point into the mirror list. Every + script should just use this function to try mirrors in order + of preference. This will not just yield the official mirrors, + but also urls for packages that are currently being tested. + """ + try: + yield os.environ['SAGE_SERVER'] + except KeyError: + pass + for mirror in self.mirrors: + yield mirror + # If all else fails: Try the packages we host ourselves + yield 'http://sagepad.org/' + + diff --git a/src/sage_bootstrap/sage_bootstrap/package.py b/src/sage_bootstrap/sage_bootstrap/package.py new file mode 100644 index 00000000000..3ab80f98dd3 --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/package.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" +Sage Packages +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import re +import os + +import logging +log = logging.getLogger() + +from sage_bootstrap.env import SAGE_ROOT + + + +class Package(object): + + def __init__(self, package_name): + self.name = package_name + self._init_checksum() + self._init_version() + + def __eq__(self, other): + return self.tarball == other.tarball + + @classmethod + def all(cls): + base = os.path.join(SAGE_ROOT, 'build', 'pkgs') + for subdir in os.listdir(base): + path = os.path.join(base, subdir) + if not os.path.isdir(path): + continue + yield cls(subdir) + + @property + def path(self): + return os.path.join(SAGE_ROOT, 'build', 'pkgs', self.name) + + def _init_checksum(self): + """ + Load the checksums from the appropriate ``checksums.ini`` file + """ + checksums_ini = os.path.join(self.path, 'checksums.ini') + assignment = re.compile('(?P[a-zA-Z0-9]*)=(?P.*)') + result = dict() + with open(checksums_ini, 'rt') as f: + for line in f.readlines(): + match = assignment.match(line) + if match is None: + continue + var, value = match.groups() + result[var] = value + self.md5 = result['md5'] + self.sha1 = result['sha1'] + self.cksum = result['cksum'] + self.sha1 = result['sha1'] + self.tarball_pattern = result['tarball'] + + VERSION_PATCHLEVEL = re.compile('(?P.*)\.p(?P[0-9]+)') + + def _init_version(self): + with open(os.path.join(self.path, 'package-version.txt')) as f: + package_version = f.read().strip() + match = self.VERSION_PATCHLEVEL.match(package_version) + if match is None: + self.version = package_version + self.patchlevel = -1 + else: + self.version = match.group('version') + self.patchlevel = match.group('patchlevel') + self.tarball = self.tarball_pattern.replace('VERSION', self.version) + + diff --git a/src/sage_bootstrap/sage_bootstrap/stdio.py b/src/sage_bootstrap/sage_bootstrap/stdio.py index e1a863573e6..322043e0aeb 100644 --- a/src/sage_bootstrap/sage_bootstrap/stdio.py +++ b/src/sage_bootstrap/sage_bootstrap/stdio.py @@ -41,3 +41,8 @@ def __getattr__(self, attr): def init_streams(config): if not config.interactive: sys.stdout = UnbufferedStream(REAL_STDOUT) + + +def flush(): + REAL_STDOUT.flush() + REAL_STDERR.flush() diff --git a/src/sage_bootstrap/sage_bootstrap/tarball.py b/src/sage_bootstrap/sage_bootstrap/tarball.py new file mode 100644 index 00000000000..04f4be7d45e --- /dev/null +++ b/src/sage_bootstrap/sage_bootstrap/tarball.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +""" +Third-Party Tarballs +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import os +import logging +log = logging.getLogger() + +from sage_bootstrap.env import SAGE_DISTFILES +from sage_bootstrap.download import Download +from sage_bootstrap.package import Package +from sage_bootstrap.mirror_list import MirrorList + + +class ChecksumError(Exception): + pass + + +class FileNotMirroredError(Exception): + pass + + +class Tarball(object): + + def __init__(self, tarball_name): + """ + A (third-party downloadable) tarball + + INPUT: + + - ``name`` - string. The full filename (``foo-1.3.tar.bz2``) + of a tarball on the Sage mirror network. + """ + self.filename = tarball_name + self.package = None + for pkg in Package.all(): + if pkg.tarball == tarball_name: + self.package = pkg + if self.package is None: + error = 'tarball {0} is not referenced by any Sage package'.format(tarball_name) + log.error(error) + raise ValueError(error) + + @property + def upstream_fqn(self): + """ + The fully-qualified (including directory) file name in the upstream directory. + """ + return os.path.join(SAGE_DISTFILES, self.filename) + + def _compute_hash(self, algorithm): + with open(self.upstream_fqn, 'rb') as f: + while True: + buf = f.read(0x100000) + if not buf: + break + algorithm.update(buf) + return algorithm.hexdigest() + + def _compute_sha1(self): + import hashlib + return self._compute_hash(hashlib.sha1()) + + def _compute_md5(self): + import hashlib + return self._compute_md5(hashlib.md5()) + + def checksum_verifies(self): + """ + Test whether the checksum of the downloaded file is correct. + """ + sha1 = self._compute_sha1() + return sha1 == self.package.sha1 + + def download(self): + """ + Download the tarball to the upstream directory. + """ + destination = os.path.join(SAGE_DISTFILES, self.filename) + if os.path.isfile(destination): + if self.checksum_verifies(): + log.info('Using cached file {destination}'.format(destination=destination)) + return + else: + # Garbage in the upstream directory? Delete and re-download + log.info('Invalid checksum for cached file {destination}, deleting' + .format(destination=destination)) + os.remove(destination) + successful_download = False + log.info('Attempting to download package {0} from mirrors'.format(self.filename)) + for mirror in MirrorList(): + url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename]) + log.info(url) + try: + Download(url, self.upstream_fqn).run() + successful_download = True + break + except IOError: + log.debug('File not on mirror') + if not successful_download: + raise FileNotMirroredError('tarball does not exist on mirror network') + if not self.checksum_verifies(): + raise ChecksumError('checksum does not match') + + def save_as(self, destination): + import shutil + shutil.copy(self.upstream_fqn, destination) + diff --git a/src/sage_bootstrap/setup.py b/src/sage_bootstrap/setup.py index ce8727c98f5..a783496b951 100755 --- a/src/sage_bootstrap/setup.py +++ b/src/sage_bootstrap/setup.py @@ -8,7 +8,7 @@ author='Volker Braun', author_email='vbraun.name@gmail.com', packages=['sage_bootstrap'], - scripts=['sage-pkg'], + scripts=['bin/sage-pkg'], version='1.0', url='https://www.sagemath.org', ) diff --git a/src/sage_bootstrap/test/capture.py b/src/sage_bootstrap/test/capture.py new file mode 100644 index 00000000000..4b9c4e17f72 --- /dev/null +++ b/src/sage_bootstrap/test/capture.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +""" +Capture output for testing purposes +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import sys +import contextlib +import logging +log = logging.getLogger() + +from sage_bootstrap.compat import StringIO + + +class LogCaptureHandler(logging.Handler): + + def __init__(self, log_capture): + self.records = log_capture.records + logging.Handler.__init__(self) + + def emit(self, record): + self.records.append(record) + + +class CapturedLog(object): + + def __init__(self): + self.records = [] + + def __enter__(self): + self.old_level = log.level + self.old_handlers = log.handlers + log.level = logging.INFO + log.handlers = [LogCaptureHandler(self)] + return self + + def __exit__(self, type, value, traceback): + log.level = self.old_level + log.handlers = self.old_handlers + + def messages(self): + return tuple((rec.levelname, rec.getMessage()) for rec in self.records) + + +@contextlib.contextmanager +def CapturedOutput(): + new_out, new_err = StringIO(), StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = new_out, new_err + yield sys.stdout, sys.stderr + finally: + sys.stdout, sys.stderr = old_out, old_err diff --git a/src/sage_bootstrap/test/test_download.py b/src/sage_bootstrap/test/test_download.py new file mode 100644 index 00000000000..135583a89d8 --- /dev/null +++ b/src/sage_bootstrap/test/test_download.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" +Test downloading files +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import unittest +import tempfile +from textwrap import dedent + + +from .capture import CapturedLog +from sage_bootstrap.download import Download +from sage_bootstrap.mirror_list import MirrorList +from sage_bootstrap.compat import StringIO + + +class DownloadTestCase(unittest.TestCase): + + def test_download_mirror_list(self): + tmp = tempfile.NamedTemporaryFile() + tmp.close() + progress = StringIO() + Download(MirrorList.URL, tmp.name, progress=progress).run() + self.assertEqual(progress.getvalue(), + '[......................................................................]\n') + with open(tmp.name, 'r') as f: + content = f.read() + self.assertTrue(content.startswith('# Sage Mirror List')) + + def test_error(self): + URL = 'http://www.sagemath.org/sage_bootstrap/this_url_does_not_exist' + progress = StringIO() + download = Download(URL, progress=progress) + log = CapturedLog() + def action(): + with log: + download.run() + self.assertRaises(IOError, action) + self.assertIsNotFoundError(log.messages()) + self.assertEqual(progress.getvalue(), + '[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]\n') + + def test_ignore_errors(self): + URL = 'http://www.sagemath.org/sage_bootstrap/this_url_does_not_exist' + with CapturedLog() as log: + Download(URL, progress=False, ignore_errors=True).run() + self.assertIsNotFoundError(log.messages()) + + def assertIsNotFoundError(self, messages): + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0][0], 'ERROR') + self.assertTrue(messages[0][1].startswith('[Errno')) + self.assertTrue(messages[0][1].endswith( + "Not Found: '//www.sagemath.org/sage_bootstrap/this_url_does_not_exist'")) + + diff --git a/src/sage_bootstrap/test/test_mirror_list.py b/src/sage_bootstrap/test/test_mirror_list.py new file mode 100644 index 00000000000..ddd8e21ca79 --- /dev/null +++ b/src/sage_bootstrap/test/test_mirror_list.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +""" +Test downloading files +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import unittest +import tempfile + +from .capture import CapturedLog +from sage_bootstrap.mirror_list import MirrorList + + + +class MirrorListTestCase(unittest.TestCase): + + def test_mirror_list(self): + with CapturedLog() as log: + ml = MirrorList() + msg = log.messages() + if len(msg) > 0: + self.assertEqual(msg[0], ('INFO', 'Downloading the Sage mirror list')) + self.assertTrue(len(ml.mirrors) >= 0) + self.assertTrue(ml.fastest.startswith('http://')) diff --git a/src/sage_bootstrap/test/test_package.py b/src/sage_bootstrap/test/test_package.py new file mode 100644 index 00000000000..bcbd9cea53b --- /dev/null +++ b/src/sage_bootstrap/test/test_package.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +Test Sage Package Handling +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + + +import unittest +from sage_bootstrap.package import Package + + +class PackageTestCase(unittest.TestCase): + + def test_package(self): + pkg = Package('pari') + self.assertTrue(pkg.name, 'pari') + self.assertTrue(pkg.path.endswith('build/pkgs/pari')) + self.assertEqual(pkg.tarball_pattern, 'pari-VERSION.tar.gz') + self.assertTrue(pkg.tarball.startswith('pari-') and + pkg.tarball.endswith('.tar.gz')) + + + def test_all(self): + pari = Package('pari') + self.assertTrue(pari in Package.all()) + diff --git a/src/sage_bootstrap/test/test_tarball.py b/src/sage_bootstrap/test/test_tarball.py new file mode 100644 index 00000000000..663bef8f486 --- /dev/null +++ b/src/sage_bootstrap/test/test_tarball.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +""" +Test Sage Third-Party Tarball Handling +""" + +#***************************************************************************** +# Copyright (C) 2015 Volker Braun +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# http://www.gnu.org/licenses/ +#***************************************************************************** + +import unittest + +from sage_bootstrap.package import Package +from sage_bootstrap.tarball import Tarball + +from .capture import CapturedLog, CapturedOutput + + + +class TarballTestCase(unittest.TestCase): + + def test_tarball(self): + pkg = Package('configure') + tarball = Tarball(pkg.tarball) + self.assertEqual(pkg, tarball.package) + with CapturedOutput() as (stdout, stderr): + with CapturedLog() as log: + tarball.download() + self.assertEqual(stdout.getvalue(), '') + self.assertTrue(tarball.checksum_verifies()) + + def test_checksum(self): + pkg = Package('configure') + tarball = Tarball(pkg.tarball) + self.assertTrue(tarball.checksum_verifies()) + with open(tarball.upstream_fqn, 'w') as f: + f.write('foobar') + self.assertFalse(tarball.checksum_verifies()) + with CapturedOutput() as (stdout, stderr): + with CapturedLog() as log: + tarball.download() + msg = log.messages() + self.assertTrue( + ('INFO', 'Attempting to download package {0} from mirrors'.format(pkg.tarball)) in msg) + self.assertEqual(stdout.getvalue(), '') + self.assertEqual(stderr.getvalue(), + '[......................................................................]\n') + self.assertTrue(tarball.checksum_verifies()) From 0acf070e6d86df8f5dcc62f0751002631cbce9d8 Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Sun, 21 Jun 2015 12:52:11 +0200 Subject: [PATCH 04/13] initial implementation of the sage-pkg tool --- src/bin/sage-env | 2 +- src/sage_bootstrap/sage_bootstrap/cmdline.py | 38 ++++++++++++++++++-- src/sage_bootstrap/sage_bootstrap/logger.py | 24 +++++++------ src/sage_bootstrap/test/test_logger.py | 26 +++++++------- 4 files changed, 62 insertions(+), 28 deletions(-) diff --git a/src/bin/sage-env b/src/bin/sage-env index 6075ca4591e..b391de338a1 100644 --- a/src/bin/sage-env +++ b/src/bin/sage-env @@ -258,7 +258,7 @@ if [ -z "${SAGE_ORIG_PATH_SET}" ]; then SAGE_ORIG_PATH=$PATH && export SAGE_ORIG_PATH SAGE_ORIG_PATH_SET=True && export SAGE_ORIG_PATH_SET fi -export PATH="$SAGE_SRC/bin:$SAGE_LOCAL/bin:$PATH" +export PATH="$SAGE_SRC/sage_bootstrap/bin:$SAGE_SRC/bin:$SAGE_LOCAL/bin:$PATH" # We offer a toolchain option, so if $SAGE_LOCAL/toolchain/toolchain-env exists source it. # Since the user might do something crazy we do not do any checks, but hope for the best. diff --git a/src/sage_bootstrap/sage_bootstrap/cmdline.py b/src/sage_bootstrap/sage_bootstrap/cmdline.py index 4d12d193d0a..4a1e053a056 100644 --- a/src/sage_bootstrap/sage_bootstrap/cmdline.py +++ b/src/sage_bootstrap/sage_bootstrap/cmdline.py @@ -23,6 +23,7 @@ log = logging.getLogger() from sage_bootstrap.env import SAGE_DISTFILES +from sage_bootstrap.package import Package from sage_bootstrap.download import Download from sage_bootstrap.mirror_list import MirrorList from sage_bootstrap.tarball import Tarball @@ -41,6 +42,15 @@ def __init__(self, argv=None): def print_help(self): print(dedent(self.__doc__).lstrip()) + print('Usage:') + for method in sorted(dir(self)): + if not method.startswith('run_'): + continue + doc = dedent(getattr(self, method).__doc__).lstrip().splitlines() + print('') + print('* ' + doc[0]) + for line in doc[1:]: + print(' ' + line) def run(self): try: @@ -67,6 +77,11 @@ class SagePkgApplication(CmdlineSubcommands): def run_config(self): """ Print the configuration + + $ sage-pkg config + Configuration: + * log = info + * interactive = True """ from sage_bootstrap.config import Configuration print(Configuration()) @@ -74,20 +89,37 @@ def run_config(self): def run_list(self): """ Print a list of all available packages + + $ sage-pkg list | sort + 4ti2 + arb + atlas + autotools + [...] + zn_poly """ - pass + for pkg in Package.all(): + print(pkg.name) def run_name(self, tarball_filename): """ Find the package name given a tarball filename + + $ sage-pkg name pari-2.8-1564-gdeac36e.tar.gz + pari """ - pass + tarball = Tarball(os.path.basename(tarball_filename)) + print(tarball.package.name) def run_tarball(self, package_name): """ Find the tarball filename given a package name + + $ sage-pkg tarball pari + pari-2.8-1564-gdeac36e.tar.gz """ - pass + package = Package(package_name) + print(package.tarball) class SageDownloadFileApplication(object): diff --git a/src/sage_bootstrap/sage_bootstrap/logger.py b/src/sage_bootstrap/sage_bootstrap/logger.py index 3b6ac5d56b4..894aecefd8c 100644 --- a/src/sage_bootstrap/sage_bootstrap/logger.py +++ b/src/sage_bootstrap/sage_bootstrap/logger.py @@ -26,7 +26,7 @@ default_formatter = logging.Formatter( - '%(levelname)s [%(module)s.%(funcName)s:%(lineno)s]: %(message)s') + '%(levelname)s [%(module)s|%(funcName)s:%(lineno)s]: %(message)s') plain_formatter = logging.Formatter('%(message)s') @@ -45,15 +45,17 @@ def filter(self, record): def init_logger(config): level = getattr(logging, config.log.upper()) logger.setLevel(level) - ch_stderr = logging.StreamHandler(sys.stderr) - ch_stderr.setLevel(logging.DEBUG) - ch_stderr.setFormatter(default_formatter) + ch_all = logging.StreamHandler(sys.stderr) + ch_all.setLevel(logging.DEBUG) + ch_all.setFormatter(default_formatter) + ch_all.addFilter(ExcludeInfoFilter()) + logger.addHandler(ch_all) if config.interactive: - ch_stderr.addFilter(ExcludeInfoFilter()) - ch_stdout = logging.StreamHandler(sys.stdout) - ch_stdout.setLevel(logging.DEBUG) - ch_stdout.setFormatter(plain_formatter) - ch_stdout.addFilter(OnlyInfoFilter()) - logger.addHandler(ch_stdout) - logger.addHandler(ch_stderr) + ch_info = logging.StreamHandler(sys.stdout) + else: + ch_info = logging.StreamHandler(sys.stderr) + ch_info.setLevel(logging.DEBUG) + ch_info.setFormatter(plain_formatter) + ch_info.addFilter(OnlyInfoFilter()) + logger.addHandler(ch_info) diff --git a/src/sage_bootstrap/test/test_logger.py b/src/sage_bootstrap/test/test_logger.py index 8f5207d1956..8f5124c782a 100644 --- a/src/sage_bootstrap/test/test_logger.py +++ b/src/sage_bootstrap/test/test_logger.py @@ -25,9 +25,9 @@ class LoggerTestCase(unittest.TestCase): def test_interactive(self): stdout, stderr = run_log_with('interactive:true') self.assertEqual(stderr.strip(), dedent(""" - WARNING [runnable.print_log:25]: This is the warning log level - CRITICAL [runnable.print_log:26]: This is the critical log level - ERROR [runnable.print_log:27]: This is the error log level + WARNING [runnable|print_log:25]: This is the warning log level + CRITICAL [runnable|print_log:26]: This is the critical log level + ERROR [runnable|print_log:27]: This is the error log level """).strip()) self.assertEqual(stdout.strip(), dedent(""" This is the info log level @@ -37,10 +37,10 @@ def test_interactive(self): def test_noninteractive(self): stdout, stderr = run_log_with('interactive:false') self.assertEqual(stderr.strip(), dedent(""" - INFO [runnable.print_log:24]: This is the info log level - WARNING [runnable.print_log:25]: This is the warning log level - CRITICAL [runnable.print_log:26]: This is the critical log level - ERROR [runnable.print_log:27]: This is the error log level + This is the info log level + WARNING [runnable|print_log:25]: This is the warning log level + CRITICAL [runnable|print_log:26]: This is the critical log level + ERROR [runnable|print_log:27]: This is the error log level """).strip()) self.assertEqual(stdout.strip(), dedent(""" This is printed @@ -53,10 +53,10 @@ def test_debug(self): """ stdout, stderr = run_log_with('log:debug,interactive:true') self.assertEqual(stderr.strip(), dedent(""" - DEBUG [runnable.print_log:23]: This is the debug log level - WARNING [runnable.print_log:25]: This is the warning log level - CRITICAL [runnable.print_log:26]: This is the critical log level - ERROR [runnable.print_log:27]: This is the error log level + DEBUG [runnable|print_log:23]: This is the debug log level + WARNING [runnable|print_log:25]: This is the warning log level + CRITICAL [runnable|print_log:26]: This is the critical log level + ERROR [runnable|print_log:27]: This is the error log level """).strip()) self.assertEqual(stdout.strip(), dedent(""" This is the info log level @@ -69,8 +69,8 @@ def test_error(self): """ stdout, stderr = run_log_with('log:error,interactive:true') self.assertEqual(stderr.strip(), dedent(""" - CRITICAL [runnable.print_log:26]: This is the critical log level - ERROR [runnable.print_log:27]: This is the error log level + CRITICAL [runnable|print_log:26]: This is the critical log level + ERROR [runnable|print_log:27]: This is the error log level """).strip()) self.assertEqual(stdout.strip(), dedent(""" This is printed From 52d012637820499097664afe2ee94974733af895 Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Thu, 25 Jun 2015 22:00:39 +0200 Subject: [PATCH 05/13] rename sage-pkg -> sage-package --- src/sage_bootstrap/bin/{sage-pkg => sage-package} | 0 src/sage_bootstrap/sage_bootstrap/cmdline.py | 10 +++++----- src/sage_bootstrap/setup.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename src/sage_bootstrap/bin/{sage-pkg => sage-package} (100%) diff --git a/src/sage_bootstrap/bin/sage-pkg b/src/sage_bootstrap/bin/sage-package similarity index 100% rename from src/sage_bootstrap/bin/sage-pkg rename to src/sage_bootstrap/bin/sage-package diff --git a/src/sage_bootstrap/sage_bootstrap/cmdline.py b/src/sage_bootstrap/sage_bootstrap/cmdline.py index 4a1e053a056..e3fa86ddb73 100644 --- a/src/sage_bootstrap/sage_bootstrap/cmdline.py +++ b/src/sage_bootstrap/sage_bootstrap/cmdline.py @@ -68,7 +68,7 @@ def run(self): class SagePkgApplication(CmdlineSubcommands): """ - sage-pkg + sage-package -------- The package script is used to manage third-party tarballs. @@ -78,7 +78,7 @@ def run_config(self): """ Print the configuration - $ sage-pkg config + $ sage-package config Configuration: * log = info * interactive = True @@ -90,7 +90,7 @@ def run_list(self): """ Print a list of all available packages - $ sage-pkg list | sort + $ sage-package list | sort 4ti2 arb atlas @@ -105,7 +105,7 @@ def run_name(self, tarball_filename): """ Find the package name given a tarball filename - $ sage-pkg name pari-2.8-1564-gdeac36e.tar.gz + $ sage-package name pari-2.8-1564-gdeac36e.tar.gz pari """ tarball = Tarball(os.path.basename(tarball_filename)) @@ -115,7 +115,7 @@ def run_tarball(self, package_name): """ Find the tarball filename given a package name - $ sage-pkg tarball pari + $ sage-package tarball pari pari-2.8-1564-gdeac36e.tar.gz """ package = Package(package_name) diff --git a/src/sage_bootstrap/setup.py b/src/sage_bootstrap/setup.py index a783496b951..5f4f7292c0e 100755 --- a/src/sage_bootstrap/setup.py +++ b/src/sage_bootstrap/setup.py @@ -8,7 +8,7 @@ author='Volker Braun', author_email='vbraun.name@gmail.com', packages=['sage_bootstrap'], - scripts=['bin/sage-pkg'], + scripts=['bin/sage-package'], version='1.0', url='https://www.sagemath.org', ) From e4aff9ac0f00632b24a614c6681ea7c74e627729 Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Thu, 25 Jun 2015 22:05:52 +0200 Subject: [PATCH 06/13] fix indentation --- src/sage_bootstrap/sage_bootstrap/stdio.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/sage_bootstrap/sage_bootstrap/stdio.py b/src/sage_bootstrap/sage_bootstrap/stdio.py index 322043e0aeb..e3984595aba 100644 --- a/src/sage_bootstrap/sage_bootstrap/stdio.py +++ b/src/sage_bootstrap/sage_bootstrap/stdio.py @@ -23,15 +23,15 @@ class UnbufferedStream(object): - def __init__(self, stream): - self.stream = stream + def __init__(self, stream): + self.stream = stream - def write(self, data): - self.stream.write(data) - self.stream.flush() + def write(self, data): + self.stream.write(data) + self.stream.flush() - def __getattr__(self, attr): - return getattr(self.stream, attr) + def __getattr__(self, attr): + return getattr(self.stream, attr) REAL_STDOUT = sys.stdout @@ -39,10 +39,10 @@ def __getattr__(self, attr): def init_streams(config): - if not config.interactive: - sys.stdout = UnbufferedStream(REAL_STDOUT) + if not config.interactive: + sys.stdout = UnbufferedStream(REAL_STDOUT) def flush(): - REAL_STDOUT.flush() - REAL_STDERR.flush() + REAL_STDOUT.flush() + REAL_STDERR.flush() From e45a0b667f908f147cabbec63fe88134061ac69a Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Thu, 25 Jun 2015 22:09:48 +0200 Subject: [PATCH 07/13] Do not assume that the confball has been downloaded before in test --- src/sage_bootstrap/test/test_tarball.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sage_bootstrap/test/test_tarball.py b/src/sage_bootstrap/test/test_tarball.py index 663bef8f486..a98b8d049b3 100644 --- a/src/sage_bootstrap/test/test_tarball.py +++ b/src/sage_bootstrap/test/test_tarball.py @@ -37,6 +37,9 @@ def test_tarball(self): def test_checksum(self): pkg = Package('configure') tarball = Tarball(pkg.tarball) + with CapturedOutput() as (stdout, stderr): + with CapturedLog() as log: + tarball.download() self.assertTrue(tarball.checksum_verifies()) with open(tarball.upstream_fqn, 'w') as f: f.write('foobar') From cc213dcd06581e5906f837f1a0ea079b375d3e3f Mon Sep 17 00:00:00 2001 From: "John H. Palmieri" Date: Tue, 7 Jul 2015 10:08:30 -0700 Subject: [PATCH 08/13] trac 18748: add documentation to scripts in sage_bootstrap/bin --- src/sage_bootstrap/bin/sage-download-file | 17 +++++++++++++ src/sage_bootstrap/bin/sage-package | 31 +++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/sage_bootstrap/bin/sage-download-file b/src/sage_bootstrap/bin/sage-download-file index 450b8843593..24750aa5e8f 100755 --- a/src/sage_bootstrap/bin/sage-download-file +++ b/src/sage_bootstrap/bin/sage-download-file @@ -1,5 +1,22 @@ #!/usr/bin/env python +# USAGE: +# +# sage-download-file --print-fastest-mirror +# +# Print out the fastest mirror. All further arguments are ignored in +# that case. +# +# sage-download-file [--quiet] url-or-tarball [destination] +# +# The single mandatory argument can be a http:// url or a tarball +# filename. In the latter case, the tarball is downloaded from the +# mirror network and its checksum is verified. +# +# If the destination is not specified: +# * a url will be downloaded and the content written to stdout +# * a tarball will be saved under {SAGE_DISTFILES} + try: import sage_bootstrap except ImportError: diff --git a/src/sage_bootstrap/bin/sage-package b/src/sage_bootstrap/bin/sage-package index 648a34fbbfb..a8d21612366 100755 --- a/src/sage_bootstrap/bin/sage-package +++ b/src/sage_bootstrap/bin/sage-package @@ -1,5 +1,36 @@ #!/usr/bin/env python +# Script to manage third-party tarballs. +# +# Usage: +# +# * Print the configuration +# +# $ sage-package config +# Configuration: +# * log = info +# * interactive = True +# +# * Print a list of all available packages +# +# $ sage-package list | sort +# 4ti2 +# arb +# atlas +# autotools +# [...] +# zn_poly +# +# * Find the package name given a tarball filename +# +# $ sage-package name pari-2.8-1564-gdeac36e.tar.gz +# pari +# +# * Find the tarball filename given a package name +# +# $ sage-package tarball pari +# pari-2.8-1564-gdeac36e.tar.gz + try: import sage_bootstrap except ImportError: From 8f4fb8e1d809417a8f56984529c2436950f60e2a Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Sun, 12 Jul 2015 15:47:25 +0200 Subject: [PATCH 09/13] More docstrings --- src/sage_bootstrap/sage_bootstrap/package.py | 22 +++++++++++++++++++- src/sage_bootstrap/sage_bootstrap/tarball.py | 9 ++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/sage_bootstrap/sage_bootstrap/package.py b/src/sage_bootstrap/sage_bootstrap/package.py index 3ab80f98dd3..58feb037b54 100644 --- a/src/sage_bootstrap/sage_bootstrap/package.py +++ b/src/sage_bootstrap/sage_bootstrap/package.py @@ -26,6 +26,21 @@ class Package(object): def __init__(self, package_name): + """ + Sage Package + + A package is defined by a subdirectory of + ``SAGE_ROOT/build/pkgs/``. The name of the package is the name + of the subdirectory; The metadata of the package is contained + in various files in the package directory. This class provides + an abstraction to the metadata, you should never need to + access the package directory directly. + + INPUT: + + -- ``package_name`` -- string. Name of the package. The Sage + convention is that all package names are lower case. + """ self.name = package_name self._init_checksum() self._init_version() @@ -35,6 +50,9 @@ def __eq__(self, other): @classmethod def all(cls): + """ + Return all packages + """ base = os.path.join(SAGE_ROOT, 'build', 'pkgs') for subdir in os.listdir(base): path = os.path.join(base, subdir) @@ -44,6 +62,9 @@ def all(cls): @property def path(self): + """ + Return the package directory + """ return os.path.join(SAGE_ROOT, 'build', 'pkgs', self.name) def _init_checksum(self): @@ -63,7 +84,6 @@ def _init_checksum(self): self.md5 = result['md5'] self.sha1 = result['sha1'] self.cksum = result['cksum'] - self.sha1 = result['sha1'] self.tarball_pattern = result['tarball'] VERSION_PATCHLEVEL = re.compile('(?P.*)\.p(?P[0-9]+)') diff --git a/src/sage_bootstrap/sage_bootstrap/tarball.py b/src/sage_bootstrap/sage_bootstrap/tarball.py index 04f4be7d45e..9a9ff627531 100644 --- a/src/sage_bootstrap/sage_bootstrap/tarball.py +++ b/src/sage_bootstrap/sage_bootstrap/tarball.py @@ -24,10 +24,16 @@ class ChecksumError(Exception): + """ + Exception raised when the checksum of the tarball does not match + """ pass class FileNotMirroredError(Exception): + """ + Exception raised when the tarball cannot be downloaded from the mirrors + """ pass @@ -114,6 +120,9 @@ def download(self): raise ChecksumError('checksum does not match') def save_as(self, destination): + """ + Save the tarball as a new file + """ import shutil shutil.copy(self.upstream_fqn, destination) From f9a69a58a4123af8861a472b15fef64e6aae8381 Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Sun, 12 Jul 2015 16:05:19 +0200 Subject: [PATCH 10/13] Convert public attributes to properties and document --- src/sage_bootstrap/sage_bootstrap/package.py | 122 +++++++++++++++++-- src/sage_bootstrap/sage_bootstrap/tarball.py | 35 +++++- 2 files changed, 143 insertions(+), 14 deletions(-) diff --git a/src/sage_bootstrap/sage_bootstrap/package.py b/src/sage_bootstrap/sage_bootstrap/package.py index 58feb037b54..e4a5ae68ce0 100644 --- a/src/sage_bootstrap/sage_bootstrap/package.py +++ b/src/sage_bootstrap/sage_bootstrap/package.py @@ -41,10 +41,112 @@ def __init__(self, package_name): -- ``package_name`` -- string. Name of the package. The Sage convention is that all package names are lower case. """ - self.name = package_name + self.__name = package_name self._init_checksum() self._init_version() + @property + def name(self): + """ + Return the package name + + A package is defined by a subdirectory of + ``SAGE_ROOT/build/pkgs/``. The name of the package is the name + of the subdirectory. + + OUTPUT: + + String. + """ + return self.__name + + @property + def md5(self): + """ + Return the MD5 checksum + + Do not use, this is ancient! Use :meth:`sha1` instead. + + OUTPUT: + + String. + """ + return self.__md5 + + @property + def sha1(self): + """ + Return the SHA1 checksum + + OUTPUT: + + String. + """ + return self.__sha1 + + @property + def cksum(self): + """ + Return the Ck sum checksum + + Do not use, this is ancient! Use :meth:`sha1` instead. + + OUTPUT: + + String. + """ + return self.__cksum + + @property + def tarball(self): + """ + Return the (primary) tarball filename + + If there are multiple tarballs (currently unsupported), this + property returns the one that is unpacked automatically. + + OUTPUT: + + String. The full-qualified tarball filename. + """ + return self.__tarball + + @property + def tarball_pattern(self): + """ + Return the (primary) tarball file pattern + + OUTPUT: + + String. The full-qualified tarball filename, but with + ``VERSION`` instead of the actual tarball filename. + """ + return self.__tarball_pattern + + @property + def version(self): + """ + Return the version + + OUTPUT: + + String. The package version. Excludes the Sage-specific + patchlevel. + """ + return self.__version + + @property + def patchlevel(self): + """ + Return the patchlevel + + OUTPUT: + + String. The patchlevel of the package. Excludes the "p" + prefix. + """ + return self.__patchlevel + def __eq__(self, other): return self.tarball == other.tarball @@ -81,10 +183,10 @@ def _init_checksum(self): continue var, value = match.groups() result[var] = value - self.md5 = result['md5'] - self.sha1 = result['sha1'] - self.cksum = result['cksum'] - self.tarball_pattern = result['tarball'] + self.__md5 = result['md5'] + self.__sha1 = result['sha1'] + self.__cksum = result['cksum'] + self.__tarball_pattern = result['tarball'] VERSION_PATCHLEVEL = re.compile('(?P.*)\.p(?P[0-9]+)') @@ -93,11 +195,11 @@ def _init_version(self): package_version = f.read().strip() match = self.VERSION_PATCHLEVEL.match(package_version) if match is None: - self.version = package_version - self.patchlevel = -1 + self.__version = package_version + self.__patchlevel = -1 else: - self.version = match.group('version') - self.patchlevel = match.group('patchlevel') - self.tarball = self.tarball_pattern.replace('VERSION', self.version) + self.__version = match.group('version') + self.__patchlevel = match.group('patchlevel') + self.__tarball = self.__tarball_pattern.replace('VERSION', self.version) diff --git a/src/sage_bootstrap/sage_bootstrap/tarball.py b/src/sage_bootstrap/sage_bootstrap/tarball.py index 9a9ff627531..bb379853a8f 100644 --- a/src/sage_bootstrap/sage_bootstrap/tarball.py +++ b/src/sage_bootstrap/sage_bootstrap/tarball.py @@ -43,21 +43,48 @@ def __init__(self, tarball_name): """ A (third-party downloadable) tarball + Note that the tarball might also be a different kind of + archive format that is supported, it does not necessarily have + to be tar. + INPUT: - ``name`` - string. The full filename (``foo-1.3.tar.bz2``) of a tarball on the Sage mirror network. """ - self.filename = tarball_name - self.package = None + self.__filename = tarball_name + self.__package = None for pkg in Package.all(): if pkg.tarball == tarball_name: - self.package = pkg + self.__package = pkg if self.package is None: error = 'tarball {0} is not referenced by any Sage package'.format(tarball_name) log.error(error) raise ValueError(error) - + + @property + def filename(self): + """ + Return the tarball filename + + OUTPUT: + + String. The full filename (``foo-1.3.tar.bz2``) of the + tarball. + """ + return self.__filename + + @property + def package(self): + """ + Return the package that the tarball belongs to + + OUTPUT: + + Instance of :class:`sage_bootstrap.package.Package` + """ + return self.__package + @property def upstream_fqn(self): """ From 075228f1c38f87975b08ecefbb5414e6dc0db9de Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Sun, 12 Jul 2015 16:28:21 +0200 Subject: [PATCH 11/13] Make package.tarball a Tarball instance --- src/sage_bootstrap/sage_bootstrap/cmdline.py | 2 +- src/sage_bootstrap/sage_bootstrap/package.py | 29 +++++++++++++++-- src/sage_bootstrap/sage_bootstrap/tarball.py | 33 ++++++++++++++------ src/sage_bootstrap/test/test_package.py | 10 ++++-- src/sage_bootstrap/test/test_tarball.py | 7 +++-- 5 files changed, 61 insertions(+), 20 deletions(-) diff --git a/src/sage_bootstrap/sage_bootstrap/cmdline.py b/src/sage_bootstrap/sage_bootstrap/cmdline.py index e3fa86ddb73..ce3036decf4 100644 --- a/src/sage_bootstrap/sage_bootstrap/cmdline.py +++ b/src/sage_bootstrap/sage_bootstrap/cmdline.py @@ -119,7 +119,7 @@ def run_tarball(self, package_name): pari-2.8-1564-gdeac36e.tar.gz """ package = Package(package_name) - print(package.tarball) + print(package.tarball.filename) class SageDownloadFileApplication(object): diff --git a/src/sage_bootstrap/sage_bootstrap/package.py b/src/sage_bootstrap/sage_bootstrap/package.py index e4a5ae68ce0..346eb976cea 100644 --- a/src/sage_bootstrap/sage_bootstrap/package.py +++ b/src/sage_bootstrap/sage_bootstrap/package.py @@ -42,9 +42,13 @@ def __init__(self, package_name): convention is that all package names are lower case. """ self.__name = package_name + self.__tarball = None self._init_checksum() self._init_version() + def __repr__(self): + return 'Package {0}'.format(self.name) + @property def name(self): """ @@ -100,15 +104,18 @@ def cksum(self): @property def tarball(self): """ - Return the (primary) tarball filename + Return the (primary) tarball If there are multiple tarballs (currently unsupported), this property returns the one that is unpacked automatically. OUTPUT: - String. The full-qualified tarball filename. + Instance of :class:`sage_bootstrap.tarball.Tarball` """ + if self.__tarball is None: + from sage_bootstrap.tarball import Tarball + self.__tarball = Tarball(self.tarball_filename, package=self) return self.__tarball @property @@ -116,6 +123,9 @@ def tarball_pattern(self): """ Return the (primary) tarball file pattern + If there are multiple tarballs (currently unsupported), this + property returns the one that is unpacked automatically. + OUTPUT: String. The full-qualified tarball filename, but with @@ -123,6 +133,20 @@ def tarball_pattern(self): """ return self.__tarball_pattern + @property + def tarball_filename(self): + """ + Return the (primary) tarball filename + + If there are multiple tarballs (currently unsupported), this + property returns the one that is unpacked automatically. + + OUTPUT: + + String. The full-qualified tarball filename. + """ + return self.tarball_pattern.replace('VERSION', self.version) + @property def version(self): """ @@ -200,6 +224,5 @@ def _init_version(self): else: self.__version = match.group('version') self.__patchlevel = match.group('patchlevel') - self.__tarball = self.__tarball_pattern.replace('VERSION', self.version) diff --git a/src/sage_bootstrap/sage_bootstrap/tarball.py b/src/sage_bootstrap/sage_bootstrap/tarball.py index bb379853a8f..9d5f2b87628 100644 --- a/src/sage_bootstrap/sage_bootstrap/tarball.py +++ b/src/sage_bootstrap/sage_bootstrap/tarball.py @@ -39,7 +39,7 @@ class FileNotMirroredError(Exception): class Tarball(object): - def __init__(self, tarball_name): + def __init__(self, tarball_name, package=None): """ A (third-party downloadable) tarball @@ -53,15 +53,25 @@ def __init__(self, tarball_name): of a tarball on the Sage mirror network. """ self.__filename = tarball_name - self.__package = None - for pkg in Package.all(): - if pkg.tarball == tarball_name: - self.__package = pkg - if self.package is None: - error = 'tarball {0} is not referenced by any Sage package'.format(tarball_name) - log.error(error) - raise ValueError(error) - + if package is None: + self.__package = None + for pkg in Package.all(): + if pkg.tarball_filename == tarball_name: + self.__package = pkg + if self.package is None: + error = 'tarball {0} is not referenced by any Sage package'.format(tarball_name) + log.error(error) + raise ValueError(error) + else: + self.__package = package + if package.tarball_filename != tarball_name: + error = 'tarball {0} is not referenced by the {1} package'.format(tarball_name, package.name) + log.error(error) + raise ValueError(error) + + def __repr__(self): + return 'Tarball {0}'.format(self.filename) + @property def filename(self): """ @@ -92,6 +102,9 @@ def upstream_fqn(self): """ return os.path.join(SAGE_DISTFILES, self.filename) + def __eq__(self, other): + return self.filename == other.filename + def _compute_hash(self, algorithm): with open(self.upstream_fqn, 'rb') as f: while True: diff --git a/src/sage_bootstrap/test/test_package.py b/src/sage_bootstrap/test/test_package.py index bcbd9cea53b..313afa0a069 100644 --- a/src/sage_bootstrap/test/test_package.py +++ b/src/sage_bootstrap/test/test_package.py @@ -16,6 +16,7 @@ import unittest from sage_bootstrap.package import Package +from sage_bootstrap.tarball import Tarball class PackageTestCase(unittest.TestCase): @@ -25,9 +26,12 @@ def test_package(self): self.assertTrue(pkg.name, 'pari') self.assertTrue(pkg.path.endswith('build/pkgs/pari')) self.assertEqual(pkg.tarball_pattern, 'pari-VERSION.tar.gz') - self.assertTrue(pkg.tarball.startswith('pari-') and - pkg.tarball.endswith('.tar.gz')) - + self.assertEqual(pkg.tarball_filename, pkg.tarball.filename) + self.assertTrue(pkg.tarball.filename.startswith('pari-') and + pkg.tarball.filename.endswith('.tar.gz')) + self.assertTrue(pkg.tarball.filename.startswith('pari-') and + pkg.tarball.filename.endswith('.tar.gz')) + self.assertIsInstance(pkg.tarball, Tarball) def test_all(self): pari = Package('pari') diff --git a/src/sage_bootstrap/test/test_tarball.py b/src/sage_bootstrap/test/test_tarball.py index a98b8d049b3..4318cc2b803 100644 --- a/src/sage_bootstrap/test/test_tarball.py +++ b/src/sage_bootstrap/test/test_tarball.py @@ -26,7 +26,8 @@ class TarballTestCase(unittest.TestCase): def test_tarball(self): pkg = Package('configure') - tarball = Tarball(pkg.tarball) + tarball = Tarball(pkg.tarball_filename) + self.assertEqual(tarball, pkg.tarball) self.assertEqual(pkg, tarball.package) with CapturedOutput() as (stdout, stderr): with CapturedLog() as log: @@ -36,7 +37,7 @@ def test_tarball(self): def test_checksum(self): pkg = Package('configure') - tarball = Tarball(pkg.tarball) + tarball = pkg.tarball with CapturedOutput() as (stdout, stderr): with CapturedLog() as log: tarball.download() @@ -49,7 +50,7 @@ def test_checksum(self): tarball.download() msg = log.messages() self.assertTrue( - ('INFO', 'Attempting to download package {0} from mirrors'.format(pkg.tarball)) in msg) + ('INFO', 'Attempting to download package {0} from mirrors'.format(pkg.tarball_filename)) in msg) self.assertEqual(stdout.getvalue(), '') self.assertEqual(stderr.getvalue(), '[......................................................................]\n') From c56991b37c05e7381ca2f27ca7183fadddb1dfe6 Mon Sep 17 00:00:00 2001 From: Volker Braun Date: Sun, 12 Jul 2015 19:26:52 +0200 Subject: [PATCH 12/13] Fix Python 2.6 test error --- src/sage_bootstrap/test/test_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sage_bootstrap/test/test_package.py b/src/sage_bootstrap/test/test_package.py index 313afa0a069..18b316771cf 100644 --- a/src/sage_bootstrap/test/test_package.py +++ b/src/sage_bootstrap/test/test_package.py @@ -31,7 +31,7 @@ def test_package(self): pkg.tarball.filename.endswith('.tar.gz')) self.assertTrue(pkg.tarball.filename.startswith('pari-') and pkg.tarball.filename.endswith('.tar.gz')) - self.assertIsInstance(pkg.tarball, Tarball) + self.assertTrue(isinstance(pkg.tarball, Tarball)) def test_all(self): pari = Package('pari') From 60a581b42aa98a233d8b6c5a66487586f5619826 Mon Sep 17 00:00:00 2001 From: "John H. Palmieri" Date: Sun, 12 Jul 2015 21:22:15 -0700 Subject: [PATCH 13/13] trac 18748: add URL for tox to sage_bootstrap/README --- src/sage_bootstrap/README | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sage_bootstrap/README b/src/sage_bootstrap/README index 9ca5362d73d..c3c7e196c4a 100644 --- a/src/sage_bootstrap/README +++ b/src/sage_bootstrap/README @@ -6,12 +6,12 @@ library here, nor should you import anything from sage_bootstrap into Sage (because this would reconfigure logging). They must be kept separate. -Everything here must support Python 2.6, 2.7, and 3.3+. Use tox to -automatically run the tests with all relevant Python versions. Tests -are written as unittest, not as doctests, because the library is not -meant to be used interactively. Note that the library comes with a -setup.py file so that tox can test it, but it is not meant to be -installed to SAGE_LOCAL. +Everything here must support Python 2.6, 2.7, and 3.3+. Use tox +(https://testrun.org/tox/latest/) to automatically run the tests with +all relevant Python versions. Tests are written as unittest, not as +doctests, because the library is not meant to be used interactively. +Note that the library comes with a setup.py file so that tox can test +it, but it is not meant to be installed to SAGE_LOCAL. Command-line utilities must be able to run as part of a pipe | filter chain. So you have to be careful about what you send to stdout. You