From 2f807016cca476ceca3d672d51f8ad68eba13072 Mon Sep 17 00:00:00 2001 From: Ryosuke Kaneko Date: Tue, 17 Apr 2012 05:33:31 +0900 Subject: [PATCH 1/3] Add genericdownloader. This module works like curl module. --- pythonz/curl.py | 8 ++--- pythonz/downloader.py | 12 +++++-- pythonz/genericdownloader.py | 69 ++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 pythonz/genericdownloader.py diff --git a/pythonz/curl.py b/pythonz/curl.py index 44494c0..16f9a0e 100644 --- a/pythonz/curl.py +++ b/pythonz/curl.py @@ -1,6 +1,5 @@ import re -import sys import subprocess from subprocess import Popen, PIPE @@ -10,11 +9,10 @@ from pythonz.exceptions import CurlFetchException class Curl(object): - def __init__(self): + @classmethod + def can_use(cls): returncode = subprocess.call("command -v curl > /dev/null", shell=True) - if returncode: - logger.log("pythonz required curl. curl was not found in your path.") - sys.exit(1) + return not returncode def read(self, url): p = Popen("curl -skL %s" % url, stdout=PIPE, shell=True) diff --git a/pythonz/downloader.py b/pythonz/downloader.py index 7678e7a..a45ca46 100644 --- a/pythonz/downloader.py +++ b/pythonz/downloader.py @@ -2,15 +2,23 @@ from pythonz.define import PYTHON_VERSIONS_URLS from pythonz.log import logger from pythonz.curl import Curl +from pythonz.genericdownloader import GenericDownloader + + +def get_concrete_downloader(): + if Curl.can_use(): + c = Curl() + else: + c = GenericDownloader() def get_headerinfo_from_url(url): - c = Curl() + c = get_concrete_downloader() return c.readheader(url) class Downloader(object): def download(self, msg, url, path): logger.info("Downloading %s as %s" % (msg, path)) - c = Curl() + c = get_concrete_downloader() c.fetch(url, path) def get_python_version_url(type, version): diff --git a/pythonz/genericdownloader.py b/pythonz/genericdownloader.py new file mode 100644 index 0000000..7a5f655 --- /dev/null +++ b/pythonz/genericdownloader.py @@ -0,0 +1,69 @@ +# coding: utf-8 + +import re +import urllib +import urllib2 +import sys +import traceback + +from pythonz.log import logger +from pythonz.exceptions import CurlFetchException + +class ProgressBar(object): + def __init__(self, out=sys.stdout): + self._term_width = 79 + self._out = out + + def update_line(self, current): + num_bar = int(current / 100.0 * (self._term_width - 5)) + return '#' * num_bar + ' ' * (self._term_width - 5 - num_bar) + \ + u' %3d' % int(current) + u'%\r' + + def reporthook(self, blocknum, bs, size): + current = (blocknum * bs * 100) / size + if current > 100: + current = 100 + self._out.write(self.update_line(current)) + self._out.flush() + + def finish(self): + self._out.write(self.update_line(100)) + self._out.flush() + +class HEADRequest(urllib2.Request): + def get_method(self): + return "HEAD" + +class GenericDownloader(object): + @classmethod + def can_use(cls): + return True + + def read(self, url): + try: + r = urllib.urlopen(url) + except: + logger.log(traceback.format_exc()) + raise CurlFetchException('Failed to fetch.') + return r.read() + + def readheader(self, url): + try: + req = HEADRequest(url) + res = urllib2.urlopen(req) + except: + logger.log(traceback.format_exc()) + raise CurlFetchException('Failed to fetch.') + if res.code != 200: + raise CurlFetchException('Failed to fetch.') + return res.info() + + def fetch(self, url, filename): + b = ProgressBar() + try: + r = urllib.urlretrieve(url, filename, b.reporthook) + sys.stdout.write('\n') + except: + sys.stdout.write('\n') + logger.log(traceback.format_exc()) + raise CurlFetchException('Failed to fetch.') From 5ebf9d8a888e7ba0e11af5df24131044a5a87b1e Mon Sep 17 00:00:00 2001 From: Ryosuke Kaneko Date: Tue, 17 Apr 2012 05:50:39 +0900 Subject: [PATCH 2/3] Fix a bug. get_concrete_downloader() didn't return value. --- pythonz/downloader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonz/downloader.py b/pythonz/downloader.py index a45ca46..db22683 100644 --- a/pythonz/downloader.py +++ b/pythonz/downloader.py @@ -7,9 +7,9 @@ def get_concrete_downloader(): if Curl.can_use(): - c = Curl() + return Curl() else: - c = GenericDownloader() + return GenericDownloader() def get_headerinfo_from_url(url): c = get_concrete_downloader() From 9a4c85b55fde4612ca8c691999d5ef25a6497102 Mon Sep 17 00:00:00 2001 From: Ryosuke Kaneko Date: Wed, 18 Apr 2012 04:47:12 +0900 Subject: [PATCH 3/3] Fix to catch appropriate exceptions and style of code. --- pythonz/genericdownloader.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pythonz/genericdownloader.py b/pythonz/genericdownloader.py index 7a5f655..463f0e3 100644 --- a/pythonz/genericdownloader.py +++ b/pythonz/genericdownloader.py @@ -16,8 +16,10 @@ def __init__(self, out=sys.stdout): def update_line(self, current): num_bar = int(current / 100.0 * (self._term_width - 5)) - return '#' * num_bar + ' ' * (self._term_width - 5 - num_bar) + \ - u' %3d' % int(current) + u'%\r' + bars = u'#' * num_bar + spaces = u' ' * (self._term_width - 5 - num_bar) + percentage = u'%3d' % int(current) + u'%\r' + return bars + spaces + u' ' + percentage def reporthook(self, blocknum, bs, size): current = (blocknum * bs * 100) / size @@ -42,7 +44,7 @@ def can_use(cls): def read(self, url): try: r = urllib.urlopen(url) - except: + except IOError: logger.log(traceback.format_exc()) raise CurlFetchException('Failed to fetch.') return r.read() @@ -51,7 +53,7 @@ def readheader(self, url): try: req = HEADRequest(url) res = urllib2.urlopen(req) - except: + except IOError: logger.log(traceback.format_exc()) raise CurlFetchException('Failed to fetch.') if res.code != 200: @@ -61,9 +63,9 @@ def readheader(self, url): def fetch(self, url, filename): b = ProgressBar() try: - r = urllib.urlretrieve(url, filename, b.reporthook) + urllib.urlretrieve(url, filename, b.reporthook) sys.stdout.write('\n') - except: + except IOError: sys.stdout.write('\n') logger.log(traceback.format_exc()) raise CurlFetchException('Failed to fetch.')