Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Bug fix for pull request #2 #3

Merged
merged 3 commits into from

2 participants

@yamaneko1212

Bug fix for #2 Add genericdownloader as substitute of curl module.
Though I want to use fetch command via popen, it can't use HEAD method, so I write genericdownloader module. This module consists of urllib and httplib and behaves like curl module.
I check these changes in FreeBSD and MacOSX and they work well ;)

pythonz/genericdownloader.py
((5 lines not shown))
+import urllib2
+import sys
+import traceback
+
+from pythonz.log import logger
+from pythonz.exceptions import CurlFetchException
+
+class ProgressBar(object):
+ def __init__(self, out=sys.stdout):
+ self._term_width = 79
+ self._out = out
+
+ def update_line(self, current):
+ num_bar = int(current / 100.0 * (self._term_width - 5))
+ return '#' * num_bar + ' ' * (self._term_width - 5 - num_bar) + \
+ u' %3d' % int(current) + u'%\r'
@saghul Owner
saghul added a note

Please don't break the like here, screens are wide enough nowadays :-)

I think this is not so bad, it's because PEP8 said that limit all lines to a maximum of 79 characters.
http://www.python.org/dev/peps/pep-0008/#maximum-line-length
But, if you don't like above, i'll modify it like this.

    def update_line(self, current):
        num_bar = int(current / 100.0 * (self._term_width - 5))
        bars = u'#' * num_bar
        spaces = u' ' * (self._term_width - 5 - num_bar)
        percentage = u'%3d' % int(current) + u'%\r'
        return bars + spaces + u' ' + percentage
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
pythonz/genericdownloader.py
((31 lines not shown))
+ self._out.flush()
+
+class HEADRequest(urllib2.Request):
+ def get_method(self):
+ return "HEAD"
+
+class GenericDownloader(object):
+ @classmethod
+ def can_use(cls):
+ return True
+
+ def read(self, url):
+ try:
+ r = urllib.urlopen(url)
+ except:
+ logger.log(traceback.format_exc())
@saghul Owner
saghul added a note

Catch the appropriate exception here.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
pythonz/genericdownloader.py
((39 lines not shown))
+ def can_use(cls):
+ return True
+
+ def read(self, url):
+ try:
+ r = urllib.urlopen(url)
+ except:
+ logger.log(traceback.format_exc())
+ raise CurlFetchException('Failed to fetch.')
+ return r.read()
+
+ def readheader(self, url):
+ try:
+ req = HEADRequest(url)
+ res = urllib2.urlopen(req)
+ except:
@saghul Owner
saghul added a note

Catch the appropriate exception here.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
pythonz/genericdownloader.py
((51 lines not shown))
+ try:
+ req = HEADRequest(url)
+ res = urllib2.urlopen(req)
+ except:
+ logger.log(traceback.format_exc())
+ raise CurlFetchException('Failed to fetch.')
+ if res.code != 200:
+ raise CurlFetchException('Failed to fetch.')
+ return res.info()
+
+ def fetch(self, url, filename):
+ b = ProgressBar()
+ try:
+ r = urllib.urlretrieve(url, filename, b.reporthook)
+ sys.stdout.write('\n')
+ except:
@saghul Owner
saghul added a note

Catch the appropriate exception here.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
@saghul
Owner

Hi,

Thanks! The diff looks fine, I just added some minor comments. Actually I'm thinking about removing the Curl downloader, doing things via Popen seems bad if we can do it with the standard library. I'll do that once I merge your pull request.

@yamaneko1212

OK, I fixed them. Please check them.

Actually I'm thinking about removing the Curl downloader, doing things via Popen seems bad if we can do it with the standard library.

Nice!

@saghul saghul merged commit 9a4c85b into saghul:master
@saghul
Owner

Done! I just merged your changes and then removed Curl :-) Thanks a lot!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
View
8 pythonz/curl.py
@@ -1,6 +1,5 @@
import re
-import sys
import subprocess
from subprocess import Popen, PIPE
@@ -10,11 +9,10 @@
from pythonz.exceptions import CurlFetchException
class Curl(object):
- def __init__(self):
+ @classmethod
+ def can_use(cls):
returncode = subprocess.call("command -v curl > /dev/null", shell=True)
- if returncode:
- logger.log("pythonz required curl. curl was not found in your path.")
- sys.exit(1)
+ return not returncode
def read(self, url):
p = Popen("curl -skL %s" % url, stdout=PIPE, shell=True)
View
12 pythonz/downloader.py
@@ -2,15 +2,23 @@
from pythonz.define import PYTHON_VERSIONS_URLS
from pythonz.log import logger
from pythonz.curl import Curl
+from pythonz.genericdownloader import GenericDownloader
+
+
+def get_concrete_downloader():
+ if Curl.can_use():
+ return Curl()
+ else:
+ return GenericDownloader()
def get_headerinfo_from_url(url):
- c = Curl()
+ c = get_concrete_downloader()
return c.readheader(url)
class Downloader(object):
def download(self, msg, url, path):
logger.info("Downloading %s as %s" % (msg, path))
- c = Curl()
+ c = get_concrete_downloader()
c.fetch(url, path)
def get_python_version_url(type, version):
View
71 pythonz/genericdownloader.py
@@ -0,0 +1,71 @@
+# coding: utf-8
+
+import re
+import urllib
+import urllib2
+import sys
+import traceback
+
+from pythonz.log import logger
+from pythonz.exceptions import CurlFetchException
+
+class ProgressBar(object):
+ def __init__(self, out=sys.stdout):
+ self._term_width = 79
+ self._out = out
+
+ def update_line(self, current):
+ num_bar = int(current / 100.0 * (self._term_width - 5))
+ bars = u'#' * num_bar
+ spaces = u' ' * (self._term_width - 5 - num_bar)
+ percentage = u'%3d' % int(current) + u'%\r'
+ return bars + spaces + u' ' + percentage
+
+ def reporthook(self, blocknum, bs, size):
+ current = (blocknum * bs * 100) / size
+ if current > 100:
+ current = 100
+ self._out.write(self.update_line(current))
+ self._out.flush()
+
+ def finish(self):
+ self._out.write(self.update_line(100))
+ self._out.flush()
+
+class HEADRequest(urllib2.Request):
+ def get_method(self):
+ return "HEAD"
+
+class GenericDownloader(object):
+ @classmethod
+ def can_use(cls):
+ return True
+
+ def read(self, url):
+ try:
+ r = urllib.urlopen(url)
+ except IOError:
+ logger.log(traceback.format_exc())
+ raise CurlFetchException('Failed to fetch.')
+ return r.read()
+
+ def readheader(self, url):
+ try:
+ req = HEADRequest(url)
+ res = urllib2.urlopen(req)
+ except IOError:
+ logger.log(traceback.format_exc())
+ raise CurlFetchException('Failed to fetch.')
+ if res.code != 200:
+ raise CurlFetchException('Failed to fetch.')
+ return res.info()
+
+ def fetch(self, url, filename):
+ b = ProgressBar()
+ try:
+ urllib.urlretrieve(url, filename, b.reporthook)
+ sys.stdout.write('\n')
+ except IOError:
+ sys.stdout.write('\n')
+ logger.log(traceback.format_exc())
+ raise CurlFetchException('Failed to fetch.')
Something went wrong with that request. Please try again.