Skip to content
This repository
tree: a201de1d70
Fetching contributors…

Cannot retrieve contributors at this time

file 148 lines (122 sloc) 5.268 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
import re
import os
import urllib2

from pkgtools.pypi import PyPI
from .types import PygError, Version, args_manager
from .utils import FileMapper, ext, right_egg, version_egg
from .log import logger


__all__ = ['WebManager', 'PackageManager', 'PREFERENCES']


## This constants holds files priority
PREFERENCES = ('.egg', '.tar.gz', '.tar.bz2', '.zip')


class WebManager(object):

    _versions_re = r'{0}-(\d+\.?(?:\d\.?|\d\w)*)-?.*'

    def __init__(self, req):
        self.pypi = PyPI(index_url=args_manager['index_url'])
        self.req = req
        self.name = self.req.name
        self.versions = None
        try:
            realname = sorted(self.pypi.search({'name': self.name}),
                              key=lambda i: i['_pypi_ordering'], reverse=True)[0]['name']
            if self.name.lower() == realname.lower():
                self.name = realname
                req.name = realname
        except (KeyError, IndexError):
            pass

        self.versions = map(Version, self.pypi.package_releases(self.name, True))
        if not self.versions:
            self.name, old = self.name.capitalize(), self.name
            self.versions = map(Version, self.pypi.package_releases(self.name, True))
        ## Slow way: we need to search versions by ourselves
        if not self.versions:
            self.name = old
            self.versions = WebManager.versions_from_html(self.name)
        self.versions = sorted((v for v in self.versions if req.match(v)), reverse=True)

    @ staticmethod
    def request(url):
        r = urllib2.Request(url)
        return urllib2.urlopen(r).read()

    @ staticmethod
    def versions_from_html(name):
        _vre = re.compile(WebManager._versions_re.format(name), re.I)
        data = WebManager.request('http://pypi.python.org/simple/{0}'.format(name))
        return map(Version, set(v.strip('.') for v in _vre.findall(data)))

    def find(self):
        for version in self.versions:
            for res in self.pypi.release_urls(self.name, str(version)):
                yield version, res['filename'], res['md5_digest'], res['url']


class PackageManager(object):
    def __init__(self, req, pref=None):
        if pref is None:
            pref = PREFERENCES
        if len(pref) < 4:
            for p in PREFERENCES:
                if p not in pref:
                    pref.append(p)

        ## For now fast=True and index_url=DEFAULT
        self.w = WebManager(req)
        self.name = self.w.name
        self.pref = pref
        self.files = FileMapper(list)
        self.files.pref = self.pref

    def arrange_items(self):
        for p in self.w.find():
            e = ext(p[3])
            self.files[e].append(p)
        ## FIXME: We have to consider the preferences!
        return self.files


class Downloader(object):
    def __init__(self, req, pref=None):
        try:
            self.pman = PackageManager(req, pref)
            self.name = self.pman.name
        except urllib2.HTTPError as e:
            logger.fatal('E: {0}', e.msg)
            raise PygError

        self.files = self.pman.arrange_items()
        if not self.files:
            logger.error('E: Did not find files to download')
            raise PygError

    def download(self, dest):
        dest = os.path.abspath(dest)

        ## We need a placeholder because of the nested for loops
        success = False

        for p in self.pman.pref:
            if success:
                break
            if not self.files[p]:
                logger.error('{0} files not found. Continue searching...', p)
                continue
            for v, name, hash, url in self.files[p]:
                if success:
                    break
                if p == '.egg' and not right_egg(name):
                    logger.info('Found egg file for another Python version: {0}. Continue searching...', version_egg(name))
                    continue
                try:
                    data = WebManager.request(url)
                except (urllib2.URLError, urllib2.HTTPError) as e:
                    logger.debug('urllib2 error: {0}', e.args)
                    continue
                if not data:
                    logger.debug('request failed')
                    continue
                if not os.path.exists(dest):
                    os.makedirs(dest)
                logger.info('Retrieving data for {0}', self.name)
                try:
                    logger.info('Writing data into {0}', name)
                    with open(os.path.join(dest, name), 'w') as f:
                        f.write(data)
                except (IOError, OSError):
                    logger.debug('error while writing data')
                    continue
                logger.info('{0} downloaded successfully', self.name)
                success = True
                self.name = name


## OLD! We are using xmlrpclib to communicate with pypi
## Maybe we can use it in the future
class LinkFinder(object):

    base_url = 'http://pypi.python.org/simple/'
    file_regex = re.compile(r'<a\s?href="(?P<href>[^"]+)">(?P<name>[^\<]+)</a><br/>')
    link_regex = re.compile(r'<a\s?href="(?P<href>[^"]+)"\srel="(?P<rel>[^"]+)">(?P<name>[^\<]+)</a><br/>')
Something went wrong with that request. Please try again.