Skip to content


Subversion checkout URL

You can clone with
Download ZIP
tree: 87075c4690
Fetching contributors…

Cannot retrieve contributors at this time

182 lines (165 sloc) 5.445 kB
# -*- coding: utf-8 -*-
from __future__ import with_statement
from StringIO import StringIO
from collections import defaultdict
import os
import re
import requests
import shutil
import subprocess
import tarfile
import tempfile
import unicodedata
import urlparse
import xmlrpclib
import json
except ImportError:
import simplejson as json
from requests import async
except RuntimeError:
class dummy_async(object):
def get(self, url, **kwargs):
return requests.get(url, **kwargs)
def map(self, rs):
return rs
async = dummy_async()
def check_urls(urls, timeout=5):
Checks a list of URLs (asynchronously if possible)
rs = [async.get(url, timeout=5) for url in urls]
return [(response.url, 200 <= response.status_code < 300) for response in]
def _check_url(url):
Checks a single URL
return requests.get(url, timeout=5).status_code == 200
def slugify(value):
Normalizes string, converts to lowercase, removes non-alpha characters,
and converts spaces to hyphens.
Slightly adapted from django.template.default_filters.slugify
value = unicode(value)
value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
value = unicode(re.sub('[^\w\s-]', '', value).strip().lower())
return re.sub('[-\s]+', '-', value)
def _transform_pypi(data):
Transform the data from PyPI into the data we want.
return dict([(key, value) for key, value in data.items() if value != "UNKNOWN" and key in PYPI_KEYS])
def get_pypi_data(slug):
Download and process data on PyPI for an app with given slug, if available.
If not available, returns empty dictionary.
client = xmlrpclib.ServerProxy('')
releases = client.package_releases(slug)
except OSError:
return {}
if not releases:
return {}
release = releases[0]
data = client.release_data(slug, release)
return _transform_pypi(data)
def _find_author_url(repourl):
Try to find the Author URL if the URL is on github or bitbucket.
scheme, netloc, path, _, _ = urlparse.urlsplit(repourl)
if netloc in ['', '']:
newpath = '/'.join(path.split('/')[:2])
return urlparse.urlunsplit((scheme, netloc, newpath, '', ''))
return None
def _transform_djangopackages(data):
Process djangopackages data.
url = data.get('repo_url', None)
data = dict([(key, value) for key, value in data.items() if key in DJANGOPACKAGES_KEYS])
data['description'] = data.pop('repo_description', None)
data['url'] = url
data['author_url'] = _find_author_url(url)
return data
def get_djangopackages_data(slug):
Download and process data from djangopackages using their API.
url = '' % slug
response = requests.get(url, timeout=5)
if response.status_code != 200:
return {'description': None}
data = json.loads(response.content)
return _transform_djangopackages(data)
def get_package_data(slug):
Get package data from Djangopackages and PyPI
data = defaultdict(lambda:None)
return data
def _bundle(workspace, setuppy, config):
Does the actual bundling for `bundle`.
fnull = open(os.devnull, 'w')
subprocess.check_call(['python', setuppy, 'sdist', '-d', workspace], stdout=fnull, stderr=fnull)
eggfile = os.path.join(workspace, os.listdir(workspace)[0])
bundle = StringIO()
tarball =, mode='w:gz')
# add the egg
tarball.add(eggfile, arcname='package.tar.gz')
# add templates
templates = config['templates'].as_dict()
for arcname, fpath in templates.items():
full_arcname = 'templates/%s' % arcname
# backwards compatibility, check for URL:
if fpath.startswith(('http://', 'https://')):
response = requests.get(fpath)
tarball.addfile(tarfile.TarInfo(full_arcname), response)
tarball.add(fpath, arcname=full_arcname)
# add license
# backwards compatibility, check for old license-text option:
if 'license-path' in config['app']:
tarball.add(config['app']['license-path'], arcname='meta/LICENSE.txt')
response = requests.get(config['app']['license-text'])
tarball.addfile(tarfile.TarInfo('meta/LICENSE.txt'), response)
# add the config
configpath = os.path.join(workspace, 'config')
with open(configpath, 'w') as fobj:
tarball.add(configpath, 'meta/config.cfg')
# close, seek, return
return bundle
def bundle_app(setuppy, config):
Bundles a and all other files required (templates/license) into
a StringIO bundle (gzipped)
distdir = tempfile.mkdtemp(prefix='djeese')
return _bundle(distdir, setuppy, config)
Jump to Line
Something went wrong with that request. Please try again.