Permalink
Switch branches/tags
Nothing to show
Find file
Fetching contributors…
Cannot retrieve contributors at this time
223 lines (184 sloc) 6.96 KB
import os
import tarfile
import shutil
import sys
import urllib2
try:
import simplejson as json
except ImportError:
import json
import pip
# @@@ fix me
try:
from pip.log import Logger
except ImportError:
from pip import Logger
from pip import call_subprocess
logger = Logger([(1, sys.stdout)])
pip.logger = logger
REPOSITORIES_FILE = os.path.join(os.path.dirname(__file__), "repositories.txt")
COMPLETED_FILE = os.path.join(os.path.dirname(__file__), "completed")
DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
WORK_DIR = os.path.join(DATA_DIR, "work")
DIST_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), "dist"))
def format_size(bytes):
if bytes > 1000*1000:
return "%.1fMb" % (bytes/1000.0/1000)
elif bytes > 10*1000:
return "%iKb" % (bytes/1000)
elif bytes > 1000:
return "%.1fKb" % (bytes/1000.0)
else:
return "%ibytes" % bytes
def build_basename(user, repository, commit):
return "%s-%s-%s" % (user, repository, commit[:7])
def read_repositories(filename):
repos = []
for line in open(filename, "rb"):
bits = line.split()
if len(bits) == 3:
kind, user, repository = bits
branch = None
elif len(bits) == 4:
kind, user, repository, branch = bits
else:
raise Exception("incompatible file format for '%s'" % filename)
repos.append((kind, user, repository, branch))
return repos
def read_json_file(filename):
if not os.path.exists(filename):
return []
items = []
for line in open(filename, "rb"):
items.append(json.loads(line))
return items
def dump_json_items(filename, items):
fp = open(filename, "wb")
for item in items:
fp.write("%s\n" % json.dumps(item))
fp.close()
def find_head_github(user, repository, branch=None):
if branch is None:
branch = "master"
url = "http://github.com/api/v2/json/repos/show/%s/%s/branches" % (user, repository)
response = urllib2.urlopen(url)
info = json.loads(response.read())
return info["branches"][branch]
def download_tarball(kind, user, repository, commit, show_progress=True):
"""
Given a repository name and SHA1 download and store the tarball
"""
basename = build_basename(user, repository, commit)
filename = os.path.join(DATA_DIR, "%s.tar.gz" % basename)
# force bitbucket to always download
if not os.path.exists(filename) or kind == "bitbucket":
if kind == "github":
url = "http://github.com/%s/%s/tarball/%s" % (user, repository, commit)
elif kind == "bitbucket":
url = "http://bitbucket.org/%s/%s/get/%s.tar.gz" % (user, repository, commit)
response = urllib2.urlopen(url)
try:
total_length = int(response.info()["content-length"])
except (ValueError, KeyError):
total_length = 0
logger.info("Fetching %s" % url)
logger.indent += 2
downloaded = 0
fp = open(filename, "wb")
try:
if show_progress:
if total_length:
logger.start_progress("Downloading tarball (%s): " % format_size(total_length))
else:
logger.start_progress("Downloading tarball (unknown size): ")
else:
logger.info("Downloading")
while 1:
chunk = response.read(4096)
if not chunk:
break
downloaded += len(chunk)
if show_progress:
if not total_length:
logger.show_progress("%s" % format_size(downloaded))
else:
logger.show_progress("%3i%% %s" % (100*downloaded/total_length, format_size(downloaded)))
fp.write(chunk)
fp.close()
finally:
if show_progress:
logger.end_progress("%s downloaded" % format_size(downloaded))
logger.indent -= 2
logger.info("Extracting %s" % os.path.basename(filename))
tar = tarfile.open(filename)
tar.extractall(path=WORK_DIR)
def build_release(dist_dir, kind, user, repository, commit):
basename = build_basename(user, repository, commit)
if kind == "github":
source_dir = os.path.join(WORK_DIR, basename)
elif kind == "bitbucket":
source_dir = os.path.join(WORK_DIR, repository)
setup_py = os.path.realpath(os.path.join(source_dir, "setup.py"))
# use setuptools hack to allow egg_info in setup.cfg to work for
# development builds (requires setuptools in dev release environment)
cmd = [
sys.executable,
"-c", "import setuptools;__file__=%r;execfile(%r)" % (setup_py, setup_py),
"sdist",
"-d", dist_dir,
]
call_subprocess(cmd,
cwd = source_dir,
show_stdout = False,
command_desc = "python setup.py sdist",
)
def run(data_dir, work_dir, repositories_file, completed_file, dist_dir):
if not os.path.exists(data_dir):
os.makedirs(data_dir)
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
os.makedirs(work_dir)
repositories = read_repositories(repositories_file)
commits = []
for kind, user, repository, branch in repositories:
if kind == "github":
head = find_head_github(user, repository, branch)
elif kind == "bitbucket":
# tip file download is the only way to get the latest from what
# i can tell
head = "tip"
else:
logger.warning("Unknown service")
continue
commits.append((kind, user, repository, head))
completed = read_json_file(completed_file)
completed_cache = dict([(c, True) for k, u, r, c in completed])
try:
for kind, user, repository, commit in commits:
current = "%s/%s (%s)" % (user, repository, commit[:7])
if commit in completed_cache:
logger.info("Skipping %s" % current)
continue
logger.info("Handling %s" % current)
logger.indent += 2
download_tarball(kind, user, repository, commit)
build_release(dist_dir, kind, user, repository, commit)
# only non-bitbucket items get to be marked completed as they
# have a distinct commit we can ensure is completed
if not kind == "bitbucket":
completed.append((kind, user, repository, commit))
logger.indent -= 2
finally:
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
dump_json_items(completed_file, completed)
def main():
# @@@ optparse
data_dir = DATA_DIR
work_dir = WORK_DIR
repositories_file = REPOSITORIES_FILE
completed_file = COMPLETED_FILE
dist_dir = DIST_DIR
run(data_dir, work_dir, repositories_file, completed_file, dist_dir)
if __name__ == "__main__":
main()