Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
tree: cda3a6811b
Fetching contributors…

Cannot retrieve contributors at this time

executable file 857 lines (662 sloc) 31.805 kb
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""Module to handle downloads for different types of Firefox and Thunderbird builds."""
from datetime import datetime
from optparse import OptionParser, OptionGroup
import os
import re
import sys
import time
import urllib
import urllib2
import mozinfo
from parser import DirectoryParser
from timezones import PacificTimezone
APPLICATIONS = ['b2g', 'firefox', 'thunderbird']
# Base URL for the path to all builds
BASE_URL = 'https://ftp.mozilla.org/pub/mozilla.org'
PLATFORM_FRAGMENTS = {'linux': 'linux-i686',
'linux64': 'linux-x86_64',
'mac': 'mac',
'mac64': 'mac64',
'win32': 'win32',
'win64': 'win64-x86_64'}
DEFAULT_FILE_EXTENSIONS = {'linux': 'tar.bz2',
'linux64': 'tar.bz2',
'mac': 'dmg',
'mac64': 'dmg',
'win32': 'exe',
'win64': 'exe'}
class NotFoundException(Exception):
"""Exception for a resource not being found (e.g. no logs)"""
def __init__(self, message, location):
self.location = location
Exception.__init__(self, ': '.join([message, location]))
class TimeoutException(Exception):
"""Exception for a download exceeding the allocated timeout"""
def __init__(self):
self.message = 'The download exceeded the allocated timeout'
Exception.__init__(self, self.message)
class Scraper(object):
"""Generic class to download an application from the Mozilla server"""
def __init__(self, directory, version, platform=None,
application='firefox', locale='en-US', extension=None,
authentication=None, retry_attempts=3, retry_delay=10,
timeout=180):
# Private properties for caching
self._target = None
self._binary = None
self.directory = directory
self.locale = locale
self.platform = platform or self.detect_platform()
self.version = version
self.extension = extension or DEFAULT_FILE_EXTENSIONS[self.platform]
self.authentication = authentication
self.retry_attempts = retry_attempts
self.retry_delay = retry_delay
self.timeout = timeout
# build the base URL
self.application = application
self.base_url = '/'.join([BASE_URL, self.application])
@property
def binary(self):
"""Return the name of the build"""
if self._binary is None:
# Retrieve all entries from the remote virtual folder
parser = DirectoryParser(self.path)
if not parser.entries:
raise NotFoundException('No entries found', self.path)
# Download the first matched directory entry
pattern = re.compile(self.binary_regex, re.IGNORECASE)
for entry in parser.entries:
try:
self._binary = pattern.match(entry).group()
break
except:
# No match, continue with next entry
continue
if self._binary is None:
raise NotFoundException("Binary not found in folder", self.path)
else:
return self._binary
@property
def binary_regex(self):
"""Return the regex for the binary filename"""
raise NotImplementedError(sys._getframe(0).f_code.co_name)
@property
def final_url(self):
"""Return the final URL of the build"""
return '/'.join([self.path, self.binary])
@property
def path(self):
"""Return the path to the build"""
return '/'.join([self.base_url, self.path_regex])
@property
def path_regex(self):
"""Return the regex for the path to the build"""
raise NotImplementedError(sys._getframe(0).f_code.co_name)
@property
def platform_regex(self):
"""Return the platform fragment of the URL"""
return PLATFORM_FRAGMENTS[self.platform];
@property
def target(self):
"""Return the target file name of the build"""
if self._target is None:
self._target = os.path.join(self.directory,
self.build_filename(self.binary))
return self._target
def build_filename(self, binary):
"""Return the proposed filename with extension for the binary"""
raise NotImplementedError(sys._getframe(0).f_code.co_name)
def detect_platform(self):
"""Detect the current platform"""
# For Mac and Linux 32bit we do not need the bits appended
if mozinfo.os == 'mac' or (mozinfo.os == 'linux' and mozinfo.bits == 32):
return mozinfo.os
else:
return "%s%d" % (mozinfo.os, mozinfo.bits)
def download(self):
"""Download the specified file"""
attempts = 0
if not os.path.isdir(self.directory):
os.makedirs(self.directory)
# Don't re-download the file
if os.path.isfile(os.path.abspath(self.target)):
print "File has already been downloaded: %s" % (self.target)
return
print 'Downloading from: %s' % (urllib.unquote(self.final_url))
tmp_file = self.target + ".part"
if self.authentication \
and self.authentication['username'] \
and self.authentication['password']:
password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
password_mgr.add_password(None,
self.final_url,
self.authentication['username'],
self.authentication['password'])
handler = urllib2.HTTPBasicAuthHandler(password_mgr)
opener = urllib2.build_opener(urllib2.HTTPHandler, handler)
urllib2.install_opener(opener)
while True:
attempts += 1
try:
start_time = datetime.now()
r = urllib2.urlopen(self.final_url)
CHUNK = 16 * 1024
with open(tmp_file, 'wb') as f:
for chunk in iter(lambda: r.read(CHUNK), ''):
f.write(chunk)
t1 = (datetime.now() - start_time).total_seconds()
if t1 >= self.timeout:
raise TimeoutException
break
except (urllib2.HTTPError, urllib2.URLError, TimeoutException):
if tmp_file and os.path.isfile(tmp_file):
os.remove(tmp_file)
print 'Download failed! Retrying... (attempt %s)' % attempts
if attempts >= self.retry_attempts:
raise
time.sleep(self.retry_delay)
os.rename(tmp_file, self.target)
class DailyScraper(Scraper):
"""Class to download a daily build from the Mozilla server"""
def __init__(self, branch='mozilla-central', build_id=None, date=None,
build_number=None, *args, **kwargs):
Scraper.__init__(self, *args, **kwargs)
self.branch = branch
# Internally we access builds via index
if build_number is not None:
self.build_index = int(build_number) - 1
else:
self.build_index = None
if build_id:
# A build id has been specified. Split up its components so the date
# and time can be extracted: '20111212042025' -> '2011-12-12 04:20:25'
self.date = datetime.strptime(build_id, '%Y%m%d%H%M%S')
self.builds, self.build_index = self.get_build_info_for_date(self.date,
has_time=True)
elif date:
# A date (without time) has been specified. Use its value and the
# build index to find the requested build for that day.
self.date = datetime.strptime(date, '%Y-%m-%d')
self.builds, self.build_index = self.get_build_info_for_date(self.date,
build_index=self.build_index)
else:
# If no build id nor date have been specified the lastest available
# build of the given branch has to be identified. We also have to
# retrieve the date of the build via its build id.
url = '%s/nightly/latest-%s/' % (self.base_url, self.branch)
print 'Retrieving the build status file from %s' % url
parser = DirectoryParser(url)
parser.entries = parser.filter(r'.*%s\.txt' % self.platform_regex)
if not parser.entries:
message = 'Status file for %s build cannot be found' % self.platform_regex
raise NotFoundException(message, url)
# Read status file for the platform, retrieve build id, and convert to a date
status_file = url + parser.entries[-1]
f = urllib.urlopen(status_file)
self.date = datetime.strptime(f.readline().strip(), '%Y%m%d%H%M%S')
self.builds, self.build_index = self.get_build_info_for_date(self.date,
has_time=True)
def get_build_info_for_date(self, date, has_time=False, build_index=None):
url = '/'.join([self.base_url, self.monthly_build_list_regex])
print 'Retrieving list of builds from %s' % url
parser = DirectoryParser(url)
regex = r'%(DATE)s-(\d+-)+%(BRANCH)s%(L10N)s$' % {
'DATE': date.strftime('%Y-%m-%d'),
'BRANCH': self.branch,
'L10N': '' if self.locale == 'en-US' else '-l10n'}
parser.entries = parser.filter(regex)
if not parser.entries:
message = 'Folder for builds on %s has not been found' % self.date.strftime('%Y-%m-%d')
raise NotFoundException(message, url)
if has_time:
# If a time is included in the date, use it to determine the build's index
regex = r'.*%s.*' % date.strftime('%H-%M-%S')
build_index = parser.entries.index(parser.filter(regex)[0])
else:
# If no index has been given, set it to the last build of the day.
if build_index is None:
build_index = len(parser.entries) - 1
return (parser.entries, build_index)
@property
def binary_regex(self):
"""Return the regex for the binary"""
regex_base_name = r'^%(APP)s-.*\.%(LOCALE)s\.%(PLATFORM)s'
regex_suffix = {'linux': r'\.%(EXT)s$',
'linux64': r'\.%(EXT)s$',
'mac': r'\.%(EXT)s$',
'mac64': r'\.%(EXT)s$',
'win32': r'(\.installer)\.%(EXT)s$',
'win64': r'(\.installer)\.%(EXT)s$'}
regex = regex_base_name + regex_suffix[self.platform]
return regex % {'APP': self.application,
'LOCALE': self.locale,
'PLATFORM': self.platform_regex,
'EXT': self.extension}
def build_filename(self, binary):
"""Return the proposed filename with extension for the binary"""
try:
# Get exact timestamp of the build to build the local file name
folder = self.builds[self.build_index]
timestamp = re.search('([\d\-]+)-\D.*', folder).group(1)
except:
# If it's not available use the build's date
timestamp = self.date.strftime('%Y-%m-%d')
return '%(TIMESTAMP)s-%(BRANCH)s-%(NAME)s' % {
'TIMESTAMP': timestamp,
'BRANCH': self.branch,
'NAME': binary}
@property
def monthly_build_list_regex(self):
"""Return the regex for the folder which contains the builds of a month."""
# Regex for possible builds for the given date
return r'nightly/%(YEAR)s/%(MONTH)s/' % {
'YEAR': self.date.year,
'MONTH': str(self.date.month).zfill(2) }
@property
def path_regex(self):
"""Return the regex for the path"""
try:
return self.monthly_build_list_regex + self.builds[self.build_index]
except:
raise NotFoundException("Specified sub folder cannot be found",
self.base_url + self.monthly_build_list_regex)
class DirectScraper(Scraper):
"""Class to download a file from a specified URL"""
def __init__(self, url, *args, **kwargs):
Scraper.__init__(self, *args, **kwargs)
self.url = url
@property
def target(self):
return urllib.splitquery(self.final_url)[0].rpartition('/')[-1]
@property
def final_url(self):
return self.url
class ReleaseScraper(Scraper):
"""Class to download a release build from the Mozilla server"""
def __init__(self, *args, **kwargs):
Scraper.__init__(self, *args, **kwargs)
@property
def binary_regex(self):
"""Return the regex for the binary"""
regex = {'linux': r'^%(APP)s-.*\.%(EXT)s$',
'linux64': r'^%(APP)s-.*\.%(EXT)s$',
'mac': r'^%(APP)s.*\.%(EXT)s$',
'mac64': r'^%(APP)s.*\.%(EXT)s$',
'win32': r'^%(APP)s.*\.%(EXT)s$',
'win64': r'^%(APP)s.*\.%(EXT)s$'}
return regex[self.platform] % {'APP': self.application,
'EXT': self.extension}
@property
def path_regex(self):
"""Return the regex for the path"""
regex = r'releases/%(VERSION)s/%(PLATFORM)s/%(LOCALE)s'
return regex % {'LOCALE': self.locale,
'PLATFORM': self.platform_regex,
'VERSION': self.version}
def build_filename(self, binary):
"""Return the proposed filename with extension for the binary"""
template = '%(APP)s-%(VERSION)s.%(LOCALE)s.%(PLATFORM)s.%(EXT)s'
return template % {'APP': self.application,
'VERSION': self.version,
'LOCALE': self.locale,
'PLATFORM': self.platform,
'EXT': self.extension}
class ReleaseCandidateScraper(ReleaseScraper):
"""Class to download a release candidate build from the Mozilla server"""
def __init__(self, build_number=None, no_unsigned=False, *args, **kwargs):
Scraper.__init__(self, *args, **kwargs)
# Internally we access builds via index
if build_number is not None:
self.build_index = int(build_number) - 1
else:
self.build_index = None
self.builds, self.build_index = self.get_build_info_for_version(self.version, self.build_index)
self.no_unsigned = no_unsigned
self.unsigned = False
def get_build_info_for_version(self, version, build_index=None):
url = '/'.join([self.base_url, self.candidate_build_list_regex])
print 'Retrieving list of candidate builds from %s' % url
parser = DirectoryParser(url)
if not parser.entries:
message = 'Folder for specific candidate builds at has not been found'
raise NotFoundException(message, url)
# If no index has been given, set it to the last build of the given version.
if build_index is None:
build_index = len(parser.entries) - 1
return (parser.entries, build_index)
@property
def candidate_build_list_regex(self):
"""Return the regex for the folder which contains the builds of
a candidate build."""
# Regex for possible builds for the given date
return r'nightly/%(VERSION)s-candidates/' % {
'VERSION': self.version }
@property
def path_regex(self):
"""Return the regex for the path"""
regex = r'%(PREFIX)s%(BUILD)s/%(UNSIGNED)s%(PLATFORM)s/%(LOCALE)s'
return regex % {'PREFIX': self.candidate_build_list_regex,
'BUILD': self.builds[self.build_index],
'LOCALE': self.locale,
'PLATFORM': self.platform_regex,
'UNSIGNED': "unsigned/" if self.unsigned else ""}
def build_filename(self, binary):
"""Return the proposed filename with extension for the binary"""
template = '%(APP)s-%(VERSION)s-build%(BUILD)s.%(LOCALE)s.%(PLATFORM)s.%(EXT)s'
return template % {'APP': self.application,
'VERSION': self.version,
'BUILD': self.builds[self.build_index],
'LOCALE': self.locale,
'PLATFORM': self.platform,
'EXT': self.extension}
def download(self):
"""Download the specified file"""
try:
# Try to download the signed candidate build
Scraper.download(self)
except NotFoundException, e:
print str(e)
# If the signed build cannot be downloaded and unsigned builds are
# allowed, try to download the unsigned build instead
if self.no_unsigned:
raise
else:
print "Signed build has not been found. Falling back to unsigned build."
self.unsigned = True
Scraper.download(self)
class TinderboxScraper(Scraper):
"""Class to download a tinderbox build from the Mozilla server.
There are two ways to specify a unique build:
1. If the date (%Y-%m-%d) is given and build_number is given where
the build_number is the index of the build on the date
2. If the build timestamp (UNIX) is given, and matches a specific build.
"""
def __init__(self, branch='mozilla-central', build_number=None, date=None,
debug_build=False, *args, **kwargs):
Scraper.__init__(self, *args, **kwargs)
self.branch = branch
self.debug_build = debug_build
self.locale_build = self.locale != 'en-US'
self.timestamp = None
# Currently any time in RelEng is based on the Pacific time zone.
self.timezone = PacificTimezone();
# Internally we access builds via index
if build_number is not None:
self.build_index = int(build_number) - 1
else:
self.build_index = None
if date is not None:
try:
self.date = datetime.fromtimestamp(float(date), self.timezone)
self.timestamp = date
except:
self.date = datetime.strptime(date, '%Y-%m-%d')
else:
self.date = None
# For localized builds we do not have to retrieve the list of builds
# because only the last build is available
if not self.locale_build:
self.builds, self.build_index = self.get_build_info(self.build_index)
try:
self.timestamp = self.builds[self.build_index]
except:
raise NotFoundException("Specified sub folder cannot be found",
self.base_url + self.monthly_build_list_regex)
@property
def binary_regex(self):
"""Return the regex for the binary"""
regex_base_name = r'^%(APP)s-.*\.%(LOCALE)s\.'
regex_suffix = {'linux': r'.*\.%(EXT)s$',
'linux64': r'.*\.%(EXT)s$',
'mac': r'.*\.%(EXT)s$',
'mac64': r'.*\.%(EXT)s$',
'win32': r'.*(\.installer)\.%(EXT)s$',
'win64': r'.*(\.installer)\.%(EXT)s$'}
regex = regex_base_name + regex_suffix[self.platform]
return regex % {'APP': self.application,
'LOCALE': self.locale,
'EXT': self.extension}
def build_filename(self, binary):
"""Return the proposed filename with extension for the binary"""
return '%(TIMESTAMP)s%(BRANCH)s%(DEBUG)s-%(NAME)s' % {
'TIMESTAMP': self.timestamp + '-' if self.timestamp else '',
'BRANCH': self.branch,
'DEBUG': '-debug' if self.debug_build else '',
'NAME': binary}
@property
def build_list_regex(self):
"""Return the regex for the folder which contains the list of builds"""
regex = 'tinderbox-builds/%(BRANCH)s-%(PLATFORM)s%(L10N)s%(DEBUG)s'
return regex % {'BRANCH': self.branch,
'PLATFORM': '' if self.locale_build else self.platform_regex,
'L10N': 'l10n' if self.locale_build else '',
'DEBUG': '-debug' if self.debug_build else ''}
def date_matches(self, timestamp):
"""Determines whether the timestamp date is equal to the argument date"""
if self.date is None:
return False
timestamp = datetime.fromtimestamp(float(timestamp), self.timezone)
if self.date.date() == timestamp.date():
return True
return False
@property
def date_validation_regex(self):
"""Return the regex for a valid date argument value"""
return r'^\d{4}-\d{1,2}-\d{1,2}$|^\d+$'
def detect_platform(self):
"""Detect the current platform"""
platform = Scraper.detect_platform(self)
# On OS X we have to special case the platform detection code and fallback
# to 64 bit builds for the en-US locale
if mozinfo.os == 'mac' and self.locale == 'en-US' and mozinfo.bits == 64:
platform = "%s%d" % (mozinfo.os, mozinfo.bits)
return platform
def get_build_info(self, build_index=None):
url = '/'.join([self.base_url, self.build_list_regex])
print 'Retrieving list of builds from %s' % url
# If a timestamp is given, retrieve just that build
regex = '^' + self.timestamp + '$' if self.timestamp else r'^\d+$'
parser = DirectoryParser(url)
parser.entries = parser.filter(regex)
# If date is given, retrieve the subset of builds on that date
if self.date is not None:
parser.entries = filter(self.date_matches, parser.entries)
if not parser.entries:
message = 'No builds have been found'
raise NotFoundException(message, url)
# If no index has been given, set it to the last build of the day.
if build_index is None:
build_index = len(parser.entries) - 1
return (parser.entries, build_index)
@property
def path_regex(self):
"""Return the regex for the path"""
if self.locale_build:
return self.build_list_regex
return '/'.join([self.build_list_regex, self.builds[self.build_index]])
@property
def platform_regex(self):
"""Return the platform fragment of the URL"""
PLATFORM_FRAGMENTS = {'linux': 'linux',
'linux64': 'linux64',
'mac': 'macosx',
'mac64': 'macosx64',
'win32': 'win32',
'win64': 'win64'}
return PLATFORM_FRAGMENTS[self.platform]
def cli():
"""Main function for the downloader"""
BUILD_TYPES = {'release': ReleaseScraper,
'candidate': ReleaseCandidateScraper,
'daily': DailyScraper,
'tinderbox': TinderboxScraper }
usage = 'usage: %prog [options]'
parser = OptionParser(usage=usage, description=__doc__)
parser.add_option('--application', '-a',
dest='application',
choices=APPLICATIONS,
default='firefox',
metavar='APPLICATION',
help='The name of the application to download, '
'default: "%default"')
parser.add_option('--directory', '-d',
dest='directory',
default=os.getcwd(),
metavar='DIRECTORY',
help='Target directory for the download, default: '
'current working directory')
parser.add_option('--build-number',
dest='build_number',
default=None,
type="int",
metavar='BUILD_NUMBER',
help='Number of the build (for candidate, daily, '
'and tinderbox builds)')
parser.add_option('--locale', '-l',
dest='locale',
default='en-US',
metavar='LOCALE',
help='Locale of the application, default: "%default"')
parser.add_option('--platform', '-p',
dest='platform',
choices=PLATFORM_FRAGMENTS.keys(),
metavar='PLATFORM',
help='Platform of the application')
parser.add_option('--type', '-t',
dest='type',
choices=BUILD_TYPES.keys(),
default='release',
metavar='BUILD_TYPE',
help='Type of build to download, default: "%default"')
parser.add_option('--url',
dest='url',
default=None,
metavar='URL',
help='URL to download.')
parser.add_option('--version', '-v',
dest='version',
metavar='VERSION',
help='Version of the application to be used by release and\
candidate builds, i.e. "3.6"')
parser.add_option('--extension',
dest='extension',
default=None,
metavar='EXTENSION',
help='File extension of the build (e.g. "zip"), default:\
the standard build extension on the platform.')
parser.add_option('--username',
dest='username',
default=None,
metavar='USERNAME',
help='Username for basic HTTP authentication.')
parser.add_option('--password',
dest='password',
default=None,
metavar='PASSWORD',
help='Password for basic HTTP authentication.')
parser.add_option('--retry-attempts',
dest='retry_attempts',
default=3,
type=int,
metavar='RETRY_ATTEMPTS',
help='Number of times the download will be attempted in '
'the event of a failure, default: %default')
parser.add_option('--retry-delay',
dest='retry_delay',
default=10,
type=int,
metavar='RETRY_DELAY',
help='Amount of time (in seconds) to wait between retry '
'attempts, default: %default')
parser.add_option('--timeout',
dest='timeout',
default=180,
type=int,
metavar='TIMEOUT',
help='Amount of time (in seconds) until download times '
'out, default: %default')
# Option group for candidate builds
group = OptionGroup(parser, "Candidate builds",
"Extra options for candidate builds.")
group.add_option('--no-unsigned',
dest='no_unsigned',
action="store_true",
help="Don't allow to download unsigned builds if signed\
builds are not available")
parser.add_option_group(group)
# Option group for daily builds
group = OptionGroup(parser, "Daily builds",
"Extra options for daily builds.")
group.add_option('--branch',
dest='branch',
default='mozilla-central',
metavar='BRANCH',
help='Name of the branch, default: "%default"')
group.add_option('--build-id',
dest='build_id',
default=None,
metavar='BUILD_ID',
help='ID of the build to download')
group.add_option('--date',
dest='date',
default=None,
metavar='DATE',
help='Date of the build, default: latest build')
parser.add_option_group(group)
# Option group for tinderbox builds
group = OptionGroup(parser, "Tinderbox builds",
"Extra options for tinderbox builds.")
group.add_option('--debug-build',
dest='debug_build',
action="store_true",
help="Download a debug build")
parser.add_option_group(group)
# TODO: option group for nightly builds
(options, args) = parser.parse_args()
# Check for required options and arguments
# Note: Will be optional when ini file support has been landed
if not options.url \
and not options.type in ['daily', 'tinderbox'] \
and not options.version:
parser.error('The version of the application to download has not been specified.')
# Instantiate scraper and download the build
scraper_keywords = {'application': options.application,
'locale': options.locale,
'platform': options.platform,
'version': options.version,
'directory': options.directory,
'extension': options.extension,
'authentication': {
'username': options.username,
'password': options.password},
'retry_attempts': options.retry_attempts,
'retry_delay': options.retry_delay,
'timeout': options.timeout}
scraper_options = {'candidate': {
'build_number': options.build_number,
'no_unsigned': options.no_unsigned},
'daily': {
'branch': options.branch,
'build_number': options.build_number,
'build_id': options.build_id,
'date': options.date},
'tinderbox': {
'branch': options.branch,
'build_number': options.build_number,
'date': options.date,
'debug_build': options.debug_build}
}
kwargs = scraper_keywords.copy()
kwargs.update(scraper_options.get(options.type, {}))
if options.url:
build = DirectScraper(options.url, **kwargs)
else:
build = BUILD_TYPES[options.type](**kwargs)
build.download()
if __name__ == "__main__":
cli()
Jump to Line
Something went wrong with that request. Please try again.