From 7762df26ddf6f3b5a7e42de7711e51ee71965d49 Mon Sep 17 00:00:00 2001 From: Henrik Skupin Date: Wed, 23 May 2012 19:36:53 +0100 Subject: [PATCH] Combine mozdownload.py and scraper.py into a single file. Fixes issue #2 --- mozdownload/__init__.py | 2 +- mozdownload/mozdownload.py | 149 ---------------------------------- mozdownload/scraper.py | 159 ++++++++++++++++++++++++++++++++++--- setup.py | 2 + 4 files changed, 151 insertions(+), 161 deletions(-) delete mode 100755 mozdownload/mozdownload.py mode change 100644 => 100755 mozdownload/scraper.py mode change 100644 => 100755 setup.py diff --git a/mozdownload/__init__.py b/mozdownload/__init__.py index 5397a997..604bfe41 100644 --- a/mozdownload/__init__.py +++ b/mozdownload/__init__.py @@ -2,4 +2,4 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from mozdownload import * +from scraper import * diff --git a/mozdownload/mozdownload.py b/mozdownload/mozdownload.py deleted file mode 100755 index a12b7762..00000000 --- a/mozdownload/mozdownload.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python - -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -"""Script to download builds for Firefox and Thunderbird from the Mozilla server.""" - - -from optparse import OptionParser, OptionGroup -import os -import sys - -import scraper - - -APPLICATIONS = ['firefox', 'thunderbird'] - -BUILD_TYPES = {'release': scraper.ReleaseScraper, - 'candidate': scraper.ReleaseCandidateScraper, - 'daily': scraper.DailyScraper, - 'tinderbox': scraper.TinderboxScraper } - - -def cli(): - """Main function for the downloader""" - - usage = 'usage: %prog [options]' - parser = OptionParser(usage=usage, description=__doc__) - parser.add_option('--application', '-a', - dest='application', - choices=APPLICATIONS, - default=APPLICATIONS[0], - metavar='APPLICATION', - help='The name of the application to download, ' - 'default: "%s"' % APPLICATIONS[0]) - parser.add_option('--directory', '-d', - dest='directory', - default=os.getcwd(), - metavar='DIRECTORY', - help='Target directory for the download, default: ' - 'current working directory') - parser.add_option('--build-number', - dest='build_number', - default=None, - type="int", - metavar='BUILD_NUMBER', - help='Number of the build (for candidate, daily, ' - 'and tinderbox builds)') - parser.add_option('--locale', '-l', - dest='locale', - default='en-US', - metavar='LOCALE', - help='Locale of the application, default: "en-US"') - parser.add_option('--platform', '-p', - dest='platform', - choices=scraper.PLATFORM_FRAGMENTS.keys(), - metavar='PLATFORM', - help='Platform of the application') - parser.add_option('--type', '-t', - dest='type', - choices=BUILD_TYPES.keys(), - default=BUILD_TYPES.keys()[0], - metavar='BUILD_TYPE', - help='Type of build to download, default: "%s"' % - BUILD_TYPES.keys()[0]) - parser.add_option('--version', '-v', - dest='version', - metavar='VERSION', - help='Version of the application to be used by release and\ - candidate builds, i.e. "3.6"') - - # Option group for candidate builds - group = OptionGroup(parser, "Candidate builds", - "Extra options for candidate builds.") - group.add_option('--no-unsigned', - dest='no_unsigned', - action="store_true", - help="Don't allow to download unsigned builds if signed\ - builds are not available") - parser.add_option_group(group) - - # Option group for daily builds - group = OptionGroup(parser, "Daily builds", - "Extra options for daily builds.") - group.add_option('--branch', - dest='branch', - default='mozilla-central', - metavar='BRANCH', - help='Name of the branch, default: "mozilla-central"') - parser.add_option('--build-id', - dest='build_id', - default=None, - metavar='BUILD_ID', - help='ID of the build to download') - group.add_option('--date', - dest='date', - default=None, - metavar='DATE', - help='Date of the build, default: latest build') - parser.add_option_group(group) - - # Option group for tinderbox builds - group = OptionGroup(parser, "Tinderbox builds", - "Extra options for tinderbox builds.") - group.add_option('--debug-build', - dest='debug_build', - action="store_true", - help="Download a debug build") - parser.add_option_group(group) - - # TODO: option group for nightly builds - (options, args) = parser.parse_args() - - # Check for required options and arguments - # Note: Will be optional when ini file support has been landed - if not options.type in ['daily', 'tinderbox'] \ - and not options.version: - parser.error('The version of the application to download has not been specified.') - - # Instantiate scraper and download the build - scraper_keywords = {'application': options.application, - 'locale': options.locale, - 'platform': options.platform, - 'version': options.version, - 'directory': options.directory} - scraper_options = {'candidate': { - 'build_number': options.build_number, - 'no_unsigned': options.no_unsigned}, - 'daily': { - 'branch': options.branch, - 'build_number': options.build_number, - 'build_id': options.build_id, - 'date': options.date}, - 'tinderbox': { - 'branch': options.branch, - 'build_number': options.build_number, - 'date': options.date, - 'debug_build': options.debug_build} - } - - kwargs = scraper_keywords.copy() - kwargs.update(scraper_options.get(options.type, {})) - build = BUILD_TYPES[options.type](**kwargs) - - build.download() - -if __name__ == "__main__": - cli() diff --git a/mozdownload/scraper.py b/mozdownload/scraper.py old mode 100644 new mode 100755 index 24a2415c..5bc3f1ea --- a/mozdownload/scraper.py +++ b/mozdownload/scraper.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -6,6 +8,7 @@ from datetime import datetime +from optparse import OptionParser, OptionGroup import os import re import sys @@ -17,6 +20,8 @@ from timezones import PacificTimezone +APPLICATIONS = ['firefox', 'thunderbird'] + # Base URL for the path to all builds BASE_URL = 'https://ftp.mozilla.org/pub/mozilla.org' @@ -35,7 +40,7 @@ def __init__(self, message, location): Exception.__init__(self, ': '.join([message, location])) -class MozillaScraper(object): +class Scraper(object): """Generic class to download an application from the Mozilla server""" def __init__(self, directory, version, platform=None, @@ -183,13 +188,13 @@ def download(self): raise -class DailyScraper(MozillaScraper): +class DailyScraper(Scraper): """Class to download a daily build from the Mozilla server""" def __init__(self, branch='mozilla-central', build_id=None, date=None, build_number=None, *args, **kwargs): - MozillaScraper.__init__(self, *args, **kwargs) + Scraper.__init__(self, *args, **kwargs) self.branch = branch # Internally we access builds via index @@ -312,11 +317,11 @@ def path_regex(self): self.base_url + self.monthly_build_list_regex) -class ReleaseScraper(MozillaScraper): +class ReleaseScraper(Scraper): """Class to download a release build from the Mozilla server""" def __init__(self, *args, **kwargs): - MozillaScraper.__init__(self, *args, **kwargs) + Scraper.__init__(self, *args, **kwargs) @property def binary_regex(self): @@ -356,7 +361,7 @@ class ReleaseCandidateScraper(ReleaseScraper): """Class to download a release candidate build from the Mozilla server""" def __init__(self, build_number=None, no_unsigned=False, *args, **kwargs): - MozillaScraper.__init__(self, *args, **kwargs) + Scraper.__init__(self, *args, **kwargs) # Internally we access builds via index if build_number is not None: @@ -425,7 +430,7 @@ def download(self): try: # Try to download the signed candidate build - MozillaScraper.download(self) + Scraper.download(self) except NotFoundException, e: print str(e) @@ -436,10 +441,10 @@ def download(self): else: print "Signed build has not been found. Falling back to unsigned build." self.unsigned = True - MozillaScraper.download(self) + Scraper.download(self) -class TinderboxScraper(MozillaScraper): +class TinderboxScraper(Scraper): """Class to download a tinderbox build from the Mozilla server. There are two ways to specify a unique build: @@ -450,7 +455,7 @@ class TinderboxScraper(MozillaScraper): def __init__(self, branch='mozilla-central', build_number=None, date=None, debug_build=False, *args, **kwargs): - MozillaScraper.__init__(self, *args, **kwargs) + Scraper.__init__(self, *args, **kwargs) self.branch = branch self.debug_build = debug_build @@ -550,7 +555,7 @@ def date_validation_regex(self): def detect_platform(self): """Detect the current platform""" - platform = MozillaScraper.detect_platform(self) + platform = Scraper.detect_platform(self) # On OS X we have to special case the platform detection code and fallback # to 64 bit builds for the en-US locale @@ -608,3 +613,135 @@ def platform_regex(self): 'win64': 'win64'} return PLATFORM_FRAGMENTS[self.platform] + + +def cli(): + """Main function for the downloader""" + + BUILD_TYPES = {'release': ReleaseScraper, + 'candidate': ReleaseCandidateScraper, + 'daily': DailyScraper, + 'tinderbox': TinderboxScraper } + + usage = 'usage: %prog [options]' + parser = OptionParser(usage=usage, description=__doc__) + parser.add_option('--application', '-a', + dest='application', + choices=APPLICATIONS, + default=APPLICATIONS[0], + metavar='APPLICATION', + help='The name of the application to download, ' + 'default: "%s"' % APPLICATIONS[0]) + parser.add_option('--directory', '-d', + dest='directory', + default=os.getcwd(), + metavar='DIRECTORY', + help='Target directory for the download, default: ' + 'current working directory') + parser.add_option('--build-number', + dest='build_number', + default=None, + type="int", + metavar='BUILD_NUMBER', + help='Number of the build (for candidate, daily, ' + 'and tinderbox builds)') + parser.add_option('--locale', '-l', + dest='locale', + default='en-US', + metavar='LOCALE', + help='Locale of the application, default: "en-US"') + parser.add_option('--platform', '-p', + dest='platform', + choices=PLATFORM_FRAGMENTS.keys(), + metavar='PLATFORM', + help='Platform of the application') + parser.add_option('--type', '-t', + dest='type', + choices=BUILD_TYPES.keys(), + default=BUILD_TYPES.keys()[0], + metavar='BUILD_TYPE', + help='Type of build to download, default: "%s"' % + BUILD_TYPES.keys()[0]) + parser.add_option('--version', '-v', + dest='version', + metavar='VERSION', + help='Version of the application to be used by release and\ + candidate builds, i.e. "3.6"') + + # Option group for candidate builds + group = OptionGroup(parser, "Candidate builds", + "Extra options for candidate builds.") + group.add_option('--no-unsigned', + dest='no_unsigned', + action="store_true", + help="Don't allow to download unsigned builds if signed\ + builds are not available") + parser.add_option_group(group) + + # Option group for daily builds + group = OptionGroup(parser, "Daily builds", + "Extra options for daily builds.") + group.add_option('--branch', + dest='branch', + default='mozilla-central', + metavar='BRANCH', + help='Name of the branch, default: "mozilla-central"') + parser.add_option('--build-id', + dest='build_id', + default=None, + metavar='BUILD_ID', + help='ID of the build to download') + group.add_option('--date', + dest='date', + default=None, + metavar='DATE', + help='Date of the build, default: latest build') + parser.add_option_group(group) + + # Option group for tinderbox builds + group = OptionGroup(parser, "Tinderbox builds", + "Extra options for tinderbox builds.") + group.add_option('--debug-build', + dest='debug_build', + action="store_true", + help="Download a debug build") + parser.add_option_group(group) + + # TODO: option group for nightly builds + (options, args) = parser.parse_args() + + # Check for required options and arguments + # Note: Will be optional when ini file support has been landed + if not options.type in ['daily', 'tinderbox'] \ + and not options.version: + parser.error('The version of the application to download has not been specified.') + + # Instantiate scraper and download the build + scraper_keywords = {'application': options.application, + 'locale': options.locale, + 'platform': options.platform, + 'version': options.version, + 'directory': options.directory} + scraper_options = {'candidate': { + 'build_number': options.build_number, + 'no_unsigned': options.no_unsigned}, + 'daily': { + 'branch': options.branch, + 'build_number': options.build_number, + 'build_id': options.build_id, + 'date': options.date}, + 'tinderbox': { + 'branch': options.branch, + 'build_number': options.build_number, + 'date': options.date, + 'debug_build': options.debug_build} + } + + kwargs = scraper_keywords.copy() + kwargs.update(scraper_options.get(options.type, {})) + build = BUILD_TYPES[options.type](**kwargs) + + build.download() + +if __name__ == "__main__": + cli() diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 index 73d419f2..7bfc6609 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/.