diff --git a/mozdownload/factory.py b/mozdownload/factory.py index 9efc57c5..456f179b 100644 --- a/mozdownload/factory.py +++ b/mozdownload/factory.py @@ -67,6 +67,11 @@ def __init__(self, scraper_type, **kwargs): error_msg = '%s build is not yet supported for fennec' % scraper_type raise NotSupportedError(error_msg) + if (kwargs.get('application') == 'devedition' + and scraper_type not in ('release', 'candidate')): + error_msg = '%s build is not yet supported for Firefox Developer Edition' % scraper_type + raise NotSupportedError(error_msg) + # Instantiate scraper and download the build scraper_keywords = {'application': kwargs.get('application', 'firefox'), 'base_url': kwargs.get('base_url', scraper.BASE_URL), diff --git a/mozdownload/scraper.py b/mozdownload/scraper.py index e72a7531..f7eb35e1 100755 --- a/mozdownload/scraper.py +++ b/mozdownload/scraper.py @@ -26,13 +26,15 @@ from mozdownload.utils import urljoin -APPLICATIONS = ('firefox', 'fennec', 'thunderbird') +APPLICATIONS = ('devedition', 'firefox', 'fennec', 'thunderbird') # Some applications contain all locales in a single build APPLICATIONS_MULTI_LOCALE = ('fennec') # Used if the application is named differently than the subfolder on the server APPLICATIONS_TO_FTP_DIRECTORY = {'fennec': 'mobile'} +# Used if the application is named differently then the binary on the server +APPLICATIONS_TO_BINARY_NAME = {'devedition': 'firefox'} # Base URL for the path to all builds BASE_URL = 'https://archive.mozilla.org/pub/' @@ -500,7 +502,7 @@ def get_build_info_for_date(self, date, build_index=None): @property def binary_regex(self): """Return the regex for the binary.""" - regex_base_name = (r'^%(APP)s(\s%(STUB_NEW)s\.%(LOCALE)s|' + + regex_base_name = (r'^%(BINARY_NAME)s(\s%(STUB_NEW)s\.%(LOCALE)s|' + r'-.*\.%(LOCALE)s\.%(PLATFORM)s)') regex_suffix = {'android-api-9': r'\.%(EXT)s$', 'android-api-11': r'\.%(EXT)s$', @@ -515,7 +517,8 @@ def binary_regex(self): 'win64': r'(\.installer%(STUB)s)?\.%(EXT)s$'} regex = regex_base_name + regex_suffix[self.platform] - return regex % {'APP': self.application, + return regex % {'BINARY_NAME': APPLICATIONS_TO_BINARY_NAME.get(self.application, + self.application), 'LOCALE': self.locale, 'PLATFORM': self.platform_regex, 'EXT': self.extension, @@ -603,15 +606,17 @@ def __init__(self, version, *args, **kwargs): @property def binary_regex(self): """Return the regex for the binary.""" - regex = {'linux': r'^%(APP)s-%(VERSION)s\.%(EXT)s$', - 'linux64': r'^%(APP)s-%(VERSION)s\.%(EXT)s$', - 'mac': r'^%(APP)s(?:\s|-)%(VERSION)s\.%(EXT)s$', - 'mac64': r'^%(APP)s(?:\s|-)%(VERSION)s\.%(EXT)s$', - 'win32': r'^%(APP)s(%(STUB_NEW)s|(?:\sSetup\s|-)%(STUB)s%(VERSION)s)\.%(EXT)s$', - 'win64': r'^%(APP)s(%(STUB_NEW)s|(?:\sSetup\s|-)%(STUB)s%(VERSION)s)\.%(EXT)s$', + regex = {'linux': r'^%(BINARY_NAME)s-%(VERSION)s\.%(EXT)s$', + 'linux64': r'^%(BINARY_NAME)s-%(VERSION)s\.%(EXT)s$', + 'mac': r'^%(BINARY_NAME)s(?:\s|-)%(VERSION)s\.%(EXT)s$', + 'mac64': r'^%(BINARY_NAME)s(?:\s|-)%(VERSION)s\.%(EXT)s$', + 'win32': + r'^%(BINARY_NAME)s(%(STUB_NEW)s|(?:\sSetup\s|-)%(STUB)s%(VERSION)s)\.%(EXT)s$', + 'win64': + r'^%(BINARY_NAME)s(%(STUB_NEW)s|(?:\sSetup\s|-)%(STUB)s%(VERSION)s)\.%(EXT)s$', } return regex[self.platform] % { - 'APP': self.application, + 'BINARY_NAME': APPLICATIONS_TO_BINARY_NAME.get(self.application, self.application), 'EXT': self.extension, 'STUB': 'Stub ' if self.is_stub_installer else '', 'STUB_NEW': ' Installer' if self.is_stub_installer else '', @@ -809,7 +814,7 @@ def get_build_info(self): @property def binary_regex(self): """Return the regex for the binary.""" - regex_base_name = (r'^(%(STUB_NEW)s|%(APP)s-.*\.%(LOCALE)s\.%(PLATFORM)s)') + regex_base_name = (r'^(%(STUB_NEW)s|%(BINARY_NAME)s-.*\.%(LOCALE)s\.%(PLATFORM)s)') regex_suffix = {'linux': r'.*\.%(EXT)s$', 'linux64': r'.*\.%(EXT)s$', 'mac': r'.*\.%(EXT)s$', @@ -819,7 +824,8 @@ def binary_regex(self): regex = regex_base_name + regex_suffix[self.platform] - return regex % {'APP': self.application, + return regex % {'BINARY_NAME': APPLICATIONS_TO_BINARY_NAME.get(self.application, + self.application), 'LOCALE': self.locale, 'PLATFORM': PLATFORM_FRAGMENTS[self.platform], 'STUB': '-stub' if self.is_stub_installer else '', @@ -983,7 +989,7 @@ def get_build_info(self): @property def binary_regex(self): """Return the regex for the binary.""" - regex_base_name = (r'^(%(STUB_NEW)s|%(APP)s-.*\.%(LOCALE)s\.%(PLATFORM)s)') + regex_base_name = (r'^(%(STUB_NEW)s|%(BINARY_NAME)s-.*\.%(LOCALE)s\.%(PLATFORM)s)') regex_suffix = {'linux': r'.*\.%(EXT)s$', 'linux64': r'.*\.%(EXT)s$', 'mac': r'.*\.%(EXT)s$', @@ -993,7 +999,8 @@ def binary_regex(self): regex = regex_base_name + regex_suffix[self.platform] - return regex % {'APP': self.application, + return regex % {'BINARY_NAME': APPLICATIONS_TO_BINARY_NAME.get(self.application, + self.application), 'LOCALE': self.locale, 'PLATFORM': PLATFORM_FRAGMENTS[self.platform], 'STUB': '-stub' if self.is_stub_installer else '', diff --git a/tests/factory/test_factory_invalid_options.py b/tests/factory/test_factory_invalid_options.py index 5aaab390..a1696184 100644 --- a/tests/factory/test_factory_invalid_options.py +++ b/tests/factory/test_factory_invalid_options.py @@ -6,6 +6,7 @@ from mozdownload import FactoryScraper from mozdownload.utils import urljoin +from mozdownload.errors import NotSupportedError import mozhttpd_base_test as mhttpd @@ -115,6 +116,26 @@ def test_try_without_revision(self): base_url=self.wdir, logger=self.logger) + def test_non_daily_fennec(self): + """Test that non-daily scrapper_type for fennec raises exception""" + self.assertRaises(NotSupportedError, FactoryScraper, + scraper_type='candidate', + destination=self.temp_dir, + base_url=self.wdir, + logger=self.logger, + application='fennec', + version='60.0b1') + + def test_non_release_non_candidate_devedition(self): + """Test that non-relase and non-candidate scrapper type for devedition raises exception""" + self.assertRaises(NotSupportedError, FactoryScraper, + scraper_type='daily', + destination=self.temp_dir, + base_url=self.wdir, + logger=self.logger, + application='devedition', + version='60.0b1') + class TestFactoryUnusedOptions(mhttpd.MozHttpdBaseTest): diff --git a/tests/remote/test_devedition.py b/tests/remote/test_devedition.py new file mode 100644 index 00000000..da943456 --- /dev/null +++ b/tests/remote/test_devedition.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Test all scraper classes for Firefox Developer Edition against the remote server""" + +import urllib + +import pytest + +import mozdownload +from mozdownload.scraper import BASE_URL +from mozdownload.utils import urljoin + + +@pytest.mark.parametrize("args,url", [ + ({'application': 'devedition', 'platform': 'linux', 'version': '60.0b1'}, + 'devedition/releases/60.0b1/linux-i686/en-US/firefox-60.0b1.tar.bz2'), + ({'application': 'devedition', 'platform': 'linux64', 'version': '60.0b1'}, + 'devedition/releases/60.0b1/linux-x86_64/en-US/firefox-60.0b1.tar.bz2'), + ({'application': 'devedition', 'platform': 'mac', 'version': '60.0b1'}, + 'devedition/releases/60.0b1/mac/en-US/Firefox 60.0b1.dmg'), + ({'application': 'devedition', 'platform': 'win32', 'version': '60.0b1'}, + 'devedition/releases/60.0b1/win32/en-US/Firefox Setup 60.0b1.exe'), + ({'application': 'devedition', 'platform': 'win64', 'version': '60.0b1'}, + 'devedition/releases/60.0b1/win64/en-US/Firefox Setup 60.0b1.exe'), + ({'application': 'devedition', 'platform': 'win32', 'version': '60.0b1', 'locale': 'de'}, + 'devedition/releases/60.0b1/win32/de/Firefox Setup 60.0b1.exe'), +]) +def test_release_scraper(tmpdir, args, url): + """Test release scraper against the remote server.""" + scraper = mozdownload.ReleaseScraper(destination=tmpdir, **args) + + if url: + assert urllib.unquote(scraper.url) == urljoin(BASE_URL, url) + + +@pytest.mark.parametrize("args,url", [ + ({'application': 'devedition', 'platform': 'linux', 'version': '60.0b1', 'build_number': 1}, + 'devedition/candidates/60.0b1-candidates/build3/linux-i686/en-US/firefox-60.0b1.tar.bz2'), + ({'application': 'devedition', 'platform': 'linux64', 'version': '60.0b1', 'build_number': 1}, + 'devedition/candidates/60.0b1-candidates/build3/linux-x86_64/en-US/firefox-60.0b1.tar.bz2'), # noqa + ({'application': 'devedition', 'platform': 'mac', 'version': '60.0b1', 'build_number': 1}, + 'devedition/candidates/60.0b1-candidates/build3/mac/en-US/Firefox 60.0b1.dmg'), + ({'application': 'devedition', 'platform': 'win32', 'version': '60.0b1', 'build_number': 1}, + 'devedition/candidates/60.0b1-candidates/build3/win32/en-US/Firefox Setup 60.0b1.exe'), + ({'application': 'devedition', 'platform': 'mac', 'version': '60.0b1', 'build_number': 1, + 'locale': 'de'}, + 'devedition/candidates/60.0b1-candidates/build3/mac/de/Firefox 60.0b1.dmg'), + ({'application': 'devedition', 'platform': 'mac', 'version': '60.0b1', 'build_number': 1, + 'extension': 'json'}, + 'devedition/candidates/60.0b1-candidates/build3/mac/en-US/firefox-60.0b1.json'), +]) +def test_candidate_scraper(tmpdir, args, url): + """Test release candidate scraper against the remote server.""" + scraper = mozdownload.ReleaseCandidateScraper(destination=tmpdir, **args) + + assert urllib.unquote(scraper.url) == urljoin(BASE_URL, url)