Added Support for Firefox Developer Edition (#504)

mozdownload/factory.py

@@ -67,6 +67,11 @@ def __init__(self, scraper_type, **kwargs):
             error_msg = '%s build is not yet supported for fennec' % scraper_type
             raise NotSupportedError(error_msg)
 
+        if (kwargs.get('application') == 'devedition'
+                and scraper_type not in ('release', 'candidate')):
+            error_msg = '%s build is not yet supported for Firefox Developer Edition' % scraper_type
+            raise NotSupportedError(error_msg)
+
         # Instantiate scraper and download the build
         scraper_keywords = {'application': kwargs.get('application', 'firefox'),
                             'base_url': kwargs.get('base_url', scraper.BASE_URL),

mozdownload/scraper.py

@@ -26,13 +26,15 @@
 from mozdownload.utils import urljoin
 
 
-APPLICATIONS = ('firefox', 'fennec', 'thunderbird')
+APPLICATIONS = ('devedition', 'firefox', 'fennec', 'thunderbird')
 
 # Some applications contain all locales in a single build
 APPLICATIONS_MULTI_LOCALE = ('fennec')
 
 # Used if the application is named differently than the subfolder on the server
 APPLICATIONS_TO_FTP_DIRECTORY = {'fennec': 'mobile'}
+# Used if the application is named differently then the binary on the server
+APPLICATIONS_TO_BINARY_NAME = {'devedition': 'firefox'}
 
 # Base URL for the path to all builds
 BASE_URL = 'https://archive.mozilla.org/pub/'
@@ -500,7 +502,7 @@ def get_build_info_for_date(self, date, build_index=None):
     @property
     def binary_regex(self):
         """Return the regex for the binary."""
-        regex_base_name = (r'^%(APP)s(\s%(STUB_NEW)s\.%(LOCALE)s|' +
+        regex_base_name = (r'^%(BINARY_NAME)s(\s%(STUB_NEW)s\.%(LOCALE)s|' +
                            r'-.*\.%(LOCALE)s\.%(PLATFORM)s)')
         regex_suffix = {'android-api-9': r'\.%(EXT)s$',
                         'android-api-11': r'\.%(EXT)s$',
@@ -515,7 +517,8 @@ def binary_regex(self):
                         'win64': r'(\.installer%(STUB)s)?\.%(EXT)s$'}
         regex = regex_base_name + regex_suffix[self.platform]
 
-        return regex % {'APP': self.application,
+        return regex % {'BINARY_NAME': APPLICATIONS_TO_BINARY_NAME.get(self.application,
+                                                                       self.application),
                         'LOCALE': self.locale,
                         'PLATFORM': self.platform_regex,
                         'EXT': self.extension,
@@ -603,15 +606,17 @@ def __init__(self, version, *args, **kwargs):
     @property
     def binary_regex(self):
         """Return the regex for the binary."""
-        regex = {'linux': r'^%(APP)s-%(VERSION)s\.%(EXT)s$',
-                 'linux64': r'^%(APP)s-%(VERSION)s\.%(EXT)s$',
-                 'mac': r'^%(APP)s(?:\s|-)%(VERSION)s\.%(EXT)s$',
-                 'mac64': r'^%(APP)s(?:\s|-)%(VERSION)s\.%(EXT)s$',
-                 'win32': r'^%(APP)s(%(STUB_NEW)s|(?:\sSetup\s|-)%(STUB)s%(VERSION)s)\.%(EXT)s$',
-                 'win64': r'^%(APP)s(%(STUB_NEW)s|(?:\sSetup\s|-)%(STUB)s%(VERSION)s)\.%(EXT)s$',
+        regex = {'linux': r'^%(BINARY_NAME)s-%(VERSION)s\.%(EXT)s$',
+                 'linux64': r'^%(BINARY_NAME)s-%(VERSION)s\.%(EXT)s$',
+                 'mac': r'^%(BINARY_NAME)s(?:\s|-)%(VERSION)s\.%(EXT)s$',
+                 'mac64': r'^%(BINARY_NAME)s(?:\s|-)%(VERSION)s\.%(EXT)s$',
+                 'win32':
+                     r'^%(BINARY_NAME)s(%(STUB_NEW)s|(?:\sSetup\s|-)%(STUB)s%(VERSION)s)\.%(EXT)s$',
+                 'win64':
+                     r'^%(BINARY_NAME)s(%(STUB_NEW)s|(?:\sSetup\s|-)%(STUB)s%(VERSION)s)\.%(EXT)s$',
                  }
         return regex[self.platform] % {
-            'APP': self.application,
+            'BINARY_NAME': APPLICATIONS_TO_BINARY_NAME.get(self.application, self.application),
             'EXT': self.extension,
             'STUB': 'Stub ' if self.is_stub_installer else '',
             'STUB_NEW': ' Installer' if self.is_stub_installer else '',
@@ -809,7 +814,7 @@ def get_build_info(self):
     @property
     def binary_regex(self):
         """Return the regex for the binary."""
-        regex_base_name = (r'^(%(STUB_NEW)s|%(APP)s-.*\.%(LOCALE)s\.%(PLATFORM)s)')
+        regex_base_name = (r'^(%(STUB_NEW)s|%(BINARY_NAME)s-.*\.%(LOCALE)s\.%(PLATFORM)s)')
         regex_suffix = {'linux': r'.*\.%(EXT)s$',
                         'linux64': r'.*\.%(EXT)s$',
                         'mac': r'.*\.%(EXT)s$',
@@ -819,7 +824,8 @@ def binary_regex(self):
 
         regex = regex_base_name + regex_suffix[self.platform]
 
-        return regex % {'APP': self.application,
+        return regex % {'BINARY_NAME': APPLICATIONS_TO_BINARY_NAME.get(self.application,
+                                                                       self.application),
                         'LOCALE': self.locale,
                         'PLATFORM': PLATFORM_FRAGMENTS[self.platform],
                         'STUB': '-stub' if self.is_stub_installer else '',
@@ -983,7 +989,7 @@ def get_build_info(self):
     @property
     def binary_regex(self):
         """Return the regex for the binary."""
-        regex_base_name = (r'^(%(STUB_NEW)s|%(APP)s-.*\.%(LOCALE)s\.%(PLATFORM)s)')
+        regex_base_name = (r'^(%(STUB_NEW)s|%(BINARY_NAME)s-.*\.%(LOCALE)s\.%(PLATFORM)s)')
         regex_suffix = {'linux': r'.*\.%(EXT)s$',
                         'linux64': r'.*\.%(EXT)s$',
                         'mac': r'.*\.%(EXT)s$',
@@ -993,7 +999,8 @@ def binary_regex(self):
 
         regex = regex_base_name + regex_suffix[self.platform]
 
-        return regex % {'APP': self.application,
+        return regex % {'BINARY_NAME': APPLICATIONS_TO_BINARY_NAME.get(self.application,
+                                                                       self.application),
                         'LOCALE': self.locale,
                         'PLATFORM': PLATFORM_FRAGMENTS[self.platform],
                         'STUB': '-stub' if self.is_stub_installer else '',

tests/factory/test_factory_invalid_options.py

@@ -6,6 +6,7 @@
 
 from mozdownload import FactoryScraper
 from mozdownload.utils import urljoin
+from mozdownload.errors import NotSupportedError
 
 import mozhttpd_base_test as mhttpd
 
@@ -115,6 +116,26 @@ def test_try_without_revision(self):
                           base_url=self.wdir,
                           logger=self.logger)
 
+    def test_non_daily_fennec(self):
+        """Test that non-daily scrapper_type for fennec raises exception"""
+        self.assertRaises(NotSupportedError, FactoryScraper,
+                          scraper_type='candidate',
+                          destination=self.temp_dir,
+                          base_url=self.wdir,
+                          logger=self.logger,
+                          application='fennec',
+                          version='60.0b1')
+
+    def test_non_release_non_candidate_devedition(self):
+        """Test that non-relase and non-candidate scrapper type for devedition raises exception"""
+        self.assertRaises(NotSupportedError, FactoryScraper,
+                          scraper_type='daily',
+                          destination=self.temp_dir,
+                          base_url=self.wdir,
+                          logger=self.logger,
+                          application='devedition',
+                          version='60.0b1')
+
 
 class TestFactoryUnusedOptions(mhttpd.MozHttpdBaseTest):
 

tests/remote/test_devedition.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""Test all scraper classes for Firefox Developer Edition against the remote server"""
+
+import urllib
+
+import pytest
+
+import mozdownload
+from mozdownload.scraper import BASE_URL
+from mozdownload.utils import urljoin
+
+
+@pytest.mark.parametrize("args,url", [
+    ({'application': 'devedition', 'platform': 'linux', 'version': '60.0b1'},
+     'devedition/releases/60.0b1/linux-i686/en-US/firefox-60.0b1.tar.bz2'),
+    ({'application': 'devedition', 'platform': 'linux64', 'version': '60.0b1'},
+     'devedition/releases/60.0b1/linux-x86_64/en-US/firefox-60.0b1.tar.bz2'),
+    ({'application': 'devedition', 'platform': 'mac', 'version': '60.0b1'},
+     'devedition/releases/60.0b1/mac/en-US/Firefox 60.0b1.dmg'),
+    ({'application': 'devedition', 'platform': 'win32', 'version': '60.0b1'},
+     'devedition/releases/60.0b1/win32/en-US/Firefox Setup 60.0b1.exe'),
+    ({'application': 'devedition', 'platform': 'win64', 'version': '60.0b1'},
+     'devedition/releases/60.0b1/win64/en-US/Firefox Setup 60.0b1.exe'),
+    ({'application': 'devedition', 'platform': 'win32', 'version': '60.0b1', 'locale': 'de'},
+     'devedition/releases/60.0b1/win32/de/Firefox Setup 60.0b1.exe'),
+])
+def test_release_scraper(tmpdir, args, url):
+    """Test release scraper against the remote server."""
+    scraper = mozdownload.ReleaseScraper(destination=tmpdir, **args)
+
+    if url:
+        assert urllib.unquote(scraper.url) == urljoin(BASE_URL, url)
+
+
+@pytest.mark.parametrize("args,url", [
+    ({'application': 'devedition', 'platform': 'linux', 'version': '60.0b1', 'build_number': 1},
+     'devedition/candidates/60.0b1-candidates/build3/linux-i686/en-US/firefox-60.0b1.tar.bz2'),
+    ({'application': 'devedition', 'platform': 'linux64', 'version': '60.0b1', 'build_number': 1},
+     'devedition/candidates/60.0b1-candidates/build3/linux-x86_64/en-US/firefox-60.0b1.tar.bz2'),  # noqa
+    ({'application': 'devedition', 'platform': 'mac', 'version': '60.0b1', 'build_number': 1},
+     'devedition/candidates/60.0b1-candidates/build3/mac/en-US/Firefox 60.0b1.dmg'),
+    ({'application': 'devedition', 'platform': 'win32', 'version': '60.0b1', 'build_number': 1},
+     'devedition/candidates/60.0b1-candidates/build3/win32/en-US/Firefox Setup 60.0b1.exe'),
+    ({'application': 'devedition', 'platform': 'mac', 'version': '60.0b1', 'build_number': 1,
+          'locale': 'de'},
+     'devedition/candidates/60.0b1-candidates/build3/mac/de/Firefox 60.0b1.dmg'),
+    ({'application': 'devedition', 'platform': 'mac', 'version': '60.0b1', 'build_number': 1,
+         'extension': 'json'},
+     'devedition/candidates/60.0b1-candidates/build3/mac/en-US/firefox-60.0b1.json'),
+])
+def test_candidate_scraper(tmpdir, args, url):
+    """Test release candidate scraper against the remote server."""
+    scraper = mozdownload.ReleaseCandidateScraper(destination=tmpdir, **args)
+
+    assert urllib.unquote(scraper.url) == urljoin(BASE_URL, url)