Add full stack emulator builds

getb2g/base.py

@@ -35,7 +35,8 @@ def handled_resources(cls, request):
             if 'prepare_%s' % res in methods:
                 parents = get_parent_resources(res)
                 if all(p not in request.resources for p in parents) \
-                        or any('prepare_%s' % p in methods for p in parents):
+                        or any('prepare_%s' % p in methods for p in parents) \
+                        or res in valid_resources['device']:
                     handled_resources.append(res)
         return handled_resources
 
@@ -88,7 +89,7 @@ def prepare_gecko(self):
         """
         Prepares the gecko directory
         """
-    prepare_gecko.groups = ['emulator', 'cli']
+    prepare_gecko.groups = ['cli']
 
 class SymbolsBase(object):
     __metaclass__ = ABCMeta
@@ -129,7 +130,7 @@ def prepare_tests(self):
         """
         Prepares the tests bundle
         """
-    prepare_tests.groups = ['gecko', 'b2g_desktop']
+    prepare_tests.groups = ['gecko', 'emulator', 'b2g_desktop']
 
     def prepare_xre(self, url=None):
         """
@@ -152,7 +153,7 @@ def prepare_emulator(self):
         """
         Prepares the emulator package
         """
-    prepare_emulator.groups = ['device', 'cli']
+    prepare_emulator.groups = ['device', 'cli', 'gecko']
 
 class UnagiBase(object):
     __metaclass__ = ABCMeta
@@ -221,7 +222,7 @@ def get_parent_resources(resource):
     parents = []
     for res in valid_resources['all']:
         if res in valid_resources:
-            if resource in  valid_resources[res]:
+            if resource in valid_resources[res]:
                 parents.append(res)
     return parents
 

getb2g/handlers/pvtbpubtinderbox.py

@@ -3,28 +3,26 @@
 import tempfile
 
 from bs4 import BeautifulSoup
-from ..base import (Base, UnagiBase, PandaBase, SymbolsBase)
+from ..base import (Base, EmulatorBase, UnagiBase, PandaBase, SymbolsBase)
 from ..mixins import TinderboxMixin
 
 import mozfile
 __all__ = ['PvtBPubTinderboxHandler']
 
-class PvtBPubTinderboxHandler(Base, UnagiBase, PandaBase, SymbolsBase, TinderboxMixin):
+class PvtBPubTinderboxHandler(Base, EmulatorBase, UnagiBase, PandaBase, SymbolsBase, TinderboxMixin):
     """
     Handles resources from pvtbuilds.mozilla.org
     """
     _base_url = 'https://pvtbuilds.mozilla.org/pub/mozilla.org/b2g/tinderbox-builds/'
-    _device_names = { 'unagi': 'unagi-eng', }
+    _device_names = { 'unagi': 'unagi-eng',
+                      'emulator': 'generic', }
+
+    def prepare_tests(self):
+        self.download_extract(lambda x: x.endswith('tests.zip'), outdir='tests')
 
     def prepare_symbols(self):
-        url = self.get_resource_url(lambda x: x.startswith('b2g') and
-                                                        x.endswith('crashreporter-symbols.zip'))
-        file_name = self.download_file(url)
-        extract_dir = os.path.join(os.path.dirname(file_name), 'symbols')
-        if os.path.isdir(extract_dir):
-            shutil.rmtree(extract_dir)
-        mozfile.extract(file_name, extract_dir)
-        os.remove(file_name)
+        self.download_extract(lambda x: x.startswith('b2g') and
+                                        x.endswith('crashreporter-symbols.zip'))
 
     def prepare_panda(self):
         url = self.get_resource_url(lambda x: x == 'boot.tar.bz2')
@@ -53,7 +51,7 @@ def prepare_panda(self):
 
         url = self.get_resource_url(lambda x: x == 'build.prop')
         self.download_file(url)
-        
+
         url = self.get_resource_url(lambda x: x == 'sources.xml')
         self.download_file(url)
 
@@ -84,6 +82,12 @@ def prepare_unagi(self):
 
         url = self.get_resource_url(lambda x: x == 'build.prop')
         self.download_file(url)
-        
+
         url = self.get_resource_url(lambda x: x == 'sources.xml')
         self.download_file(url)
+
+    def prepare_emulator(self):
+        self.download_extract(lambda x: x == 'emulator.tar.gz', outdir='emulator')
+        self.download_extract(lambda x: x == 'gaia.zip')
+        self.download_file(lambda x: x == 'sources.xml')
+        self.download_file(lambda x: x == 'build.prop')

getb2g/mixins/download.py

@@ -1,11 +1,13 @@
 import os
+import shutil
 import sys
 import urllib2
 import urlparse
 
 from ..errors import MissingDataException
 from ..prompt import prompt_user_pass
 
+import mozfile
 import mozlog
 log = mozlog.getLogger('GetB2G')
 
@@ -48,8 +50,36 @@ def _chunk_report(self, bytes_so_far, total_size):
             sys.stdout.write('\n')
         sys.stdout.flush()
 
+    def download_extract(self, url, file_name=None, silent=False, outdir=None):
+        file_name = self.download_file(url, file_name=file_name, silent=silent)
+        files = mozfile.extract(file_name)
+        os.remove(file_name)
+
+        if not outdir and len(files) > 1:
+            name = os.path.basename(file_name)
+            for suffix in ('.zip', '.tar.gz', '.tar.bz2'):
+                if name.endswith(suffix):
+                     outdir = name[:-len(suffix)]
+                     break
+
+        if outdir:
+            outdir = os.path.join(self.metadata['workdir'], outdir)
+            if os.path.isdir(outdir):
+                shutil.rmtree(outdir)
+
+            if len(files) == 1:
+                shutil.move(files[0], outdir)
+            elif len(files) > 1:
+                os.makedirs(outdir)
+                for f in files:
+                    shutil.move(f, os.path.join(outdir, os.path.basename(f)))
+        log.debug("extracted '%s' to '%s'" % (file_name, outdir or files[0]))
+
 
     def download_file(self, url, file_name=None, silent=False):
+        if hasattr(url, '__call__'):
+            url = self.get_resource_url(url)
+
         domain = urlparse.urlparse(url)
         domain = '%s://%s' % (domain.scheme, domain.netloc)
         auth = self.load_auth(domain)

getb2g/request.py

@@ -1,6 +1,7 @@
 from base import valid_resources
 from errors import InvalidResourceException, MultipleDeviceResourceException
 import handlers
+import pprint
 
 import mozlog
 log = mozlog.getLogger('GetB2G')
@@ -35,7 +36,9 @@ def dispatch(self):
 
         # sort the handlers based on how many resources they can handle,
         # we want to use as few as possible so resources come from the same place 
+        pp = pprint.PrettyPrinter(indent=4)
         potential_handlers.sort(key=lambda x: x[1], reverse=True)
+        log.debug("potential handlers:\n%s" % pp.pformat(potential_handlers))
 
         # the order of resources is important as resources later in the list might
         # be affected by values set by resources earlier in the list

setup.py

@@ -7,7 +7,7 @@
 import sys
 
 PACKAGE_NAME = "getb2g"
-PACKAGE_VERSION = "2.8"
+PACKAGE_VERSION = "2.9"
 
 desc = """Get the latest B2G nightly builds for a specific device"""
 deps = ['BeautifulSoup4',