bug 696161: create simplified talos buildbot for a-team usage. r=rail…

…/catlee

process/factory.py

@@ -6905,12 +6905,18 @@ def __init__(self, OS, supportUrlBase, envName, buildBranch, branchName,
         if self.addonTester:
             self.addDownloadExtensionStep()
         self.addSetupSteps()
+        self.addPluginInstallSteps()
+        self.addPagesetInstallSteps()
+        self.addAddOnInstallSteps()
         if self.remoteTests:
             self.addPrepareDeviceStep()
         self.addUpdateConfigStep()
         self.addRunTestStep()
         self.addRebootStep()
 
+    def _propertyIsSet(self, step, prop):
+        return step.build.getProperties().has_key(prop)
+
     def addInfoSteps(self):
         self.addStep(OutputStep(
          name='tinderboxprint_slavename',
@@ -7009,7 +7015,7 @@ def addCleanupSteps(self):
     def addDmgInstaller(self):
         if self.OS in ('leopard', 'tiger', 'snowleopard'):
             self.addStep(DownloadFile(
-             url="%s/tools/buildfarm/utils/installdmg.sh" % self.supportUrlBase,
+             url=WithProperties("%s/tools/buildfarm/utils/installdmg.sh" % self.supportUrlBase),
              workdir=self.workdirBase,
             ))
 
@@ -7031,7 +7037,7 @@ def addUnpackBuildSteps(self):
         if (self.releaseTester and (self.OS in ('xp', 'vista', 'win7', 'w764'))): 
             #build is packaged in a windows installer 
             self.addStep(DownloadFile( 
-             url="%s/tools/buildfarm/utils/firefoxInstallConfig.ini" % self.supportUrlBase,
+             url=WithProperties("%s/tools/buildfarm/utils/firefoxInstallConfig.ini" % self.supportUrlBase),
              workdir=self.workdirBase,
             ))
             self.addStep(SetProperty(
@@ -7190,7 +7196,7 @@ def check_sdk(cmd, step):
     def addSetupSteps(self):
         if not self.remoteTests:
             self.addStep(DownloadFile(
-             url="%s/tools/buildfarm/maintenance/count_and_reboot.py" % self.supportUrlBase,
+             url=WithProperties("%s/tools/buildfarm/maintenance/count_and_reboot.py" % self.supportUrlBase),
              workdir=self.workdirBase,
             ))
 
@@ -7203,15 +7209,15 @@ def addSetupSteps(self):
 
         if self.customTalos is None and not self.remoteTests:
             self.addStep(DownloadFile(
-              url="%s/zips/talos.zip" % self.supportUrlBase,
+              url=WithProperties("%s/zips/talos.zip" % self.supportUrlBase),
               workdir=self.workdirBase,
             ))
             self.addStep(UnpackFile(
              filename='talos.zip',
              workdir=self.workdirBase,
             ))
             self.addStep(DownloadFile(
-             url="%s/xpis/pageloader.xpi" % self.supportUrlBase,
+             url=WithProperties("%s/xpis/pageloader.xpi" % self.supportUrlBase),
              workdir=os.path.join(self.workdirBase, "talos/page_load_test"))
             )
         elif self.remoteTests:
@@ -7256,11 +7262,12 @@ def addSetupSteps(self):
              workdir=self.workdirBase,
             ))
 
+    def addPluginInstallSteps(self):
         if self.plugins:
             #32 bit (includes mac browsers)
             if self.OS in ('xp', 'vista', 'win7', 'fedora', 'tegra_android', 'leopard', 'snowleopard', 'leopard-o'):
                 self.addStep(DownloadFile(
-                 url="%s/%s" % (self.supportUrlBase, self.plugins['32']),
+                 url=WithProperties("%s/%s" % (self.supportUrlBase, self.plugins['32'])),
                  workdir=os.path.join(self.workdirBase, "talos/base_profile"),
                 ))
                 self.addStep(UnpackFile(
@@ -7270,27 +7277,29 @@ def addSetupSteps(self):
             #64 bit
             if self.OS in ('w764', 'fedora64'):
                 self.addStep(DownloadFile(
-                 url="%s/%s" % (self.supportUrlBase, self.plugins['64']),
+                 url=WithProperties("%s/%s" % (self.supportUrlBase, self.plugins['64'])),
                  workdir=os.path.join(self.workdirBase, "talos/base_profile"),
                 ))
                 self.addStep(UnpackFile(
                  filename=os.path.basename(self.plugins['64']),
                  workdir=os.path.join(self.workdirBase, "talos/base_profile"),
                 ))
 
+    def addPagesetInstallSteps(self):
         for pageset in self.pagesets:
             self.addStep(DownloadFile(
-             url="%s/%s" % (self.supportUrlBase, pageset),
+             url=WithProperties("%s/%s" % (self.supportUrlBase, pageset)),
              workdir=os.path.join(self.workdirBase, "talos/page_load_test"),
             ))
             self.addStep(UnpackFile(
              filename=os.path.basename(pageset),
              workdir=os.path.join(self.workdirBase, "talos/page_load_test"),
             ))
 
+    def addAddOnInstallSteps(self):
         for addOn in self.talosAddOns:
             self.addStep(DownloadFile(
-             url="%s/%s" % (self.supportUrlBase, addOn),
+             url=WithProperties("%s/%s" % (self.supportUrlBase, addOn)),
              workdir=os.path.join(self.workdirBase, "talos"),
             ))
             self.addStep(UnpackFile(
@@ -7344,6 +7353,7 @@ def get_addon_url(build):
          name="Download extension",
          ignore_certs=True,
          wget_args=['-O', TalosFactory.extName],
+         doStepIf=lambda step: self._propertyIsSet(step, 'addonUrl')
         ))
 
     def addPrepareDeviceStep(self):
@@ -7433,6 +7443,81 @@ def do_disconnect(cmd):
              env=self.env,
             ))
 
+class RuntimeTalosFactory(TalosFactory):
+    def __init__(self, configOptions=None, plugins=None, pagesets=None,
+                 supportUrlBase=None, talosAddOns=None, addonTester=True,
+                 *args, **kwargs):
+        if not configOptions:
+            # TalosFactory/MozillaUpdateConfig require this format for this variable
+            # MozillaUpdateConfig allows for adding additional options at runtime,
+            # which is how this factory is intended to be used.
+            configOptions = {'suites': []}
+        # For the rest of these, make them overridable with WithProperties
+        if not plugins:
+            plugins = {'32': '%(plugin)s', '64': '%(plugin)s'}
+        if not pagesets:
+            pagesets = ['%(pageset1)s', '%(pageset2)s']
+        if not supportUrlBase:
+            supportUrlBase = '%(supportUrlBase)s'
+        if not talosAddOns:
+            talosAddOns = ['%(talosAddon1)s', '%(talosAddon2)s']
+        TalosFactory.__init__(self, *args, configOptions=configOptions,
+                              plugins=plugins, pagesets=pagesets,
+                              supportUrlBase=supportUrlBase,
+                              talosAddOns=talosAddOns, addonTester=addonTester,
+                              **kwargs)
+
+    def addInfoSteps(self):
+        pass
+
+    def addPluginInstallSteps(self):
+        if self.plugins:
+            self.addStep(DownloadFile(
+                url=WithProperties("%s/%s" % (self.supportUrlBase, '%(plugin)s')),
+                workdir=os.path.join(self.workdirBase, "talos/base_profile"),
+                doStepIf=lambda step: self._propertyIsSet(step, 'plugin'),
+                filename_property='plugin_base'
+            ))
+            self.addStep(UnpackFile(
+                filename=WithProperties('%(plugin_base)s'),
+                workdir=os.path.join(self.workdirBase, "talos/base_profile"),
+                doStepIf=lambda step: self._propertyIsSet(step, 'plugin')
+            ))
+
+    def addPagesetInstallSteps(self):
+        # XXX: This is really hacky, it would be better to extract the property
+        # name from the format string.
+        n = 1
+        for pageset in self.pagesets:
+            self.addStep(DownloadFile(
+             url=WithProperties("%s/%s" % (self.supportUrlBase, pageset)),
+             workdir=os.path.join(self.workdirBase, "talos/page_load_test"),
+             doStepIf=lambda step, n=n: self._propertyIsSet(step, 'pageset%d' % n),
+             filename_property='pageset%d_base' % n
+            ))
+            self.addStep(UnpackFile(
+             filename=WithProperties('%(pageset' + str(n) + '_base)s'),
+             workdir=os.path.join(self.workdirBase, "talos/page_load_test"),
+             doStepIf=lambda step, n=n: self._propertyIsSet(step, 'pageset%d' % n)
+            ))
+            n += 1
+
+    def addAddOnInstallSteps(self):
+        n = 1
+        for addOn in self.talosAddOns:
+            self.addStep(DownloadFile(
+             url=WithProperties("%s/%s" % (self.supportUrlBase, addOn)),
+             workdir=os.path.join(self.workdirBase, "talos"),
+             doStepIf=lambda step, n=n: self._propertyIsSet(step, 'talosAddon%d' % n),
+             filename_property='talosAddon%d_base' % n
+            ))
+            self.addStep(UnpackFile(
+             filename=WithProperties('%(talosAddon' + str(n) + '_base)s'),
+             workdir=os.path.join(self.workdirBase, "talos"),
+             doStepIf=lambda step, n=n: self._propertyIsSet(step, 'talosAddon%d' % n)
+            ))
+            n += 1
+
 class TryTalosFactory(TalosFactory):
     def addDownloadBuildStep(self):
         def get_url(build):

steps/misc.py

@@ -459,11 +459,12 @@ def start(self):
             self.addCompleteLog("errors", "Automation Error: %s" % str(e))
             return self.finished(FAILURE)
 
+        renderedUrl = self.build.getProperties().render(url)
         if self.url_property:
-            self.setProperty(self.url_property, url, "DownloadFile")
+            self.setProperty(self.url_property, renderedUrl, "DownloadFile")
         if self.filename_property:
             self.setProperty(self.filename_property,
-                    os.path.basename(url), "DownloadFile")
+                    os.path.basename(renderedUrl), "DownloadFile")
 
         if self.ignore_certs:
             self.setCommand(["wget"] + self.wget_args + ["-N", "--no-check-certificate", url])

steps/talos.py

@@ -48,6 +48,12 @@ def setBuild(self, build):
         self.super_class.setBuild(self, build)
         title = build.slavename
 
+        try:
+            self.addOptions += self.getProperty('configurationOptions').split(',')
+        # Property doesn't exist, that's fine
+        except KeyError:
+            pass
+
         #if we are an addonTester then the addonName/addonUrl build property should be set
         #  if it's not set this will throw a key error and the run will go red - which should be the expected result
         if self.addonTester: