updates mozharness to match 43f7600b8c80

--HG-- extra : amend_source : 6b20080def0274c691f38344d466d12eeb11be4a
mozilla · Jul 7, 2015 · 1ed5d81 · 1ed5d81
1 parent b223e5f
commit 1ed5d81
Show file tree

Hide file tree

Showing 23 changed files with 205 additions and 110 deletions.
diff --git a/testing/mozharness/README.txt b/testing/mozharness/README.txt
@@ -1,7 +1,7 @@
 mozharness.json is a manifest file that locks or "pins" mozharness to a repository and a revision.
 
 
-the remaining contents in this directory are a copy of hg.m.o/build/mozharness at revision: 4d855a6835ed
+the remaining contents in this directory are a copy of hg.m.o/build/mozharness at revision: 43f7600b8c80
 
 NOTE: in gecko tree based mozharness jobs are still only in development and hg.m.o/build/mozharness is still
 the source of truth
diff --git a/testing/mozharness/configs/android/android_panda_releng.py b/testing/mozharness/configs/android/android_panda_releng.py
@@ -10,17 +10,6 @@
         'python': '/tools/buildbot/bin/python',
         'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
     },
-    "run_file_names": {
-        "mochitest": "runtestsremote.py",
-        "reftest": "remotereftest.py",
-        "crashtest": "remotereftest.py",
-        "jsreftest": "remotereftest.py",
-        "robocop": "runtestsremote.py",
-        "instrumentation": "runinstrumentation.py",
-        "xpcshell": "remotexpcshelltests.py",
-        "jittest": "jit_test.py",
-        "cppunittest": "remotecppunittests.py"
-    },
     "hostutils_url":  "http://talos-remote.pvt.build.mozilla.org/tegra/tegra-host-utils.Linux.1109310.2.zip",
     "verify_path":  "/builds/sut_tools/verify.py",
     "install_app_path":  "/builds/sut_tools/installApp.py",

diff --git a/testing/mozharness/configs/b2g/taskcluster-phone-blobfree.py b/testing/mozharness/configs/b2g/taskcluster-phone-blobfree.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+config = {
+    "default_vcs": "tc-vcs",
+    "default_actions": [
+        'checkout-sources',
+        'build',
+        'build-symbols',
+        'prep-upload'
+    ],
+    "upload": {
+        "default": {
+            "upload_dep_target_exclusions": []
+        }
+    },
+    "env": {
+        "GAIA_OPTIMIZE": "1",
+        "WGET_OPTS": "-c -q"
+    },
+    "build_targets": ["blobfree"],
+    "is_automation": True,
+    "repo_remote_mappings": {
+        'https://android.googlesource.com/': 'https://git.mozilla.org/external/aosp',
+        'git://codeaurora.org/': 'https://git.mozilla.org/external/caf',
+        'https://git.mozilla.org/b2g': 'https://git.mozilla.org/b2g',
+        'git://github.com/mozilla-b2g/': 'https://git.mozilla.org/b2g',
+        'git://github.com/mozilla/': 'https://git.mozilla.org/b2g',
+        'https://git.mozilla.org/releases': 'https://git.mozilla.org/releases',
+        'http://android.git.linaro.org/git-ro/': 'https://git.mozilla.org/external/linaro',
+        'git://github.com/apitrace/': 'https://git.mozilla.org/external/apitrace',
+    },
+}
diff --git a/testing/mozharness/configs/b2g/taskcluster-spark-blobfree.py b/testing/mozharness/configs/b2g/taskcluster-spark-blobfree.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+config = {
+    "default_vcs": "tc-vcs",
+    "default_actions": [
+        'checkout-sources',
+        'build',
+        'build-symbols',
+        'prep-upload'
+    ],
+    "upload": {
+        "default": {
+            "upload_dep_target_exclusions": []
+        }
+    },
+    "env": {
+        "GAIA_OPTIMIZE": "1",
+        "WGET_OPTS": "-c -q",
+        "LIGHTSABER": "1",
+        "B2G_PATH": "%(work_dir)s",
+        "BOWER_FLAGS": "--allow-root",
+        "GAIA_DISTRIBUTION_DIR": "%(work_dir)s/gaia/distros/spark",
+        "WGET_OPTS": "-c -q",
+    },
+    "is_automation": True,
+    "build_targets": ["blobfree"],
+    "repo_remote_mappings": {
+        'https://android.googlesource.com/': 'https://git.mozilla.org/external/aosp',
+        'git://codeaurora.org/': 'https://git.mozilla.org/external/caf',
+        'https://git.mozilla.org/b2g': 'https://git.mozilla.org/b2g',
+        'git://github.com/mozilla-b2g/': 'https://git.mozilla.org/b2g',
+        'git://github.com/mozilla/': 'https://git.mozilla.org/b2g',
+        'https://git.mozilla.org/releases': 'https://git.mozilla.org/releases',
+        'http://android.git.linaro.org/git-ro/': 'https://git.mozilla.org/external/linaro',
+        'git://github.com/apitrace/': 'https://git.mozilla.org/external/apitrace',
+    },
+}
diff --git a/testing/mozharness/configs/builds/build_pool_specifics.py b/testing/mozharness/configs/builds/build_pool_specifics.py
@@ -39,5 +39,8 @@
         'graph_server': 'graphs.mozilla.org',
         'symbol_server_host': "symbolpush.mozilla.org",
         'stage_server': 'stage.mozilla.org',
+        # use the relengapi proxy to talk to tooltool
+        "tooltool_servers": ['http://relengapi/tooltool/'],
+        "tooltool_url": 'http://relengapi/tooltool/',
     },
 }
diff --git a/testing/mozharness/configs/merge_day/aurora_to_beta.py b/testing/mozharness/configs/merge_day/aurora_to_beta.py
@@ -1,12 +1,40 @@
 config = {
     "log_name": "aurora_to_beta",
 
-    "branding_dirs": [
-        "mobile/android/config/mozconfigs/android-api-11/",
-        "mobile/android/config/mozconfigs/android-api-9-10-constrained/",
-        "mobile/android/config/mozconfigs/android-x86/",
+    "replacements": [
+        # File, from, to
+        ("{}/{}".format(d, f),
+        "ac_add_options --with-branding=mobile/android/branding/aurora",
+        "ac_add_options --with-branding=mobile/android/branding/beta")
+        for d in ["mobile/android/config/mozconfigs/android-api-11/",
+                  "mobile/android/config/mozconfigs/android-api-9-10-constrained/",
+                  "mobile/android/config/mozconfigs/android-x86/"]
+        for f in ["debug", "nightly", "l10n-nightly"]
+    ] + [
+        # File, from, to
+        ("{}/{}".format(d, f),
+        "ac_add_options --with-branding=browser/branding/aurora",
+        "ac_add_options --with-branding=browser/branding/nightly")
+        for d in ["browser/config/mozconfigs/linux32",
+                  "browser/config/mozconfigs/linux64",
+                  "browser/config/mozconfigs/win32",
+                  "browser/config/mozconfigs/win64",
+                  "browser/config/mozconfigs/macosx64"]
+        for f in ["debug", "nightly", "l10n-mozconfig"]
+    ] + [
+        ("browser/config/mozconfigs/macosx-universal/nightly",
+         "ac_add_options --with-branding=browser/branding/aurora",
+         "ac_add_options --with-branding=browser/branding/nightly"),
+        ("browser/confvars.sh",
+         "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-aurora",
+         "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release"),
+        ("browser/confvars.sh",
+         "MAR_CHANNEL_ID=firefox-mozilla-aurora",
+         "MAR_CHANNEL_ID=firefox-mozilla-beta"),
+        ("browser/config/mozconfigs/whitelist",
+         "ac_add_options --with-branding=browser/branding/aurora",
+         "ac_add_options --with-branding=browser/branding/nightly"),
     ],
-    "branding_files": ["debug", "l10n-nightly", "nightly"],
 
     # Disallow sharing, since we want pristine .hg directories.
     # "vcs_share_base": None,

diff --git a/testing/mozharness/configs/merge_day/beta_to_release.py b/testing/mozharness/configs/merge_day/beta_to_release.py
@@ -1,12 +1,23 @@
 config = {
     "log_name": "beta_to_release",
-
-    "branding_dirs": [
-        "mobile/android/config/mozconfigs/android-api-11/",
-        "mobile/android/config/mozconfigs/android-api-9-10-constrained/",
-        "mobile/android/config/mozconfigs/android-x86/",
+    "replacements": [
+        # File, from, to
+        ("{}/{}".format(d, f),
+        "ac_add_options --with-branding=mobile/android/branding/beta",
+        "ac_add_options --with-branding=mobile/android/branding/official")
+        for d in ["mobile/android/config/mozconfigs/android-api-11/",
+                  "mobile/android/config/mozconfigs/android-api-9-10-constrained/",
+                  "mobile/android/config/mozconfigs/android-x86/"]
+        for f in ["debug", "nightly", "l10n-nightly", "l10n-release", "release"]
+    ] + [
+        # File, from, to
+        ("browser/confvars.sh",
+         "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release",
+         "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release"),
+        ("browser/confvars.sh",
+         "MAR_CHANNEL_ID=firefox-mozilla-beta",
+         "MAR_CHANNEL_ID=firefox-mozilla-release"),
     ],
-    "branding_files": ["release", "l10n-release", "l10n-nightly", "nightly"],
 
     # Disallow sharing, since we want pristine .hg directories.
     # "vcs_share_base": None,

diff --git a/testing/mozharness/configs/merge_day/central_to_aurora.py b/testing/mozharness/configs/merge_day/central_to_aurora.py
@@ -20,12 +20,7 @@
                   "browser/config/mozconfigs/win32",
                   "browser/config/mozconfigs/win64",
                   "browser/config/mozconfigs/macosx64"]
-        for f in ["debug", "nightly"]
-    ] + [
-        ("browser/config/mozconfigs/macosx-universal/nightly",
-        "ac_add_options --with-branding=browser/branding/nightly",
-        "ac_add_options --with-branding=browser/branding/aurora")
-
+        for f in ["debug", "nightly", "l10n-mozconfig"]
     ] + [
         # File, from, to
         ("{}/l10n-nightly".format(d),
@@ -53,6 +48,12 @@
         ("browser/confvars.sh",
          "MAR_CHANNEL_ID=firefox-mozilla-central",
          "MAR_CHANNEL_ID=firefox-mozilla-aurora"),
+        ("browser/config/mozconfigs/macosx-universal/nightly",
+         "ac_add_options --with-branding=browser/branding/nightly",
+         "ac_add_options --with-branding=browser/branding/aurora"),
+        ("browser/config/mozconfigs/whitelist",
+         "ac_add_options --with-branding=browser/branding/nightly",
+         "ac_add_options --with-branding=browser/branding/aurora"),
     ],
     "locale_files": [
         "browser/locales/shipped-locales",

diff --git a/testing/mozharness/configs/merge_day/release_to_esr.py b/testing/mozharness/configs/merge_day/release_to_esr.py
@@ -4,6 +4,15 @@
 
 config = {
     "log_name": "relese_to_esr",
+    "replacements": [
+        # File, from, to
+        ("browser/confvars.sh",
+         "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release",
+         "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-esr"),
+        ("browser/confvars.sh",
+         "MAR_CHANNEL_ID=firefox-mozilla-release",
+         "MAR_CHANNEL_ID=firefox-mozilla-esr"),
+    ],
     # Disallow sharing, since we want pristine .hg directories.
     # "vcs_share_base": None,
     # "hg_share_base": None,

diff --git a/testing/mozharness/configs/vcs_sync/beagle.py b/testing/mozharness/configs/vcs_sync/beagle.py
@@ -738,7 +738,7 @@
     ],
     "pip_index": False,
 
-    "default_notify_from": "developer-services@mozilla.org",
+    "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
     "notify_config": [{
         "to": "releng-ops-trial@mozilla.com",
         "failure_only": False,

diff --git a/testing/mozharness/configs/vcs_sync/build-repos.py b/testing/mozharness/configs/vcs_sync/build-repos.py
@@ -83,7 +83,7 @@
     ],
     "pip_index": False,
 
-    "default_notify_from": "developer-services@mozilla.org",
+    "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
     "notify_config": [{
         "to": "releng-ops-trial@mozilla.com",
         "failure_only": False,

diff --git a/testing/mozharness/configs/vcs_sync/gecko-git.py b/testing/mozharness/configs/vcs_sync/gecko-git.py
@@ -380,11 +380,11 @@
         "remote_path": "/home/pmoore/upload/gecko-git-upload",
     }],
 
-    "default_notify_from": "developer-services@mozilla.org",
+    "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
     "notify_config": [{
-        "to": "pmoore@mozilla.com",
+        "to": "releng-ops-trial@mozilla.com",
         "failure_only": False,
-        "skip_empty_messages": False,
+        "skip_empty_messages": True,
     }],
 
     # Disallow sharing, since we want pristine .hg and .git directories.

diff --git a/testing/mozharness/configs/vcs_sync/l10n.py b/testing/mozharness/configs/vcs_sync/l10n.py
@@ -281,7 +281,7 @@
     ],
     "pip_index": False,
 
-    "default_notify_from": "developer-services@mozilla.org",
+    "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
     "notify_config": [{
         "to": "releng-ops-trial@mozilla.com",
         "failure_only": False,

diff --git a/testing/mozharness/configs/vcs_sync/project-branches.py b/testing/mozharness/configs/vcs_sync/project-branches.py
@@ -102,7 +102,7 @@
 
     "combined_mapfile": "combined_gecko_mapfile",
 
-    "default_notify_from": "developer-services@mozilla.org",
+    "default_notify_from": "developer-services+%s@mozilla.org" % hostname,
     "notify_config": [{
         "to": "releng-ops-trial@mozilla.com",
         "failure_only": False,

diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1067,12 +1067,18 @@ def _get_tooltool_auth_file(self):
         # set the default authentication file based on platform; this
         # corresponds to where puppet puts the token
         if 'tooltool_authentication_file' in self.config:
-            return self.config['tooltool_authentication_file']
-
-        if self._is_windows():
-            return r'c:\builds\relengapi.tok'
+            fn = self.config['tooltool_authentication_file']
+        elif self._is_windows():
+            fn = r'c:\builds\relengapi.tok'
         else:
-            return '/builds/relengapi.tok'
+            fn = '/builds/relengapi.tok'
+
+        # if the file doesn't exist, don't pass it to tooltool (it will just
+        # fail).  In taskcluster, this will work OK as the relengapi-proxy will
+        # take care of auth.  Everywhere else, we'll get auth failures if
+        # necessary.
+        if os.path.exists(fn):
+            return fn
 
     def _run_tooltool(self):
         self._assert_cfg_valid_for_action(
@@ -1095,7 +1101,9 @@ def _run_tooltool(self):
             c['tooltool_bootstrap'],
         ]
         cmd.extend(c['tooltool_script'])
-        cmd.extend(['--authentication-file', self._get_tooltool_auth_file()])
+        auth_file = self._get_tooltool_auth_file()
+        if auth_file:
+            cmd.extend(['--authentication-file', auth_file])
         self.info(str(cmd))
         self.run_command(cmd, cwd=dirs['abs_src_dir'], halt_on_failure=True)
 

diff --git a/testing/mozharness/mozharness/mozilla/tooltool.py b/testing/mozharness/mozharness/mozilla/tooltool.py
@@ -33,12 +33,18 @@ def _get_auth_file(self):
         # set the default authentication file based on platform; this
         # corresponds to where puppet puts the token
         if 'tooltool_authentication_file' in self.config:
-            return self.config['tooltool_authentication_file']
-
-        if self._is_windows():
-            return r'c:\builds\relengapi.tok'
+            fn = self.config['tooltool_authentication_file']
+        elif self._is_windows():
+            fn = r'c:\builds\relengapi.tok'
         else:
-            return '/builds/relengapi.tok'
+            fn = '/builds/relengapi.tok'
+
+        # if the file doesn't exist, don't pass it to tooltool (it will just
+        # fail).  In taskcluster, this will work OK as the relengapi-proxy will
+        # take care of auth.  Everywhere else, we'll get auth failures if
+        # necessary.
+        if os.path.exists(fn):
+            return fn
 
     def tooltool_fetch(self, manifest, bootstrap_cmd=None,
                        output_dir=None, privileged=False, cache=None):

diff --git a/testing/mozharness/mozharness/mozilla/vcstools.py b/testing/mozharness/mozharness/mozilla/vcstools.py
@@ -39,14 +39,20 @@ def _pre_checkout(self, action):
                     )
                     self.chmod(file_path, 0755)
         else:
+            # We simply verify that everything is in order
+            # or if the user forgot to specify developer mode
             for vcs_tool in VCS_TOOLS:
                 file_path = self.which(vcs_tool)
-                if file_path is None:
-                    file_path = self.query_exe(vcs_tool)
 
-                    if not self.is_exe(file_path):
-                        self.critical("%s is not executable." % file_path)
+                # If the tool is specified and it is a list is
+                # because we're running on Windows and we won't check
+                if type(self.query_exe(vcs_tool)) is list:
+                    continue
+
+                if not self.is_exe(file_path):
+                    self.critical("%s is not executable." % file_path)
 
+                if file_path is None:
                     self.fatal("This machine is missing %s, if this is your "
                                "local machine you can use --cfg "
                                "developer_config.py" % vcs_tool)
diff --git a/testing/mozharness/scripts/android_emulator_unittest.py b/testing/mozharness/scripts/android_emulator_unittest.py
@@ -595,7 +595,7 @@ def verify_emulator(self):
         max_restarts = 3
         emulator_ok = self._retry(max_restarts, 30, self._verify_emulator_and_restart_on_fail, "Check emulator")
         if not emulator_ok:
-            self.fatal('Unable to start emulator after %d attempts' % max_restarts)
+            self.fatal('INFRA-ERROR: Unable to start emulator after %d attempts' % max_restarts)
         # Start logcat for the emulator. The adb process runs until the
         # corresponding emulator is killed. Output is written directly to
         # the blobber upload directory so that it is uploaded automatically

diff --git a/testing/mozharness/scripts/android_panda.py b/testing/mozharness/scripts/android_panda.py
@@ -432,7 +432,7 @@ def _query_abs_base_cmd(self, suite_category, suite):
         c = self.config
         dirs = self.query_abs_dirs()
         options = []
-        run_file = c['run_file_names'][suite_category]
+        run_file = self.tree_config["suite_definitions"][suite_category]["run_filename"]
         base_cmd = ['python', '-u']
         base_cmd.append(os.path.join((dirs["abs_%s_dir" % suite_category]), run_file))
         self.device_ip = socket.gethostbyname(self.mozpool_device)

diff --git a/testing/mozharness/scripts/b2g_build.py b/testing/mozharness/scripts/b2g_build.py
@@ -540,6 +540,7 @@ def build(self):
         dirs = self.query_abs_dirs()
         gecko_config = self.load_gecko_config()
         build_targets = gecko_config.get('build_targets', [])
+        build_targets.extend(self.config.get("build_targets", []))
         if not build_targets:
             cmds = [self.generate_build_command()]
         else: