Merge default -> production

--HG--
branch : production

configs/merge_day/release_to_esr.py

@@ -1,6 +1,6 @@
-NEW_ESR_REPO = "ssh://hg.mozilla.org/releases/mozilla-esr38"
-OLD_ESR_REPO = "https://hg.mozilla.org/releases/mozilla-esr31"
-OLD_ESR_CHANGESET = "450086c0ded0"
+NEW_ESR_REPO = "ssh://hg.mozilla.org/releases/mozilla-esr45"
+OLD_ESR_REPO = "https://hg.mozilla.org/releases/mozilla-esr38"
+OLD_ESR_CHANGESET = "16351963d75c"
 
 config = {
     "log_name": "relese_to_esr",

mozharness/base/python.py

@@ -562,6 +562,10 @@ def influxdb_recording_init(self):
         self.recording = False
         self.post = None
         self.posturl = None
+        self.res_props = os.path.join(
+            self.query_abs_dirs()['abs_obj_dir'], '.mozbuild', 'build_resources.json'
+        )
+        self.rmtree(self.res_props)
 
         try:
             site_packages_path = self.query_python_site_packages_path()
@@ -631,6 +635,90 @@ def influxdb_recording_post_action(self, action, success=None):
             ],
         }])
 
+    def _get_resource_usage(self, res, name, iolen, cpulen):
+        c = {}
+        p = {}
+        if self.buildbot_config:
+            c = self.buildbot_config.get('properties', {})
+        if self.buildbot_properties:
+            p = self.buildbot_properties
+
+        data = [
+            # Build properties
+            c.get('buildername'),
+            c.get('product'),
+            c.get('platform'),
+            c.get('branch'),
+            c.get('slavename'),
+            c.get('revision'),
+            p.get('gaia_revision'),
+            c.get('buildid'),
+
+            # Mach step properties
+            name,
+            res.get('start'),
+            res.get('end'),
+            res.get('duration'),
+            res.get('cpu_percent'),
+        ]
+        # The io and cpu_times fields are arrays, though they aren't always
+        # present if a step completes before resource utilization is measured.
+        # We add the arrays if they exist, otherwise we just do an array of None
+        # to fill up the stat point.
+        data.extend(res.get('io', [None] * iolen))
+        data.extend(res.get('cpu_times', [None] * cpulen))
+        return data
+
+    @PostScriptAction('build')
+    def record_mach_stats(self, action, success=None):
+        if not self.recording:
+            return
+        if not os.path.exists(self.res_props):
+            self.info('No build_resources.json found, not logging stats')
+            return
+        with open(self.res_props) as fh:
+            resources = json.load(fh)
+            data = {
+                "points": [
+                ],
+                "name": "mach",
+                "columns": [
+                    # Build properties
+                    "buildername",
+                    "product",
+                    "platform",
+                    "branch",
+                    "slavename",
+                    "gecko_revision",
+                    "gaia_revision",
+                    "buildid",
+
+                    # Mach step properties
+                    "name",
+                    "start",
+                    "end",
+                    "duration",
+                    "cpu_percent",
+                ],
+            }
+            # The io and cpu_times fields aren't static - they may vary based
+            # on the specific platform being measured. Mach records the field
+            # names, which we use as the column names here.
+            data['columns'].extend(resources['io_fields'])
+            data['columns'].extend(resources['cpu_times_fields'])
+            iolen = len(resources['io_fields'])
+            cpulen = len(resources['cpu_times_fields'])
+
+            # The top-level data has the overall resource usage, which we record
+            # under the name 'TOTAL' to separate it from the individual tiers.
+            data['points'].append(self._get_resource_usage(resources, 'TOTAL', iolen, cpulen))
+
+            # Each tier also has the same resource stats as the top-level.
+            for tier in resources['tiers']:
+                data['points'].append(self._get_resource_usage(tier, tier['name'], iolen, cpulen))
+
+            self.record_influx_stat([data])
+
     def record_influx_stat(self, json_data):
         try:
             r = self.post(self.posturl, data=json.dumps(json_data), timeout=5)

mozharness/mozilla/building/buildbase.py

@@ -813,12 +813,15 @@ def query_build_env(self, replace_dict=None, **kwargs):
             env['MOZ_PGO'] = '1'
 
         if c.get('enable_signing'):
-            moz_sign_cmd = subprocess.list2cmdline(
-                self.query_moz_sign_cmd(formats=None)
-            )
-            # windows fix. This is passed to mach build env and we call that
-            # with python, not with bash so we need to fix the slashes here
-            env['MOZ_SIGN_CMD'] = moz_sign_cmd.replace('\\', '\\\\\\\\')
+            if os.environ.get('MOZ_SIGNING_SERVERS'):
+                moz_sign_cmd = subprocess.list2cmdline(
+                    self.query_moz_sign_cmd(formats=None)
+                )
+                # windows fix. This is passed to mach build env and we call that
+                # with python, not with bash so we need to fix the slashes here
+                env['MOZ_SIGN_CMD'] = moz_sign_cmd.replace('\\', '\\\\\\\\')
+            else:
+                self.warning("signing disabled because MOZ_SIGNING_SERVERS is not set")
 
         # to activate the right behaviour in mozonfigs while we transition
         if c.get('enable_release_promotion'):

mozharness/mozilla/tooltool.py

@@ -46,11 +46,12 @@ def tooltool_fetch(self, manifest, bootstrap_cmd=None,
         tooltool = self.query_exe('tooltool.py', return_type='list')
 
         if self.config.get("developer_mode"):
-            if not os.path.exists(str(tooltool)):
-                tooltool = self._fetch_tooltool_py()
-            cmd = [tooltool,
-                   "--authentication-file",
-                   get_credentials_path()]
+            tooltool = [bin for bin in tooltool if os.path.exists(bin)]
+            if tooltool:
+                cmd = [tooltool[0]]
+            else:
+                cmd = [self._fetch_tooltool_py()]
+            cmd.extend(["--authentication-file", get_credentials_path()])
         else:
             cmd = tooltool
 

scripts/b2g_build.py

@@ -157,6 +157,7 @@ def __init__(self, require_config_file=False, config={},
 
         dirs = self.query_abs_dirs()
         self.objdir = os.path.join(dirs['work_dir'], 'objdir-gecko')
+        self.abs_dirs['abs_obj_dir'] = self.objdir
         if self.config.get("update_type", "ota") == "fota":
             self.make_updates_cmd = ['./build.sh', 'gecko-update-fota']
             self.extra_update_attrs = 'isOsUpdate="true"'