bug 1490119: support mar cert replacement in update verify. r=nthomas

lib/python/release/updates/verify.py

@@ -12,27 +12,29 @@ class UpdateVerifyConfig(object):
     key_write_order = ("release", "product", "platform", "build_id", "locales",
                        "channel", "patch_types", "from", "aus_server",
                        "ftp_server_from", "ftp_server_to", "to",
-                       "mar_channel_IDs", "to_build_id", "to_display_version",
-                       "to_app_version", "updater_package")
+                       "mar_channel_IDs", "override_certs", "to_build_id",
+                       "to_display_version", "to_app_version", "updater_package")
     global_keys = ("product", "channel", "aus_server", "to", "to_build_id",
-                   "to_display_version", "to_app_version")
+                   "to_display_version", "to_app_version", "override_certs")
     release_keys = ("release", "build_id", "locales", "patch_types", "from",
                     "ftp_server_from", "ftp_server_to", "mar_channel_IDs",
                     "platform", "updater_package")
     first_only_keys = ("from", "aus_server", "to", "to_build_id",
-                       "to_display_version", "to_app_version")
+                       "to_display_version", "to_app_version", "override_certs")
     compare_attrs = global_keys + ("releases",)
 
     def __init__(self, product=None, channel=None,
                  aus_server=None, to=None, to_build_id=None,
-                 to_display_version=None, to_app_version=None):
+                 to_display_version=None, to_app_version=None,
+                 override_certs=None):
         self.product = product
         self.channel = channel
         self.aus_server = aus_server
         self.to = to
         self.to_build_id = to_build_id
         self.to_display_version = to_display_version
         self.to_app_version = to_app_version
+        self.override_certs = override_certs
         self.releases = []
 
     def __eq__(self, other):
@@ -167,7 +169,8 @@ def getChunk(self, chunks, thisChunk):
                                        self.aus_server, self.to,
                                        self.to_build_id,
                                        self.to_display_version,
-                                       self.to_app_version)
+                                       self.to_app_version,
+                                       self.override_certs)
         for t in allTests:
             build_id, locale, from_path = t
             if from_path == "None":

release/mar_certs/README

@@ -0,0 +1,24 @@
+These certificates are imported from mozilla-central (https://hg.mozilla.org/mozilla-central/file/tip/toolkit/mozapps/update/updater)
+and used to support staging update verify jobs. These jobs end up replacing the certificates within the binaries
+(through a binary search and replace), and must all be the same length for this to work correctly. If we recreate
+these certificates, and the resulting public certificates are not the same length anymore, the commonName may be
+changed to line them up again. https://github.com/google/der-ascii is a useful tool for doing this. For example:
+
+To convert the certificate to ascii:
+der2ascii -i dep1.der -o dep1.ascii
+
+Then use your favourite editor to change the commonName field. That block will look something like:
+    SEQUENCE {
+      SET {
+        SEQUENCE {
+          # commonName
+          OBJECT_IDENTIFIER { 2.5.4.3 }
+          PrintableString { "CI MAR signing key 1" }
+        }
+      }
+    }
+
+You can pad the PrintableString with spaces to increase the length of the cert (1 space = 1 byte).
+
+Then, convert back to der:
+ascii2der -i dep1.ascii -o newdep1.der

release/replace-updater-certs.py

@@ -0,0 +1,33 @@
+import os.path
+import sys
+
+cert_dir = sys.argv[1]
+# Read twice, because strings cannot be copied
+updater_data = open(sys.argv[2], "rb").read()
+new_updater = open(sys.argv[2], "rb").read()
+outfile = sys.argv[3]
+
+cert_pairs = sys.argv[4:]
+
+if (len(cert_pairs) % 2) != 0:
+    print("Certs must be provided in pairs")
+    sys.exit(1)
+
+for find_cert, replace_cert in zip(*[iter(cert_pairs)]*2):
+    find = open(os.path.join(cert_dir, find_cert), "rb").read()
+    replace = open(os.path.join(cert_dir, replace_cert), "rb").read()
+    print("Looking for {}...".format(find_cert))
+    if find in new_updater:
+        print("Replacing {} with {}".format(find_cert, replace_cert))
+        new_updater = new_updater.replace(find, replace)
+    else:
+        print("Didn't find {}...".format(find_cert))
+
+if len(updater_data) != len(new_updater):
+    print("WARNING: new updater is not the same length as the old one (old: {}, new: {})".format(len(updater_data), len(new_updater)))
+
+if updater_data == new_updater:
+    print("WARNING: updater is unchanged")
+
+with open(outfile, 'wb+') as f:
+    f.write(new_updater)

release/updates/verify.sh

@@ -17,6 +17,7 @@ to=""
 to_build_id=""
 to_app_version=""
 to_display_version=""
+override_certs=""
 diff_summary_log="$PWD/diff-summary.log"
 if [ -e ${diff_summary_log} ]; then
   rm ${diff_summary_log}
@@ -27,6 +28,11 @@ pushd `dirname $0` &>/dev/null
 MY_DIR=$(pwd)
 popd &>/dev/null
 retry="$MY_DIR/../../buildfarm/utils/retry.py -s 1 -r 3"
+cert_replacer="$MY_DIR/../replace-updater-certs.py"
+
+dep_overrides="nightly_aurora_level3_primary.der dep1.der nightly_aurora_level3_secondary.der dep2.der release_primary.der dep1.der release_secondary.der dep2.der"
+nightly_overrides="dep1.der nightly_aurora_level3_primary.der dep2.der nightly_aurora_level3_secondary.der release_primary.der nightly_aurora_level3_primary.der release_secondary.der nightly_aurora_level3_secondary.der"
+release_overrides="dep1.der release_primary.der dep2.der release_secondary.der nightly_aurora_level3_primary.der release_primary.der nightly_aurora_level3_secondary.der release_secondary.der"
 
 runmode=0
 config_file="updates.cfg"
@@ -206,6 +212,28 @@ do
             fi
         done
 
+        if [ ! -z "$override_certs" ]; then
+            echo "Replacing certs in updater binary"
+            cp "${updater}" "${updater}.orig"
+            case ${override_certs} in
+              dep)
+                overrides=${dep_overrides}
+                ;;
+              nightly)
+                overrides=${nightly_overrides}
+                ;;
+              release)
+                overrides=${release_overrides}
+                ;;
+              *)
+                echo "Unknown override cert - skipping"
+                ;;
+            esac
+            python "${cert_replacer}" "${MY_DIR}/../mar_certs" "${updater}.orig" "${updater}" ${overrides}
+        else
+            echo "override_certs is '${override_certs}', not replacing any certificates"
+        fi
+
         if [ "$updater" == "null" ]; then
             echo "Couldn't find updater binary"
             continue