Gecko incremental update tools

tools/update-tools/build-gecko-mar.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2012 Mozilla Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Build a gecko (OTA) full or incremental MAR
+
+import argparse
+import os
+import shutil
+import tempfile
+import update_tools
+
+def unwrap_mar(mar, verbose):
+    print "Extracting MAR for incremental update: %s" % mar
+    tmpdir = tempfile.mkdtemp()
+    bz2_mar = update_tools.BZip2Mar(mar, verbose=verbose)
+    bz2_mar.extract(tmpdir)
+    return tmpdir
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("mar", metavar="MAR", help="Destination MAR file")
+    parser.add_argument("--dir", metavar="DIR", default=None,
+        help="Source directory. When building an \"incremental\" MAR, this can " +
+             "also be a MAR for convenience. Default: $PWD")
+
+    parser.add_argument("--to", metavar="TO", default=None,
+        help="This is a synonym for --dir")
+    parser.add_argument("--from", metavar="FROM", dest="from_dir", default=None,
+        help="The base directory or MAR to build an incremental MAR from. This " +
+             "will build an incremental update MAR between FROM and TO")
+    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
+        default=False, help="Enable verbose logging")
+
+    args = parser.parse_args()
+
+    if os.path.isdir(args.mar):
+        parser.error("MAR destination is a directory: %s" % args.mar)
+
+    if not args.dir:
+        args.dir = args.to if args.to else os.getcwd()
+
+    if not args.from_dir and not os.path.isdir(args.dir):
+        parser.error("Path is not a directory: %s" % args.dir)
+
+    to_tmpdir = from_tmpdir = None
+    if args.from_dir and os.path.isfile(args.dir):
+        to_tmpdir = unwrap_mar(args.dir, args.verbose)
+        args.dir = to_tmpdir
+
+    if args.from_dir and os.path.isfile(args.from_dir):
+        from_tmpdir = unwrap_mar(args.from_dir, args.verbose)
+        args.from_dir = from_tmpdir
+
+    try:
+        builder = update_tools.GeckoMarBuilder()
+        builder.build_gecko_mar(args.dir, args.mar, from_dir=args.from_dir)
+        update_type = "incremental" if args.from_dir else "full"
+
+        print "Built %s update MAR: %s" % (update_type, args.mar)
+    except Exception, e:
+        parser.error(e)
+    finally:
+        if to_tmpdir:
+            shutil.rmtree(to_tmpdir)
+        if from_tmpdir:
+            shutil.rmtree(from_tmpdir)
+
+if __name__ == "__main__":
+    main()

tools/update-tools/update_tools.py

@@ -104,28 +104,76 @@ def _access_check(fn, mode):
                     return name
     return None
 
-class PrebuiltTool(object):
-    def __init__(self, name):
+class B2GConfig(object):
+    CONFIG_VARS = ("GECKO_PATH", "GECKO_OBJDIR")
+
+    def __init__(self):
+        shell = ". load-config.sh"
+        for var in self.CONFIG_VARS:
+            shell += "\necho $%s" % var
+
+        result = run_command(["bash", "-c", shell], cwd=b2g_dir,
+                             env={"B2G_DIR": b2g_dir})
+
+        if not result:
+            raise Exception("Couldn't parse result of load-config.sh")
+
+        lines = result.splitlines()
+        if len(lines) != len(self.CONFIG_VARS):
+            raise Exception("Wrong number of config vars: %d" % len(lines))
+
+        for i in range(len(self.CONFIG_VARS)):
+            setattr(self, self.CONFIG_VARS[i].lower(), lines[i].strip())
+
+        self.init_gecko_path()
+        if not self.gecko_objdir:
+            self.gecko_objdir = os.path.join(self.gecko_path, "objdir-gecko")
+
+    def init_gecko_path(self):
+        if not self.gecko_path:
+            self.gecko_path = os.path.join(b2g_dir, "gecko")
+
+        if os.path.exists(self.gecko_path):
+            return
+
+        relative_gecko_path = os.path.join(b2g_dir, self.gecko_path)
+        if os.path.exists(relative_gecko_path):
+            self.gecko_path = relative_gecko_path
+            return
+
+        raise Exception("B2G gecko directory not found: %s" % self.gecko_path)
+
+    def get_gecko_host_bin(self, path):
+        return os.path.join(self.gecko_objdir, "dist", "host", "bin", path)
+
+class Tool(object):
+    def __init__(self, path, prebuilt=False):
+        self.tool = path
+        if prebuilt:
+            self.init_prebuilt(path)
+
+        if not os.path.exists(self.tool):
+            raise Exception("Couldn't find %s " % self.tool)
+
+    def init_prebuilt(self, path):
         host_dir = "linux-x86"
         if platform.system() == "Darwin":
             host_dir = "darwin-x86"
 
-        self.tool = os.path.join(bin_dir, host_dir, name)
-        if not os.path.exists(self.tool):
-            raise Exception("Couldn't find %s " % self.tool)
+        self.tool = os.path.join(bin_dir, host_dir, path)
 
     def get_tool(self):
         return self.tool
 
-    def run(self, *args):
-        return run_command((self.tool,) + args)
+    def run(self, *args, **kwargs):
+        return run_command((self.tool,) + args, **kwargs)
 
-class AdbTool(PrebuiltTool):
+class AdbTool(Tool):
     DEVICE   = ("-d")
     EMULATOR = ("-e")
 
     def __init__(self, device=None):
-        PrebuiltTool.__init__(self, "adb")
+        Tool.__init__(self, "adb", prebuilt=True)
         self.adb_args = ()
         if device in (self.DEVICE, self.EMULATOR):
             self.adb_args = device
@@ -134,7 +182,7 @@ def __init__(self, device=None):
 
     def run(self, *args):
         adb_args = self.adb_args + args
-        return PrebuiltTool.run(self, *adb_args)
+        return Tool.run(self, *adb_args)
 
     def shell(self, *args):
         return self.run("shell", *args)
@@ -163,9 +211,9 @@ def get_cmdline(self, pid):
         # cmdline is null byte separated and has a trailing null byte
         return result.split("\x00")[:-1]
 
-class MarTool(PrebuiltTool):
+class MarTool(Tool):
     def __init__(self):
-        PrebuiltTool.__init__(self, "mar")
+        Tool.__init__(self, b2g_config.get_gecko_host_bin("mar"))
 
     def list_entries(self, mar_path):
         result = self.run("-t", mar_path)
@@ -177,13 +225,80 @@ def list_entries(self, mar_path):
             entries.append(words[2])
         return entries
 
+    def create(self, mar_path, src_dir=None):
+        if not src_dir:
+            src_dir = os.getcwd()
+
+        mar_args = ["-c", mar_path]
+
+        # The MAR tool wants a listing of each file to add
+        for root, dirs, files in os.walk(src_dir):
+            for f in files:
+                file_path = os.path.join(root, f)
+                mar_args.append(os.path.relpath(file_path, src_dir))
+
+        self.run(*mar_args, cwd=src_dir)
+
+    def extract(self, mar_path, dest_dir=None):
+        self.run("-x", mar_path, cwd=dest_dir)
+
     def is_gecko_mar(self, mar_path):
         return not self.is_fota_mar(mar_path)
 
     def is_fota_mar(self, mar_path):
         entries = self.list_entries(mar_path)
         return "update.zip" in entries
 
+class BZip2Mar(object):
+    def __init__(self, mar_file, verbose=False):
+        self.mar_file = mar_file
+        self.verbose = verbose
+        self.mar_tool = MarTool()
+        self.bzip2_tool = which("bzip2")
+        if not self.bzip2_tool:
+            raise Exception("Couldn't find bzip2 on the PATH")
+
+    def bzip2(self, *args):
+        bzargs = [self.bzip2_tool]
+        if self.verbose:
+            bzargs.append("-v")
+        bzargs.extend(args)
+
+        return run_command(bzargs)
+
+    def create(self, src_dir):
+        if not os.path.exists(src_dir):
+            raise Exception("Source directory doesn't exist: %s" % src_dir)
+
+        temp_dir = tempfile.mkdtemp()
+        for root, dirs, files in os.walk(src_dir):
+            for f in files:
+                path = os.path.join(root, f)
+                rel_file = os.path.relpath(path, src_dir)
+                out_file = os.path.join(temp_dir, rel_file)
+                out_dir = os.path.dirname(out_file)
+                if not os.path.exists(out_dir):
+                    os.makedirs(out_dir)
+
+                shutil.copy(path, out_file)
+                self.bzip2("-z", out_file)
+                os.rename(out_file + ".bz2", out_file)
+
+        self.mar_tool.create(self.mar_file, src_dir=temp_dir)
+
+    def extract(self, dest_dir):
+        if not os.path.exists(dest_dir):
+            os.makedirs(dest_dir)
+            if not os.path.exists(dest_dir):
+                raise Exception("Couldn't create directory: %s" % dest_dir)
+
+        self.mar_tool.extract(self.mar_file, dest_dir=dest_dir)
+        for root, dirs, files in os.walk(dest_dir):
+            for f in files:
+                path = os.path.join(root, f)
+                os.rename(path, path + ".bz2")
+                self.bzip2("-d", path + ".bz2")
+
 class FotaZip(zipfile.ZipFile):
     UPDATE_BINARY  = "META-INF/com/google/android/update-binary"
     UPDATER_SCRIPT = "META-INF/com/google/android/updater-script"
@@ -260,29 +375,12 @@ def __init__(self):
     def __del__(self):
         shutil.rmtree(self.stage_dir)
 
-    def find_gecko_dir(self):
-        gecko_dir = run_command(["bash", "-c",
-                                 ". load-config.sh; echo -n $GECKO_PATH"],
-                                 cwd=b2g_dir, env={"B2G_DIR": b2g_dir})
-
-        if len(gecko_dir) == 0:
-            gecko_dir = os.path.join(b2g_dir, "gecko")
-
-        if os.path.exists(gecko_dir):
-            return gecko_dir
-
-        relative_gecko_dir = os.path.join(b2g_dir, gecko_dir)
-        if os.path.exists(relative_gecko_dir):
-            return relative_gecko_dir
-
-        raise Exception("B2G gecko directory not found: %s" % gecko_dir)
-
     def build_mar(self, signed_zip, output_mar):
         with FotaZip(signed_zip) as fota_zip:
             fota_zip.validate(signed=True)
 
         mar_tool = MarTool()
-        make_full_update = os.path.join(self.find_gecko_dir(), "tools",
+        make_full_update = os.path.join(b2g_config.gecko_path, "tools",
             "update-packaging", "make_full_update.sh")
         if not os.path.exists(make_full_update):
             raise Exception("Couldn't find %s " % make_full_update)
@@ -299,6 +397,34 @@ def build_mar(self, signed_zip, output_mar):
         run_command([make_full_update, output_mar, mar_dir],
             env={"MAR": mar_tool.get_tool()})
 
+class GeckoMarBuilder(object):
+    def __init__(self):
+        self.mar_tool = MarTool()
+        self.mbsdiff_tool = Tool(b2g_config.get_gecko_host_bin("mbsdiff"))
+        packaging_dir = os.path.join(b2g_config.gecko_path, "tools",
+            "update-packaging")
+
+        self.make_full_update = os.path.join(packaging_dir,
+            "make_full_update.sh")
+        if not os.path.exists(self.make_full_update):
+            raise Exception("Couldn't find %s " % make_full_update)
+
+        self.make_incremental_update = os.path.join(packaging_dir,
+            "make_incremental_update.sh")
+        if not os.path.exists(self.make_incremental_update):
+            raise Exception("Couldn't find %s " % make_incremental_update)
+
+    def build_gecko_mar(self, src_dir, output_mar, from_dir=None):
+        if from_dir:
+            args = [self.make_incremental_update, output_mar, from_dir, src_dir]
+        else:
+            args = [self.make_full_update, output_mar, src_dir]
+
+        run_command(args, env={
+            "MAR": self.mar_tool.get_tool(),
+            "MBSDIFF": self.mbsdiff_tool.get_tool()
+        })
+
 class UpdateXmlBuilder(object):
     DEFAULT_URL_TEMPLATE = "http://localhost/%(patch_name)s"
     DEFAULT_UPDATE_TYPE = "minor"
@@ -570,3 +696,5 @@ def override_update_url(self):
     def restart_b2g(self):
         print "Restarting B2G"
         self.adb.shell("stop b2g; start b2g")
+
+b2g_config = B2GConfig()

tools/update-tools/wrap-mar.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2012 Mozilla Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# AUS MARs will also have each entry bz2 compressed. This tool can extract or
+# rebuild these "wrapped" MARs.
+#
+# Warning: If you unwrap, edit a file, then re-wrap a MAR you will lose any
+# metadata that existed in the original MAR, such as signatures.
+
+import argparse
+import os
+import update_tools
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("mar", metavar="MAR", help="MAR archive to (un)wrap")
+    parser.add_argument("dir", metavar="DIR", help="Source or destination " +
+        "directory for (un)wrapping MAR.")
+    parser.add_argument("-u", "--unwrap", dest="unwrap", action="store_true",
+        default=False, help="Unwrap MAR to DIR")
+    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
+        default=False, help="Verbose (un)wrapping")
+
+    args = parser.parse_args()
+    if os.path.isfile(args.dir):
+        parser.error("Path is not a directory: %s" % args.dir)
+
+    try:
+        mar = update_tools.BZip2Mar(args.mar, verbose=args.verbose)
+        action = mar.extract if args.unwrap else mar.create
+        action(args.dir)
+
+        if args.unwrap:
+            print "Unwrapped MAR to %s" % args.dir
+        else:
+            print "Wrapped MAR to %s" % args.mar
+
+    except Exception, e:
+        parser.error(e)
+
+if __name__ == "__main__":
+    main()