Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

New update tools for building a full / incremental gecko MAR, and wra…

…pping /

unwrapping a bz2 compressed MAR.
  • Loading branch information...
commit 1f89f7ab3e862660424cb202b7498b2c891c192d 1 parent b3fa904
@marshall marshall authored
View
BIN  tools/update-tools/bin/darwin-x86/mar
Binary file not shown
View
BIN  tools/update-tools/bin/linux-x86/mar
Binary file not shown
View
82 tools/update-tools/build-gecko-mar.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2012 Mozilla Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Build a gecko (OTA) full or incremental MAR
+
+import argparse
+import os
+import shutil
+import tempfile
+import update_tools
+
+def unwrap_mar(mar, verbose):
+ print "Extracting MAR for incremental update: %s" % mar
+ tmpdir = tempfile.mkdtemp()
+ bz2_mar = update_tools.BZip2Mar(mar, verbose=verbose)
+ bz2_mar.extract(tmpdir)
+ return tmpdir
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("mar", metavar="MAR", help="Destination MAR file")
+ parser.add_argument("--dir", metavar="DIR", default=None,
+ help="Source directory. When building an \"incremental\" MAR, this can " +
+ "also be a MAR for convenience. Default: $PWD")
+
+ parser.add_argument("--to", metavar="TO", default=None,
+ help="This is a synonym for --dir")
+ parser.add_argument("--from", metavar="FROM", dest="from_dir", default=None,
+ help="The base directory or MAR to build an incremental MAR from. This " +
+ "will build an incremental update MAR between FROM and TO")
+ parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
+ default=False, help="Enable verbose logging")
+
+ args = parser.parse_args()
+
+ if os.path.isdir(args.mar):
+ parser.error("MAR destination is a directory: %s" % args.mar)
+
+ if not args.dir:
+ args.dir = args.to if args.to else os.getcwd()
+
+ if not args.from_dir and not os.path.isdir(args.dir):
+ parser.error("Path is not a directory: %s" % args.dir)
+
+ to_tmpdir = from_tmpdir = None
+ if args.from_dir and os.path.isfile(args.dir):
+ to_tmpdir = unwrap_mar(args.dir, args.verbose)
+ args.dir = to_tmpdir
+
+ if args.from_dir and os.path.isfile(args.from_dir):
+ from_tmpdir = unwrap_mar(args.from_dir, args.verbose)
+ args.from_dir = from_tmpdir
+
+ try:
+ builder = update_tools.GeckoMarBuilder()
+ builder.build_gecko_mar(args.dir, args.mar, from_dir=args.from_dir)
+ update_type = "incremental" if args.from_dir else "full"
+
+ print "Built %s update MAR: %s" % (update_type, args.mar)
+ except Exception, e:
+ parser.error(e)
+ finally:
+ if to_tmpdir:
+ shutil.rmtree(to_tmpdir)
+ if from_tmpdir:
+ shutil.rmtree(from_tmpdir)
+
+if __name__ == "__main__":
+ main()
View
190 tools/update-tools/update_tools.py
@@ -104,28 +104,78 @@ def _access_check(fn, mode):
return name
return None
-class PrebuiltTool(object):
- def __init__(self, name):
+class B2GConfig(object):
+ CONFIG_VARS = ("GECKO_PATH", "GECKO_OBJDIR")
+
+ def __init__(self):
+ shell = ". load-config.sh"
+ for var in self.CONFIG_VARS:
+ shell += "\necho $%s" % var
+
+ result = run_command(["bash", "-c", shell], cwd=b2g_dir,
+ env={"B2G_DIR": b2g_dir})
+
+ if not result:
+ raise Exception("Couldn't parse result of load-config.sh")
+
+ lines = result.splitlines()
+ if len(lines) != len(self.CONFIG_VARS):
+ raise Exception("Wrong number of config vars: %d" % len(lines))
+
+ i = 0
+ for var in self.CONFIG_VARS:
+ setattr(self, var.lower(), lines[i].strip())
+ i += 1
+
+ self.init_gecko_path()
+ if not self.gecko_objdir:
+ self.gecko_objdir = os.path.join(self.gecko_path, "objdir-gecko")
+
+ def init_gecko_path(self):
+ if not self.gecko_path or len(self.gecko_path) == 0:
+ self.gecko_path = os.path.join(b2g_dir, "gecko")
+
+ if os.path.exists(self.gecko_path):
+ return
+
+ relative_gecko_path = os.path.join(b2g_dir, self.gecko_path)
+ if os.path.exists(relative_gecko_path):
+ self.gecko_path = relative_gecko_path
+ return
+
+ raise Exception("B2G gecko directory not found: %s" % self.gecko_path)
+
+ def get_gecko_host_bin(self, path):
+ return os.path.join(self.gecko_objdir, "dist", "host", "bin", path)
+
+class Tool(object):
+ def __init__(self, path, prebuilt=False):
+ self.tool = path
+ if prebuilt:
+ self.init_prebuilt(path)
+
+ if not os.path.exists(self.tool):
+ raise Exception("Couldn't find %s " % self.tool)
+
+ def init_prebuilt(self, path):
host_dir = "linux-x86"
if platform.system() == "Darwin":
host_dir = "darwin-x86"
- self.tool = os.path.join(bin_dir, host_dir, name)
- if not os.path.exists(self.tool):
- raise Exception("Couldn't find %s " % self.tool)
+ self.tool = os.path.join(bin_dir, host_dir, path)
def get_tool(self):
return self.tool
- def run(self, *args):
- return run_command((self.tool,) + args)
+ def run(self, *args, **kwargs):
+ return run_command((self.tool,) + args, **kwargs)
-class AdbTool(PrebuiltTool):
+class AdbTool(Tool):
DEVICE = ("-d")
EMULATOR = ("-e")
def __init__(self, device=None):
- PrebuiltTool.__init__(self, "adb")
+ Tool.__init__(self, "adb", prebuilt=True)
self.adb_args = ()
if device in (self.DEVICE, self.EMULATOR):
self.adb_args = device
@@ -134,7 +184,7 @@ def __init__(self, device=None):
def run(self, *args):
adb_args = self.adb_args + args
- return PrebuiltTool.run(self, *adb_args)
+ return Tool.run(self, *adb_args)
def shell(self, *args):
return self.run("shell", *args)
@@ -163,9 +213,9 @@ def get_cmdline(self, pid):
# cmdline is null byte separated and has a trailing null byte
return result.split("\x00")[:-1]
-class MarTool(PrebuiltTool):
+class MarTool(Tool):
def __init__(self):
- PrebuiltTool.__init__(self, "mar")
+ Tool.__init__(self, b2g_config.get_gecko_host_bin("mar"))
def list_entries(self, mar_path):
result = self.run("-t", mar_path)
@@ -177,6 +227,23 @@ def list_entries(self, mar_path):
entries.append(words[2])
return entries
+ def create(self, mar_path, src_dir=None):
+ if not src_dir:
+ src_dir = os.getcwd()
+
+ mar_args = ["-c", mar_path]
+
+ # The MAR tool wants a listing of each file to add
+ for root, dirs, files in os.walk(src_dir):
+ for f in files:
+ file_path = os.path.join(root, f)
+ mar_args.append(os.path.relpath(file_path, src_dir))
+
+ self.run(*mar_args, cwd=src_dir)
+
+ def extract(self, mar_path, dest_dir=None):
+ self.run("-x", mar_path, cwd=dest_dir)
+
def is_gecko_mar(self, mar_path):
return not self.is_fota_mar(mar_path)
@@ -184,6 +251,56 @@ def is_fota_mar(self, mar_path):
entries = self.list_entries(mar_path)
return "update.zip" in entries
+class BZip2Mar(object):
+ def __init__(self, mar_file, verbose=False):
+ self.mar_file = mar_file
+ self.verbose = verbose
+ self.mar_tool = MarTool()
+ self.bzip2_tool = which("bzip2")
+ if not self.bzip2_tool:
+ raise Exception("Couldn't find bzip2 on the PATH")
+
+ def bzip2(self, *args):
+ bzargs = [self.bzip2_tool]
+ if self.verbose:
+ bzargs.append("-v")
+ bzargs.extend(args)
+
+ return run_command(bzargs)
+
+ def create(self, src_dir):
+ if not os.path.exists(src_dir):
+ raise Exception("Source directory doesn't exist: %s" % src_dir)
+
+ temp_dir = tempfile.mkdtemp()
+ for root, dirs, files in os.walk(src_dir):
+ for f in files:
+ path = os.path.join(root, f)
+ rel_file = os.path.relpath(path, src_dir)
+ out_file = os.path.join(temp_dir, rel_file)
+ out_dir = os.path.dirname(out_file)
+ if not os.path.exists(out_dir):
+ os.makedirs(out_dir)
+
+ shutil.copy(path, out_file)
+ self.bzip2("-z", out_file)
+ os.rename(out_file + ".bz2", out_file)
+
+ self.mar_tool.create(self.mar_file, src_dir=temp_dir)
+
+ def extract(self, dest_dir):
+ if not os.path.exists(dest_dir):
+ os.makedirs(dest_dir)
+ if not os.path.exists(dest_dir):
+ raise Exception("Couldn't create directory: %s" % dest_dir)
+
+ self.mar_tool.extract(self.mar_file, dest_dir=dest_dir)
+ for root, dirs, files in os.walk(dest_dir):
+ for f in files:
+ path = os.path.join(root, f)
+ os.rename(path, path + ".bz2")
+ self.bzip2("-d", path + ".bz2")
+
class FotaZip(zipfile.ZipFile):
UPDATE_BINARY = "META-INF/com/google/android/update-binary"
UPDATER_SCRIPT = "META-INF/com/google/android/updater-script"
@@ -260,29 +377,12 @@ def __init__(self):
def __del__(self):
shutil.rmtree(self.stage_dir)
- def find_gecko_dir(self):
- gecko_dir = run_command(["bash", "-c",
- ". load-config.sh; echo -n $GECKO_PATH"],
- cwd=b2g_dir, env={"B2G_DIR": b2g_dir})
-
- if len(gecko_dir) == 0:
- gecko_dir = os.path.join(b2g_dir, "gecko")
-
- if os.path.exists(gecko_dir):
- return gecko_dir
-
- relative_gecko_dir = os.path.join(b2g_dir, gecko_dir)
- if os.path.exists(relative_gecko_dir):
- return relative_gecko_dir
-
- raise Exception("B2G gecko directory not found: %s" % gecko_dir)
-
def build_mar(self, signed_zip, output_mar):
with FotaZip(signed_zip) as fota_zip:
fota_zip.validate(signed=True)
mar_tool = MarTool()
- make_full_update = os.path.join(self.find_gecko_dir(), "tools",
+ make_full_update = os.path.join(b2g_config.gecko_path, "tools",
"update-packaging", "make_full_update.sh")
if not os.path.exists(make_full_update):
raise Exception("Couldn't find %s " % make_full_update)
@@ -299,6 +399,34 @@ def build_mar(self, signed_zip, output_mar):
run_command([make_full_update, output_mar, mar_dir],
env={"MAR": mar_tool.get_tool()})
+class GeckoMarBuilder(object):
+ def __init__(self):
+ self.mar_tool = MarTool()
+ self.mbsdiff_tool = Tool(b2g_config.get_gecko_host_bin("mbsdiff"))
+ packaging_dir = os.path.join(b2g_config.gecko_path, "tools",
+ "update-packaging")
+
+ self.make_full_update = os.path.join(packaging_dir,
+ "make_full_update.sh")
+ if not os.path.exists(self.make_full_update):
+ raise Exception("Couldn't find %s " % make_full_update)
+
+ self.make_incremental_update = os.path.join(packaging_dir,
+ "make_incremental_update.sh")
+ if not os.path.exists(self.make_incremental_update):
+ raise Exception("Couldn't find %s " % make_incremental_update)
+
+ def build_gecko_mar(self, src_dir, output_mar, from_dir=None):
+ if from_dir:
+ args = [self.make_incremental_update, output_mar, from_dir, src_dir]
+ else:
+ args = [self.make_full_update, output_mar, src_dir]
+
+ run_command(args, env={
+ "MAR": self.mar_tool.get_tool(),
+ "MBSDIFF": self.mbsdiff_tool.get_tool()
+ })
+
class UpdateXmlBuilder(object):
DEFAULT_URL_TEMPLATE = "http://localhost/%(patch_name)s"
DEFAULT_UPDATE_TYPE = "minor"
@@ -570,3 +698,5 @@ def override_update_url(self):
def restart_b2g(self):
print "Restarting B2G"
self.adb.shell("stop b2g; start b2g")
+
+b2g_config = B2GConfig()
View
55 tools/update-tools/wrap-mar.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2012 Mozilla Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# AUS MARs will also have each entry bz2 compressed. This tool can extract or
+# rebuild these "wrapped" MARs.
+#
+# Warning: If you unwrap, edit a file, then re-wrap a MAR you will lose any
+# metadata that existed in the original MAR, such as signatures.
+
+import argparse
+import os
+import update_tools
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("mar", metavar="MAR", help="MAR archive to (un)wrap")
+ parser.add_argument("dir", metavar="DIR", help="Source or destination " +
+ "directory for (un)wrapping MAR.")
+ parser.add_argument("-u", "--unwrap", dest="unwrap", action="store_true",
+ default=False, help="Unwrap MAR to DIR")
+ parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
+ default=False, help="Verbose (un)wrapping")
+
+ args = parser.parse_args()
+ if os.path.isfile(args.dir):
+ parser.error("Path is not a directory: %s" % args.dir)
+
+ try:
+ mar = update_tools.BZip2Mar(args.mar, verbose=args.verbose)
+ action = mar.extract if args.unwrap else mar.create
+ action(args.dir)
+
+ if args.unwrap:
+ print "Unwrapped MAR to %s" % args.dir
+ else:
+ print "Wrapped MAR to %s" % args.mar
+
+ except Exception, e:
+ parser.error(e)
+
+if __name__ == "__main__":
+ main()
Please sign in to comment.
Something went wrong with that request. Please try again.