Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Merge pull request #170 from marshall/incrementalUpdate

Gecko incremental update tools
  • Loading branch information...
commit fbee23615cebc9348a1db4ec1021eb863b98aa2d 2 parents 2742f88 + 39ec851
Marshall Culpepper marshall authored
BIN  tools/update-tools/bin/darwin-x86/mar
Binary file not shown
BIN  tools/update-tools/bin/linux-x86/mar
Binary file not shown
82 tools/update-tools/build-gecko-mar.py
... ... @@ -0,0 +1,82 @@
  1 +#!/usr/bin/env python
  2 +#
  3 +# Copyright (C) 2012 Mozilla Foundation
  4 +#
  5 +# Licensed under the Apache License, Version 2.0 (the "License");
  6 +# you may not use this file except in compliance with the License.
  7 +# You may obtain a copy of the License at
  8 +#
  9 +# http://www.apache.org/licenses/LICENSE-2.0
  10 +#
  11 +# Unless required by applicable law or agreed to in writing, software
  12 +# distributed under the License is distributed on an "AS IS" BASIS,
  13 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 +# See the License for the specific language governing permissions and
  15 +# limitations under the License.
  16 +#
  17 +# Build a gecko (OTA) full or incremental MAR
  18 +
  19 +import argparse
  20 +import os
  21 +import shutil
  22 +import tempfile
  23 +import update_tools
  24 +
  25 +def unwrap_mar(mar, verbose):
  26 + print "Extracting MAR for incremental update: %s" % mar
  27 + tmpdir = tempfile.mkdtemp()
  28 + bz2_mar = update_tools.BZip2Mar(mar, verbose=verbose)
  29 + bz2_mar.extract(tmpdir)
  30 + return tmpdir
  31 +
  32 +def main():
  33 + parser = argparse.ArgumentParser()
  34 + parser.add_argument("mar", metavar="MAR", help="Destination MAR file")
  35 + parser.add_argument("--dir", metavar="DIR", default=None,
  36 + help="Source directory. When building an \"incremental\" MAR, this can " +
  37 + "also be a MAR for convenience. Default: $PWD")
  38 +
  39 + parser.add_argument("--to", metavar="TO", default=None,
  40 + help="This is a synonym for --dir")
  41 + parser.add_argument("--from", metavar="FROM", dest="from_dir", default=None,
  42 + help="The base directory or MAR to build an incremental MAR from. This " +
  43 + "will build an incremental update MAR between FROM and TO")
  44 + parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
  45 + default=False, help="Enable verbose logging")
  46 +
  47 + args = parser.parse_args()
  48 +
  49 + if os.path.isdir(args.mar):
  50 + parser.error("MAR destination is a directory: %s" % args.mar)
  51 +
  52 + if not args.dir:
  53 + args.dir = args.to if args.to else os.getcwd()
  54 +
  55 + if not args.from_dir and not os.path.isdir(args.dir):
  56 + parser.error("Path is not a directory: %s" % args.dir)
  57 +
  58 + to_tmpdir = from_tmpdir = None
  59 + if args.from_dir and os.path.isfile(args.dir):
  60 + to_tmpdir = unwrap_mar(args.dir, args.verbose)
  61 + args.dir = to_tmpdir
  62 +
  63 + if args.from_dir and os.path.isfile(args.from_dir):
  64 + from_tmpdir = unwrap_mar(args.from_dir, args.verbose)
  65 + args.from_dir = from_tmpdir
  66 +
  67 + try:
  68 + builder = update_tools.GeckoMarBuilder()
  69 + builder.build_gecko_mar(args.dir, args.mar, from_dir=args.from_dir)
  70 + update_type = "incremental" if args.from_dir else "full"
  71 +
  72 + print "Built %s update MAR: %s" % (update_type, args.mar)
  73 + except Exception, e:
  74 + parser.error(e)
  75 + finally:
  76 + if to_tmpdir:
  77 + shutil.rmtree(to_tmpdir)
  78 + if from_tmpdir:
  79 + shutil.rmtree(from_tmpdir)
  80 +
  81 +if __name__ == "__main__":
  82 + main()
188 tools/update-tools/update_tools.py
@@ -104,28 +104,76 @@ def _access_check(fn, mode):
104 104 return name
105 105 return None
106 106
107   -class PrebuiltTool(object):
108   - def __init__(self, name):
  107 +class B2GConfig(object):
  108 + CONFIG_VARS = ("GECKO_PATH", "GECKO_OBJDIR")
  109 +
  110 + def __init__(self):
  111 + shell = ". load-config.sh"
  112 + for var in self.CONFIG_VARS:
  113 + shell += "\necho $%s" % var
  114 +
  115 + result = run_command(["bash", "-c", shell], cwd=b2g_dir,
  116 + env={"B2G_DIR": b2g_dir})
  117 +
  118 + if not result:
  119 + raise Exception("Couldn't parse result of load-config.sh")
  120 +
  121 + lines = result.splitlines()
  122 + if len(lines) != len(self.CONFIG_VARS):
  123 + raise Exception("Wrong number of config vars: %d" % len(lines))
  124 +
  125 + for i in range(len(self.CONFIG_VARS)):
  126 + setattr(self, self.CONFIG_VARS[i].lower(), lines[i].strip())
  127 +
  128 + self.init_gecko_path()
  129 + if not self.gecko_objdir:
  130 + self.gecko_objdir = os.path.join(self.gecko_path, "objdir-gecko")
  131 +
  132 + def init_gecko_path(self):
  133 + if not self.gecko_path:
  134 + self.gecko_path = os.path.join(b2g_dir, "gecko")
  135 +
  136 + if os.path.exists(self.gecko_path):
  137 + return
  138 +
  139 + relative_gecko_path = os.path.join(b2g_dir, self.gecko_path)
  140 + if os.path.exists(relative_gecko_path):
  141 + self.gecko_path = relative_gecko_path
  142 + return
  143 +
  144 + raise Exception("B2G gecko directory not found: %s" % self.gecko_path)
  145 +
  146 + def get_gecko_host_bin(self, path):
  147 + return os.path.join(self.gecko_objdir, "dist", "host", "bin", path)
  148 +
  149 +class Tool(object):
  150 + def __init__(self, path, prebuilt=False):
  151 + self.tool = path
  152 + if prebuilt:
  153 + self.init_prebuilt(path)
  154 +
  155 + if not os.path.exists(self.tool):
  156 + raise Exception("Couldn't find %s " % self.tool)
  157 +
  158 + def init_prebuilt(self, path):
109 159 host_dir = "linux-x86"
110 160 if platform.system() == "Darwin":
111 161 host_dir = "darwin-x86"
112 162
113   - self.tool = os.path.join(bin_dir, host_dir, name)
114   - if not os.path.exists(self.tool):
115   - raise Exception("Couldn't find %s " % self.tool)
  163 + self.tool = os.path.join(bin_dir, host_dir, path)
116 164
117 165 def get_tool(self):
118 166 return self.tool
119 167
120   - def run(self, *args):
121   - return run_command((self.tool,) + args)
  168 + def run(self, *args, **kwargs):
  169 + return run_command((self.tool,) + args, **kwargs)
122 170
123   -class AdbTool(PrebuiltTool):
  171 +class AdbTool(Tool):
124 172 DEVICE = ("-d")
125 173 EMULATOR = ("-e")
126 174
127 175 def __init__(self, device=None):
128   - PrebuiltTool.__init__(self, "adb")
  176 + Tool.__init__(self, "adb", prebuilt=True)
129 177 self.adb_args = ()
130 178 if device in (self.DEVICE, self.EMULATOR):
131 179 self.adb_args = device
@@ -134,7 +182,7 @@ def __init__(self, device=None):
134 182
135 183 def run(self, *args):
136 184 adb_args = self.adb_args + args
137   - return PrebuiltTool.run(self, *adb_args)
  185 + return Tool.run(self, *adb_args)
138 186
139 187 def shell(self, *args):
140 188 return self.run("shell", *args)
@@ -163,9 +211,9 @@ def get_cmdline(self, pid):
163 211 # cmdline is null byte separated and has a trailing null byte
164 212 return result.split("\x00")[:-1]
165 213
166   -class MarTool(PrebuiltTool):
  214 +class MarTool(Tool):
167 215 def __init__(self):
168   - PrebuiltTool.__init__(self, "mar")
  216 + Tool.__init__(self, b2g_config.get_gecko_host_bin("mar"))
169 217
170 218 def list_entries(self, mar_path):
171 219 result = self.run("-t", mar_path)
@@ -177,6 +225,23 @@ def list_entries(self, mar_path):
177 225 entries.append(words[2])
178 226 return entries
179 227
  228 + def create(self, mar_path, src_dir=None):
  229 + if not src_dir:
  230 + src_dir = os.getcwd()
  231 +
  232 + mar_args = ["-c", mar_path]
  233 +
  234 + # The MAR tool wants a listing of each file to add
  235 + for root, dirs, files in os.walk(src_dir):
  236 + for f in files:
  237 + file_path = os.path.join(root, f)
  238 + mar_args.append(os.path.relpath(file_path, src_dir))
  239 +
  240 + self.run(*mar_args, cwd=src_dir)
  241 +
  242 + def extract(self, mar_path, dest_dir=None):
  243 + self.run("-x", mar_path, cwd=dest_dir)
  244 +
180 245 def is_gecko_mar(self, mar_path):
181 246 return not self.is_fota_mar(mar_path)
182 247
@@ -184,6 +249,56 @@ def is_fota_mar(self, mar_path):
184 249 entries = self.list_entries(mar_path)
185 250 return "update.zip" in entries
186 251
  252 +class BZip2Mar(object):
  253 + def __init__(self, mar_file, verbose=False):
  254 + self.mar_file = mar_file
  255 + self.verbose = verbose
  256 + self.mar_tool = MarTool()
  257 + self.bzip2_tool = which("bzip2")
  258 + if not self.bzip2_tool:
  259 + raise Exception("Couldn't find bzip2 on the PATH")
  260 +
  261 + def bzip2(self, *args):
  262 + bzargs = [self.bzip2_tool]
  263 + if self.verbose:
  264 + bzargs.append("-v")
  265 + bzargs.extend(args)
  266 +
  267 + return run_command(bzargs)
  268 +
  269 + def create(self, src_dir):
  270 + if not os.path.exists(src_dir):
  271 + raise Exception("Source directory doesn't exist: %s" % src_dir)
  272 +
  273 + temp_dir = tempfile.mkdtemp()
  274 + for root, dirs, files in os.walk(src_dir):
  275 + for f in files:
  276 + path = os.path.join(root, f)
  277 + rel_file = os.path.relpath(path, src_dir)
  278 + out_file = os.path.join(temp_dir, rel_file)
  279 + out_dir = os.path.dirname(out_file)
  280 + if not os.path.exists(out_dir):
  281 + os.makedirs(out_dir)
  282 +
  283 + shutil.copy(path, out_file)
  284 + self.bzip2("-z", out_file)
  285 + os.rename(out_file + ".bz2", out_file)
  286 +
  287 + self.mar_tool.create(self.mar_file, src_dir=temp_dir)
  288 +
  289 + def extract(self, dest_dir):
  290 + if not os.path.exists(dest_dir):
  291 + os.makedirs(dest_dir)
  292 + if not os.path.exists(dest_dir):
  293 + raise Exception("Couldn't create directory: %s" % dest_dir)
  294 +
  295 + self.mar_tool.extract(self.mar_file, dest_dir=dest_dir)
  296 + for root, dirs, files in os.walk(dest_dir):
  297 + for f in files:
  298 + path = os.path.join(root, f)
  299 + os.rename(path, path + ".bz2")
  300 + self.bzip2("-d", path + ".bz2")
  301 +
187 302 class FotaZip(zipfile.ZipFile):
188 303 UPDATE_BINARY = "META-INF/com/google/android/update-binary"
189 304 UPDATER_SCRIPT = "META-INF/com/google/android/updater-script"
@@ -260,29 +375,12 @@ def __init__(self):
260 375 def __del__(self):
261 376 shutil.rmtree(self.stage_dir)
262 377
263   - def find_gecko_dir(self):
264   - gecko_dir = run_command(["bash", "-c",
265   - ". load-config.sh; echo -n $GECKO_PATH"],
266   - cwd=b2g_dir, env={"B2G_DIR": b2g_dir})
267   -
268   - if len(gecko_dir) == 0:
269   - gecko_dir = os.path.join(b2g_dir, "gecko")
270   -
271   - if os.path.exists(gecko_dir):
272   - return gecko_dir
273   -
274   - relative_gecko_dir = os.path.join(b2g_dir, gecko_dir)
275   - if os.path.exists(relative_gecko_dir):
276   - return relative_gecko_dir
277   -
278   - raise Exception("B2G gecko directory not found: %s" % gecko_dir)
279   -
280 378 def build_mar(self, signed_zip, output_mar):
281 379 with FotaZip(signed_zip) as fota_zip:
282 380 fota_zip.validate(signed=True)
283 381
284 382 mar_tool = MarTool()
285   - make_full_update = os.path.join(self.find_gecko_dir(), "tools",
  383 + make_full_update = os.path.join(b2g_config.gecko_path, "tools",
286 384 "update-packaging", "make_full_update.sh")
287 385 if not os.path.exists(make_full_update):
288 386 raise Exception("Couldn't find %s " % make_full_update)
@@ -299,6 +397,34 @@ def build_mar(self, signed_zip, output_mar):
299 397 run_command([make_full_update, output_mar, mar_dir],
300 398 env={"MAR": mar_tool.get_tool()})
301 399
  400 +class GeckoMarBuilder(object):
  401 + def __init__(self):
  402 + self.mar_tool = MarTool()
  403 + self.mbsdiff_tool = Tool(b2g_config.get_gecko_host_bin("mbsdiff"))
  404 + packaging_dir = os.path.join(b2g_config.gecko_path, "tools",
  405 + "update-packaging")
  406 +
  407 + self.make_full_update = os.path.join(packaging_dir,
  408 + "make_full_update.sh")
  409 + if not os.path.exists(self.make_full_update):
  410 + raise Exception("Couldn't find %s " % make_full_update)
  411 +
  412 + self.make_incremental_update = os.path.join(packaging_dir,
  413 + "make_incremental_update.sh")
  414 + if not os.path.exists(self.make_incremental_update):
  415 + raise Exception("Couldn't find %s " % make_incremental_update)
  416 +
  417 + def build_gecko_mar(self, src_dir, output_mar, from_dir=None):
  418 + if from_dir:
  419 + args = [self.make_incremental_update, output_mar, from_dir, src_dir]
  420 + else:
  421 + args = [self.make_full_update, output_mar, src_dir]
  422 +
  423 + run_command(args, env={
  424 + "MAR": self.mar_tool.get_tool(),
  425 + "MBSDIFF": self.mbsdiff_tool.get_tool()
  426 + })
  427 +
302 428 class UpdateXmlBuilder(object):
303 429 DEFAULT_URL_TEMPLATE = "http://localhost/%(patch_name)s"
304 430 DEFAULT_UPDATE_TYPE = "minor"
@@ -570,3 +696,5 @@ def override_update_url(self):
570 696 def restart_b2g(self):
571 697 print "Restarting B2G"
572 698 self.adb.shell("stop b2g; start b2g")
  699 +
  700 +b2g_config = B2GConfig()
55 tools/update-tools/wrap-mar.py
... ... @@ -0,0 +1,55 @@
  1 +#!/usr/bin/env python
  2 +#
  3 +# Copyright (C) 2012 Mozilla Foundation
  4 +#
  5 +# Licensed under the Apache License, Version 2.0 (the "License");
  6 +# you may not use this file except in compliance with the License.
  7 +# You may obtain a copy of the License at
  8 +#
  9 +# http://www.apache.org/licenses/LICENSE-2.0
  10 +#
  11 +# Unless required by applicable law or agreed to in writing, software
  12 +# distributed under the License is distributed on an "AS IS" BASIS,
  13 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 +# See the License for the specific language governing permissions and
  15 +# limitations under the License.
  16 +#
  17 +# AUS MARs will also have each entry bz2 compressed. This tool can extract or
  18 +# rebuild these "wrapped" MARs.
  19 +#
  20 +# Warning: If you unwrap, edit a file, then re-wrap a MAR you will lose any
  21 +# metadata that existed in the original MAR, such as signatures.
  22 +
  23 +import argparse
  24 +import os
  25 +import update_tools
  26 +
  27 +def main():
  28 + parser = argparse.ArgumentParser()
  29 + parser.add_argument("mar", metavar="MAR", help="MAR archive to (un)wrap")
  30 + parser.add_argument("dir", metavar="DIR", help="Source or destination " +
  31 + "directory for (un)wrapping MAR.")
  32 + parser.add_argument("-u", "--unwrap", dest="unwrap", action="store_true",
  33 + default=False, help="Unwrap MAR to DIR")
  34 + parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
  35 + default=False, help="Verbose (un)wrapping")
  36 +
  37 + args = parser.parse_args()
  38 + if os.path.isfile(args.dir):
  39 + parser.error("Path is not a directory: %s" % args.dir)
  40 +
  41 + try:
  42 + mar = update_tools.BZip2Mar(args.mar, verbose=args.verbose)
  43 + action = mar.extract if args.unwrap else mar.create
  44 + action(args.dir)
  45 +
  46 + if args.unwrap:
  47 + print "Unwrapped MAR to %s" % args.dir
  48 + else:
  49 + print "Wrapped MAR to %s" % args.mar
  50 +
  51 + except Exception, e:
  52 + parser.error(e)
  53 +
  54 +if __name__ == "__main__":
  55 + main()

0 comments on commit fbee236

Please sign in to comment.
Something went wrong with that request. Please try again.