From df079d14285425ac531ec396eff3ccac7efaea17 Mon Sep 17 00:00:00 2001 From: Kevin Grandon Date: Fri, 20 Dec 2013 07:38:07 +0100 Subject: [PATCH] Revert "Bug 945553 - Rewrite install-gaia.py to install-gaia.js" This reverts commit b9129ad69a9ea91fda0c3f4c626c2f2eaf15d230. --- Makefile | 41 ++++++- build/install-gaia.js | 95 ----------------- build/install-gaia.py | 188 +++++++++++++++++++++++++++++++++ build/otoro-install-busybox.sh | 17 +++ build/utils.js | 114 -------------------- 5 files changed, 243 insertions(+), 212 deletions(-) delete mode 100644 build/install-gaia.js create mode 100644 build/install-gaia.py create mode 100755 build/otoro-install-busybox.sh diff --git a/Makefile b/Makefile index ea6448c4800d..dcb4b237eb47 100644 --- a/Makefile +++ b/Makefile @@ -339,7 +339,6 @@ TEST_DIRS ?= $(CURDIR)/tests define BUILD_CONFIG { \ - "ADB" : "$(adb)", \ "GAIA_DIR" : "$(CURDIR)", \ "PROFILE_DIR" : "$(CURDIR)$(SEP)$(PROFILE_FOLDER)", \ "PROFILE_FOLDER" : "$(PROFILE_FOLDER)", \ @@ -352,7 +351,6 @@ define BUILD_CONFIG "HOMESCREEN" : "$(HOMESCREEN)", \ "GAIA_PORT" : "$(GAIA_PORT)", \ "GAIA_LOCALES_PATH" : "$(GAIA_LOCALES_PATH)", \ - "GAIA_INSTALL_PARENT" : "$(GAIA_INSTALL_PARENT)", \ "LOCALES_FILE" : "$(subst \,\\,$(LOCALES_FILE))", \ "GAIA_KEYBOARD_LAYOUTS" : "$(GAIA_KEYBOARD_LAYOUTS)", \ "BUILD_APP_NAME" : "$(BUILD_APP_NAME)", \ @@ -901,8 +899,45 @@ forward: # But if you're working on just gaia itself, and you already have B2G firmware # on your phone, and you have adb in your path, then you can use the # install-gaia target to update the gaia files and reboot b2g + +# APP_NAME and APP_PID are used in ifeq calls so they need to be defined +# globally +APP_NAME := $(shell cat *apps/${BUILD_APP_NAME}/manifest.webapp | grep name | head -1 | cut -d '"' -f 4 | cut -b 1-15) +APP_PID := $(shell adb shell b2g-ps | grep '^${APP_NAME}' | sed 's/^${APP_NAME}\s*//' | awk '{ print $$2 }') +install-gaia: TARGET_FOLDER = webapps/$(BUILD_APP_NAME).$(GAIA_DOMAIN) install-gaia: adb-remount $(PROFILE_FOLDER) - @$(call run-js-command, install-gaia) + @$(ADB) start-server +ifeq ($(BUILD_APP_NAME),*) + @echo 'Stopping b2g' + @$(ADB) shell stop b2g +else ifeq ($(BUILD_APP_NAME), system) + @echo 'Stopping b2g' + @$(ADB) shell stop b2g +endif + @$(ADB) shell rm -r $(MSYS_FIX)/cache/* > /dev/null + +ifeq ($(BUILD_APP_NAME),*) + python build/install-gaia.py "$(ADB)" "$(MSYS_FIX)$(GAIA_INSTALL_PARENT)" "$(PROFILE_FOLDER)" +else + @echo "Pushing manifest.webapp application.zip for ${BUILD_APP_NAME}..." + @$(ADB) push $(PROFILE_FOLDER)/$(TARGET_FOLDER)/manifest.webapp $(MSYS_FIX)$(GAIA_INSTALL_PARENT)/$(TARGET_FOLDER)/manifest.webapp + @$(ADB) push $(PROFILE_FOLDER)/$(TARGET_FOLDER)/application.zip $(MSYS_FIX)$(GAIA_INSTALL_PARENT)/$(TARGET_FOLDER)/application.zip +endif + +ifdef VARIANT_PATH + $(ADB) shell 'rm -r $(MSYS_FIX)/data/local/svoperapps' + $(ADB) push $(PROFILE_FOLDER)/svoperapps $(MSYS_FIX)/data/local/svoperapps +endif +ifeq ($(BUILD_APP_NAME),*) + @echo "Installed gaia into $(PROFILE_FOLDER)/." + @echo 'Starting b2g' + @$(ADB) shell start b2g +else ifeq ($(BUILD_APP_NAME), system) + @echo 'Starting b2g' + @$(ADB) shell start b2g +else ifneq (${APP_PID},) + @$(ADB) shell kill ${APP_PID} +endif # Copy demo media to the sdcard. # If we've got old style directories on the phone, rename them first. diff --git a/build/install-gaia.js b/build/install-gaia.js deleted file mode 100644 index 4f14e909ded1..000000000000 --- a/build/install-gaia.js +++ /dev/null @@ -1,95 +0,0 @@ -'use strict'; - -var utils = require('./utils'); -var adb = new utils.Commander('adb'); -var sh = new utils.Commander('sh'); - -var gaiaDir; -var remotePath; -var profileFolder; - -function installGaia() { - var webapps_path = remotePath + '/webapps'; - adb.run(['shell', 'rm', '-r', webapps_path]); - adb.run(['shell', 'rm', '/data/local/user.js']); - adb.run(['push', utils.joinPath(profileFolder, 'webapps'), webapps_path]); - adb.run(['push', utils.joinPath(profileFolder, 'user.js'), - '/data/local/user.js']); - - var indexedDbFile = utils.getFile(profileFolder, 'indexedDB'); - if (indexedDbFile.exists() && indexedDbFile.isDirectory()) { - adb.run(['push', indexedDbFile.path, '/data/local/indexedDB']); - } - return true; -} - -function getPid(appName) { - var tempFileName = 'tmpFile'; - sh.run(['-c', 'adb shell b2g-ps > ' + tempFileName]); - var tempFile = utils.getFile(utils.joinPath(gaiaDir, tempFileName)); - var content = utils.getFileContent(tempFile); - var pidMap = utils.psParser(content); - sh.run(['-c', 'rm ' + tempFileName]); - return pidMap[appName] ? pidMap[appName].pid : null; -} - -function execute(options) { - const paths = utils.getEnvPath(); - const gaiaInstallParent = options.GAIA_INSTALL_PARENT; - const buildAppName = options.BUILD_APP_NAME; - - remotePath = gaiaInstallParent || '/system/b2g/webapps'; - - adb.initPath(paths); - sh.initPath(paths); - - gaiaDir = options.GAIA_DIR; - - adb.run(['start-server']); - - var profile = utils.getFile(options.PROFILE_DIR); - if (profile.isDirectory()) { - profileFolder = options.PROFILE_DIR; - } else { - throw new Error(' -*- build/install-gaia.js: cannot locate' + - 'profile folder in ' + options.PROFILE_FOLDER); - } - - if (buildAppName === '*' || buildAppName === 'system') { - adb.run(['shell', 'stop', 'b2g']); - } else { - var appPid = getPid(buildAppName); - if (appPid) { - adb.run(['shell', 'kill', appPid]); - } - } - - adb.run(['shell','rm -r /cache/*']); - - if (buildAppName === '*') { - installGaia(); - } else { - var targetFolder = utils.joinPath( - profileFolder, 'webapps', - buildAppName + '.' + options.GAIA_DOMAIN); - adb.run(['push', - utils.joinPath(targetFolder, 'manifest.webapp'), - remotePath + '/' + targetFolder + '/manifest.webapp']); - adb.run(['push', - utils.joinPath(targetFolder, 'application.zip'), - remotePath + '/' + targetFolder + '/application.zip']); - } - - if (options.VARIANT_PATH) { - var svoperappsUrl = '/data/local/svoperapps'; - adb.run(['shell', 'rm -r ' + svoperappsUrl]); - adb.run(['push', utils.joinPath(profileFolder, 'svoperapps'), - svoperappsUrl]) - } - - if (buildAppName === '*' || buildAppName === 'system') { - adb.run(['shell', 'start', 'b2g']); - } -} - -exports.execute = execute; diff --git a/build/install-gaia.py b/build/install-gaia.py new file mode 100644 index 000000000000..d7617b9a33eb --- /dev/null +++ b/build/install-gaia.py @@ -0,0 +1,188 @@ +"""Usage: python %prog [ADB_PATH] [REMOTE_PATH] [PROFILE_FOLDER] + +ADB_PATH is the path to the |adb| executable we should run. +REMOTE_PATH is the path to push the gaia webapps directory to. +PROFILE_FOLDER is the name of the profile folder. defaults to `profile` + +Used by |make install-gaia| to push files to a device. You shouldn't run +this file directly. + +""" + +import sys +import os +import hashlib +import subprocess +from tempfile import mkstemp + +def compute_local_hash(filename, hashes): + h = hashlib.sha1() + with open(filename,'rb') as f: + for chunk in iter(lambda: f.read(256 * h.block_size), b''): + h.update(chunk) + hashes[filename] = h.hexdigest() + +def compute_local_hashes_in_dir(dir, hashes): + def visit(arg, dirname, names): + for filename in [os.path.join(dirname, name) for name in names]: + if not os.path.isfile(filename): + continue + compute_local_hash(filename, hashes) + + os.path.walk(dir, visit, None) + +def compute_local_hashes(): + hashes = {} + compute_local_hashes_in_dir('webapps', hashes) + compute_local_hash('user.js', hashes) + return hashes + +def adb_push(local, remote): + global adb_cmd + subprocess.check_call([adb_cmd, 'push', local, remote]) + +def adb_shell(cmd, ignore_error=False): + global adb_cmd + + # Output the return code so we can check whether the command executed + # successfully. + new_cmd = cmd + '; echo "RETURN CODE: $?"' + + # universal_newlines=True because adb shell returns CRLF separators. + proc = subprocess.Popen([adb_cmd, 'shell', new_cmd], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + (stdout, stderr) = proc.communicate() + if stderr.strip(): + raise Exception('adb shell "%s" returned the following unexpected error: "%s"' % + (cmd, stderr.strip())) + if proc.returncode != 0: + raise Exception('adb shell "%s" exited with error %d' % (cmd, proc.returncode)) + + split = [line for line in stdout.split('\n') if line.strip()] + if not ignore_error and not split[-1].startswith('RETURN CODE: 0'): + raise Exception('adb shell "%s" did not complete successfully. Output:\n%s' % (cmd, stdout)) + + # Don't return the "RETURN CODE: 0" line! + return split[0:-1] + + +def compute_remote_hashes(): + hashes = {} + adb_out = adb_shell('cd /data/local && find . -type f | xargs sha1sum') + for (hash, filename) in [line.split() for line in adb_out]: + # Strip off './' from the filename. + if filename.startswith('./'): + filename = filename[2:] + else: + raise Exception('Unexpected filename %s' % filename) + + hashes[filename] = hash + return hashes + +INDEXED_DB_FOLDER = 'indexedDB/' + +def remove_from_remote(local_hashes, remote_hashes): + """Remove any files from the remote device which don't appear in + local_hashes. + + """ + + # Keep indexedDB content + to_keep = set() + for path in remote_hashes: + if path[:len(INDEXED_DB_FOLDER)] == INDEXED_DB_FOLDER: + to_keep.add(path) + + to_remove = list(set(remote_hashes.keys()) - set(local_hashes.keys()) - to_keep) + + if not to_remove: + return + + print 'Removing from device:\n%s\n' % '\n'.join(to_remove) + # Chunk to_remove into 25 files at a time so we don't send too much over + # adb_shell at once. + for files in [to_remove[pos:pos + 25] for pos in xrange(0, len(to_remove), 25)]: + adb_shell('cd /data/local && rm -f %s' % ' '.join(files)) + +def push_to_remote(local_hashes, remote_hashes): + global adb_cmd + + to_push = set() + for (k, v) in local_hashes.items(): + if k not in remote_hashes or remote_hashes[k] != local_hashes[k]: + to_push.add(k) + + if not to_push: + return + + print 'Pushing to device:\n%s' % '\n'.join(list(to_push)) + + tmpfile, tmpfilename = mkstemp() + try: + subprocess.check_call(['tar', '-czf', tmpfilename] + list(to_push)) + adb_push(tmpfilename, '/data/local') + basename = os.path.basename(tmpfilename) + adb_shell('cd /data/local && tar -xzf %s && rm %s' % (basename, basename)) + finally: + os.remove(tmpfilename) + +def install_gaia_fast(): + global profile_folder + os.chdir(profile_folder) + try: + local_hashes = compute_local_hashes() + remote_hashes = compute_remote_hashes() + remove_from_remote(local_hashes, remote_hashes) + push_to_remote(local_hashes, remote_hashes) + finally: + os.chdir('..') + +def install_gaia_slow(): + global adb_cmd, remote_path, profile_folder + webapps_path = remote_path + '/webapps' + adb_shell("rm -r " + webapps_path, ignore_error=True) + adb_shell("rm /data/local/user.js", ignore_error=True) + adb_push(profile_folder + '/webapps', webapps_path) + adb_push(profile_folder + '/user.js', '/data/local/user.js') + +def install_preload_data(): + global profile_folder + db_path = profile_folder + '/indexedDB/' + if os.path.exists(db_path): + adb_push(db_path, '/data/local/indexedDB') + +def install_gaia(): + global remote_path + try: + if remote_path == "/system/b2g": + # XXX Force slow method until we fix the fast one to support + # files in both /system/b2g and /data/local + # install_gaia_fast() + install_gaia_slow() + else: + install_gaia_fast() + except: + # If anything goes wrong, fall back to the slow method. + install_gaia_slow() + install_preload_data() + +if __name__ == '__main__': + if len(sys.argv) > 4: + print >>sys.stderr, 'Too many arguments!\n' + print >>sys.stderr, \ + 'Usage: python %s [ADB_PATH] [REMOTE_PATH] [PROFILE_FOLDER]\n' % __FILE__ + sys.exit(1) + + adb_cmd = 'adb' + remote_path = '/data/local/webapps' + profile_folder = 'profile' + if len(sys.argv) >= 2: + adb_cmd = sys.argv[1] + if len(sys.argv) >= 3: + remote_path = sys.argv[2] + if len(sys.argv) >= 4: + profile_folder = sys.argv[3] + + install_gaia() diff --git a/build/otoro-install-busybox.sh b/build/otoro-install-busybox.sh new file mode 100755 index 000000000000..8fa138bca987 --- /dev/null +++ b/build/otoro-install-busybox.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# This shell script installs busybox on the device. +# This lets us take the fast path in install-gaia.py. + +# Remount file system with read/write permissions +adb shell "mount -o rw,remount -t rootfs /" +adb shell "mkdir -p /system/vendor/bin" +adb push busybox-armv6l /vendor/bin/busybox +adb shell "chmod 555 /vendor/bin/busybox" + +# Perform the symbolic links +adb shell "for x in \`busybox --list\`; do ln -s /vendor/bin/busybox /vendor/bin/\$x; done" + +# Remount file system with read-only permissions +adb shell "mount -o ro,remount -t rootfs /" + diff --git a/build/utils.js b/build/utils.js index 956ebe277b94..ba6ce47eaabc 100644 --- a/build/utils.js +++ b/build/utils.js @@ -41,16 +41,10 @@ function ls(dir, recursive, exclude) { return results; } -function getOsType() { - return Cc["@mozilla.org/xre/app-info;1"] - .getService(Ci.nsIXULRuntime).OS; -} - function getFileContent(file) { try { let fileStream = Cc['@mozilla.org/network/file-input-stream;1'] .createInstance(Ci.nsIFileInputStream); - fileStream.init(file, 1, 0, false); let converterStream = Cc['@mozilla.org/intl/converter-input-stream;1'] @@ -587,114 +581,10 @@ function getApp(parent, appname, gaiadir, distdir) { return app; } -function getAppName(manifestPath) { - var file = getFile(manifestPath); - var content = getJSON(file); - return content.name; -} - function normalizeString(appname) { return appname.replace(' ', '-').toLowerCase().replace(/\W/g, ''); } -/** - * We can use Commander to execute a shell command. - * - * Note: it requires to inititialize execute path before trigger run. - * ex: adb = new Commander('adb'); - */ -function Commander(cmd) { - var command = - (getOsType().indexOf('WIN') !== -1 && cmd.indexOf('.exe') === -1) ? - cmd + '.exe' : cmd; - var _path; - var _file = null; - - // paths can be string or array, we'll eventually store one workable - // path as _path. - this.initPath = function (paths) { - if (typeof paths === 'string') { - _path = paths; - } else if (typeof paths === 'object' && paths.length) { - for (var p in paths) { - try { - var result = getFile(paths[p], command); - if (result && result.exists()) { - _path = paths[p]; - _file = result; - break; - } - } catch (e) { - // Windows may throw error if we parse invalid folder name, - // so we need to catch the error and continue seaching other - // path. - continue; - } - } - } - if (!_file) { - throw new Error('it does not support ' + command + ' command'); - } - }; - - this.run = function (args, callback) { - var process = Cc["@mozilla.org/process/util;1"] - .createInstance(Ci.nsIProcess); - var output = null; - try { - process.init(_file); - process.run(true, args, args.length); - log(command + ' ' + args.join(' ')); - } catch (e) { - throw new Error('having trouble when execute ' + command + - ' ' + args.join(' ')); - } - return output; - }; -}; - -// Get PATH of the environment -function getEnvPath() { - var os = getOsType(); - if (!os) { - throw new Error('cannot not read system type'); - } - var env = Cc["@mozilla.org/process/environment;1"]. - getService(Ci.nsIEnvironment); - var p = env.get('PATH'); - var isMsys = env.get('OSTYPE') ? true : false; - if (os.indexOf('WIN')!== -1 && !isMsys) { - paths = p.split(';'); - } else { - paths = p.split(':'); - } - return paths; -} - -// We parse list like ps aux and b2g-ps into object -function psParser(out) { - var rows = out.split('\n'); - if (rows.length < 2) - return {}; - - var result = {}; - var colNames = getCols(rows[0]); - for (var r = 1; r < rows.length; r++) { - var cols = getCols(rows[r]); - result[cols[0]] = {}; - for (var c = 1; c < cols.length; c++) { - result[cols[0]][colNames[c]] = cols[c]; - } - } - return result; -} - -function getCols(row) { - return row.trim().split(/\s+/).map(function(name) { - return normalizeString(name); - }); -} - exports.isSubjectToBranding = isSubjectToBranding; exports.ls = ls; exports.getFileContent = getFileContent; @@ -712,18 +602,14 @@ exports.getGaia = getGaia; exports.getBuildConfig = getBuildConfig; exports.getAppsByList = getAppsByList; exports.getApp = getApp; -exports.getAppName = getAppName; exports.getXML = getXML; exports.getTempFolder = getTempFolder; exports.normalizeString = normalizeString; -exports.Commander = Commander; -exports.getEnvPath = getEnvPath; // ===== the following functions support node.js compitable interface. exports.FILE_TYPE_FILE = FILE_TYPE_FILE; exports.FILE_TYPE_DIRECTORY = FILE_TYPE_DIRECTORY; exports.deleteFile = deleteFile; exports.listFiles = listFiles; -exports.psParser = psParser; exports.fileExists = fileExists; exports.mkdirs = mkdirs; exports.joinPath = joinPath;