diff --git a/CHANGES b/CHANGES deleted file mode 100644 index 3b73c738..00000000 --- a/CHANGES +++ /dev/null @@ -1,318 +0,0 @@ -=== 0.4 === -2011-07-24 Aki Sasaki -* scripts/mozmill_updates.py (MozmillUpdate.run_mozmill): - Added more substrs to MozmillErrorList. - -2011-07-21 Aki Sasaki -* scripts/mozmill_updates.py (MozmillUpdate.run_mozmill): - Created report_json in work_dir, then copied to upload_dir at the end. - -* scripts/mozmill_updates.py (MozmillUpdate._clobber): - No longer mkdir_p work_dir / upload_dir. - -* mozharness/base/python.py (create_virtualenv): - Now using abspath of virtualenv_path. - -* mozharness/base/script.py (BaseScript.run): - Copied log files to upload_dir at end of run(). - -* mozharness/base/script.py (BaseScript.copy_to_upload_dir): - Added max_backups, short_desc, long_desc. - -* test/test_base_script.py (TestHelperFunctions._create_temp_file): - Allowed for custom contents - -2011-07-20 Aki Sasaki -* mozharness/base/script.py (OSMixin.rmtree): - return 0 on success, -1 on unable to remove path - -* mozharness/base/script.py (BaseScript.copy_to_upload_dir): - Quick'n'dirty. Need to test, write tests. This has been on my todo list for a long time; glad to have it. - -2011-07-19 Aki Sasaki -* scripts/mozmill_updates.py (MozmillUpdate): - Use VirtualenvMixin and virtualenv_config_options from new mozharness/base/python.py file. - Specify virtualenv_modules in __init__; if this diverges between versions we can move to config files. - -* mozharness/base/python.py: Created file. - Tear out the virtualenv creation/usage generic functions from scripts/mozmill_updates.py and into this generic library. - -* TODO: Removed file - -* mozharness/base/script.py (query_abs_dirs): - put upload_dir in work_dir/ - keep logs under base_work_dir/ - TODO: copy_to_upload_dir with file rotation, copy logs there - This will let me clobber work_dir without blowing away logs - -2011-07-18 Aki Sasaki -* mozharness/scripts/mozmill_updates.py: - Added --create-virtualenv action. Not sure if this is useful or needed, but I was asked why I didn't, so here it is. - Added error checking + report.json parsing. - -* mozharness/base/script.py (run): - Exit self.return_code. Initialize in __init__. - -* mozharness/base/script.py (new_log_obj): - Change default log_name from 'test' to 'log' - -2011-07-15 Aki Sasaki -* mozharness/scripts/mozmill_updates.py: - Added file. - -* mozharness/base/config.py (_create_config_parser): - Set default work_dir to 'build' instead of 'work_dir' - -* mozharness/base/script.py (query_abs_dirs): - Set default upload_dir to 'upload' instead of 'upload_dir' - -2011-07-12 Aki Sasaki -* mozharness/base/config.py (parse_config_file): - Explicitly close fh when using .json config files - -* mozharness/base/vcs/mercurial.py (MercurialMixin): removed - -* mozharness/base/vcs/mercurial.py (MercurialScript): removed - -* mozharness/base/vcs/vcsbase.py (VCSMixin): - Created to use MercurialVCS (or any other VCS object). - -* mozharness/base/vcs/vcsbase.py (VCSScript): - Quick'n'dirty wrapper around VCSScript + BaseScript. - -* mozharness/base/vcs/vcsbase.py (MercurialScript): - Quick'n'dirty wrapper around VCSScript, with self.default_vcs = 'hg' - -* mozharness/l10n/multi_locale_build.py (MultiLocaleBuild): - s,scm_checkout_repos,vcs_checkout_repos. - Pointed to the new vcsbase.MercurialScript - -* scripts/signdebs.py (MaemoDebSigner.create_repos): - s,scm_checkout_repos,vcs_checkout_repos. - Pointed to the new vcsbase.MercurialScript - -* configs/multi_locale/*: - s,dir_name,dest,g - -* configs/deb_repos/*: - s,dir_name,dest,g - -* test/networked/test_base_vcs_vcsbase.py renamed from test_base_vcs_mercurial.py - Run a vcs_checkout. Clobber no longer applies, though I may want to re-add. - -2011-05-03 Aki Sasaki - -* mozharness/base/vcs/mercurial.py (MercurialVCS.push): - If hg_ver < 1.6.0, don't use --new-branch. (for windows, since mozilla-build appears to have hg 1.5.4. we need to decide whether we want some sort of exception or fatal here? i'll let script writers decide that.) - -* mozharness/base/script.py (ShellMixin.get_output_from_command): - Explicitly closed tmp_stdout, tmp_stderr, fh filehandles for windows. - -* test/test_base_vcs_mercurial.py (*): - Altered to pass some tests on windows. - TestMakeAbsolute.* disabled on windows. - init_hgrepo.sh called via os.path.join and explicitly calling bash due to windows. - -* test/test_base_script.py (*): - Altered to pass tests on windows. - test_chmod no longer running on windows. - test_chdir uses os.path.join() now. - -* unit.sh: - Added pylint as a dependency in the header. - Ignoring perma-warnings from pylint. - Platform detection to try to work on windows, mac, linux - -2011-05-02 Aki Sasaki - -* mozharness/base/vcs/mercurial.py (MercurialVCS.*): - Added self.info() calls at the beginnings of methods to announce what we're planning to do. - -* mozharness/base/vcs/script.py (ShellMixin.get_output_from_command): - Allow for throw_exception=True. - Clean up *before* self.fatal() or raise - -* mozharness/base/vcs/script.py (ShellMixin.run_command): - Allow for throw_exception=True - -* mozharness/base/vcs/mercurial.py (MercurialVCS._ensure_shared_repo_and_revision): - self.info() if nuking dest due to lack of DEST/.hg/sharedpath - Don't nuke shared_path if not needed. - -* mozharness/base/vcs/mercurial.py (MercurialVCS.ensure_repo_and_revision): - Try to pull dest if it exists, rather than nuke/clone. Not sure if this is the desired behavior, but it's the behavior specified in the unit tests. - -* mozharness/base/vcs/mercurial.py (MercurialVCS.pull): - self.error() if dest doesn't exist - -* mozharness/base/vcs/mercurial.py (MercurialVCS.clone): - self.info() that we're about to rmtree(dest) - -* mozharness/base/log.py (LogMixin.dump_exception): - Allow for message=None - -* mozharness/base/log.py (LogMixin.log): - Changed to use _log_level_at_least and _print, so we can hide wanted errors during unittest runs. - -* mozharness/base/log.py (LogMixin._log_level_at_least): Added method. - -* mozharness/base/log.py (LogMixin._print): Added method. - -* mozharness/base/log.py (LogMixin.debug): - No longer checking to see if log_level is debug, as that check happens in log() now. - -* test/test_base_vcs_mercurial: - Ported the rest of the tests from build/tools. - -2011-04-28 Aki Sasaki - -* test/helper_files/init_hgrepo.sh: Added file (copied from build/tools) - -* mozharness/base/script.py (ShellMixin.query_env): - Fixed partial_env logic. - -* test/test_base_script.py: - Added test_chmod method. - Renamed TestSummary to TestScriptLogging. - Added query_env testing. - -* mozharness/base/script.py (OSMixin.chown): Removed method. - I can't see using it as a non-root user; as a root user we'll need to sudo. - -* mozharness/base/script.py (OSMixin.chmod): - Fixed logging. - -* scripts/configtest.py: - Ignore test_malformed* config files. - -* configs/test/test_malformed.json: Added file. - -* configs/test/test_malformed.py: Added file. - -* mozharness/base/config.py (parse_config_file): - Fixed python config files. - Raise assertion if config file does not end in .json or .py. - Raise assertion if config file is not found. - -* test/test_base_config.py: - TestJsonConfig -> TestParseConfigFile. - Added python config file tests, illegal/malformed config file checking. - -* configs/test/test.py: Added file. - -* mozharness/l10n/locales.py (LocalesMixin.query_locales): - Fixed some borked logic. - -* mozharness/l10n/locales.py (LocalesMixin.parse_locales_file): - Removed duplicate ignore_locales logic that should be handled in query_locales. - -* test/helper_files/locales.json: Added file. - -* test/helper_files/locales.txt: Added file. - -* test/test_l10n_locales.py: Added file. - -* test/*: Renamed to match directory structure. - -2011-04-27 Aki Sasaki - -* test/networked/test_script.py: Added file. - -* test/test_script.py (TestHelperFunctions.test_download_file): Removed method. - -* unit.sh: - Added a separate networked/non-networked call to nosetests. In the future, we should add an option to skip the network tests. - -* mozharness/base/vcs/hgtool.py: Created softlink -> sourcetool.py. - -* mozharness/base/vcs/sourcetool.py: - Was able to test a successful |sourcetool.py repo|, as well as a |sourcetool.py -s shared_dir repo|. - Created ability to change the default VCS by looking at the filename (softlink to hgtool.py and it'll use hg by default; softlink to gittool.py and it'll use git when we enable it. I also see potential svntool.py and cvstool.py at some point.) - -* mozharness/base/vcs/sourcetool.py (SourceTool._pre_config_lock): - Verified that self.config['vcs'] is set; else self.fatal() - -* mozharness/base/vcs/sourcetool.py (SourceTool.source): - Cleaned up old TODO comments - -* mozharness/base/vcs/mercurial.py: - Worked through the various scenarios in sourcetool.py; fixing bugs. - -2011-04-26 Aki Sasaki - -* scripts/sourcetool.py: Moved from scripts/hgtool.py. - -* test/test_log.py (TestLog.test_console_log): Removed method. - Coverage didn't change at all (console logging is tested in test_script.py) but we removed a spurious line of output from unit.sh. - -* mozharness/base/log.py (LogMixin.warn): Removed method. - Also removed references to it. - -* mozharness/base/script.py (ShellMixin.get_output_from_command): - fixed the NamedTemporaryFile issue between 2.5 and 2.6, by allowing you to name your own tmpfile if you want to munge the contents later. - -* mozharness/base/script.py (BaseScript): - Pulled all os and shell methods out to mixins. - -* mozharness/base/vcs/mercurial.py (MercurialVCS): Added class. - This will hopefully be a standalone object that takes the baseScript log_obj and perhaps config, but will allow for multiple VCS objects per BaseScript. - First pass: changed standalone functions to a MercurialVCS object. - -2011-04-25 Aki Sasaki - -* mozharness/base/log.py (BaseLogger.log_message): Renamed from log() to be compatible with LogMixin. - -* mozharness/base/log.py (LogMixin): New class. - Pull the generic logging methods from BaseScript to this new class. - This is specifically to allow for non-BaseScript objects to use the log_obj from BaseScript (set a LogMixin's self.log_obj to the logging object) - -* mozharness/base/script.py (BaseScript): - Pull the generic logging methods from BaseScript to a new LogMixin class. - Change BaseScript to inherit LogMixin by default. - -* mozharness/base/config.py (BaseConfig.parse_args): - Fixed the args list sent to parse_args (sys.argv[1:]) - Set self.options and self.args after parse_args for later access in BaseScript._pre_config_lock - -* mozharness/base/config.py (ExtendedOptionParser): - Removed self.variables hack and add_option override. - -* mozharness/base/script.py (__init__): - Added a _pre_config_lock() call before _config_lock() to allow for things like hgtool reading the buildprops json into self.config. - Moved --simple-log and --multi-log options to mozharness.base.config._create_config_parser. - -* mozharness/base/script.py (_config_lock): - Renamed to match _pre_config_lock (was _lock_config). - -* mozharness/base/config.py (_create_config_parser): - Allow for --ACTION to do the same thing as --only-ACTION. - Added action and logging OptionGroups. - Moved --simple-log and --multi-log options here. - Disable noop option argument. - Removed dependency on config_parser.variables (config_parser.defaults.keys() gives the same info). - - The noop option disable forced a change of s,self.config['noop'],self.config.get('noop'), throughout mozharness. - -* scripts/hgtool.py: New file. - -* CHANGES: New file. - -* TODO: New file. - -* mozharness/__init__.py: Bumped version to 0.4. - -2011-04-21 Aki Sasaki - -* mozharness/base/script.py (MercurialMixin, MercurialScript): - Moved to mozharness/base/vcs/mercurial.py. - Adjusted other files accordingly. - -* mozharness/base/script.py (BaseScript.dump_exception): New function. - -* mozharness/base/script.py (BaseScript.copyfile): - Adjusted to use dump_exception and a tighter try/except. - -* mozharness/base/config.py, script.py, scripts/configtest.py, test/*: - Switched simplejson/json import order, as simplejson is faster. - -=== 0.3 and before are non-changelogged. === diff --git a/configs/signing/android_mozilla-beta.py b/configs/signing/android_mozilla-beta.py index 4bbc2c4b..bb60b557 100644 --- a/configs/signing/android_mozilla-beta.py +++ b/configs/signing/android_mozilla-beta.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -LOCALES = ["en-US", "multi"] +ADDITIONAL_LOCALES = ["en-US", "multi"] TAG = "default" #AUS_SERVER = "dev-stage01.build.mozilla.org" AUS_SERVER = "aus2-staging.mozilla.org" @@ -12,9 +12,7 @@ #DOWNLOAD_BASE_URL = "http://%s%s" % (FTP_SERVER, FTP_UPLOAD_BASE_DIR) DOWNLOAD_BASE_URL = "http://ftp.mozilla.org/pub/mozilla.org/mobile/candidates/%(version)s-candidates/build%(buildnum)d" APK_BASE_NAME = "fennec-%(version)s.%(locale)s.android-arm.apk" -# because sign_android-0.8.sh renamed these wrong :( -BUILDID_BASE_URL = DOWNLOAD_BASE_URL + "/%(platform)_info.txt" -OLD_STYLE_BUILDID_BASE_URL = DOWNLOAD_BASE_URL + "/linux-android_info.txt" +BUILDID_BASE_URL = DOWNLOAD_BASE_URL + "/%(platform)s_info.txt" FFXBLD_SSH_KEY = '~/.ssh/ffxbld_dsa' #FFXBLD_SSH_KEY = '~/staging_ssh/ffxbld_dsa' CLTBLD_SSH_KEY = '~/.ssh/cltbld_dsa' @@ -34,17 +32,16 @@ """ KEYSTORE = "/home/cltsign/.android/android-release.keystore" -WORK_DIR = "build" JAVA_HOME = "/tools/jdk-1.6.0_17" JARSIGNER = "%s/bin/jarsigner" % JAVA_HOME KEY_ALIAS = "release" config = { - "log_name": "sign_android", - "work_dir": WORK_DIR, + "log_name": "sign_android_beta", + "work_dir": "beta", - "locales": LOCALES, + "additional_locales": ADDITIONAL_LOCALES, "locales_file": "buildbot-configs/mozilla/l10n-changesets_mobile-beta.json", "release_config_file": "buildbot-configs/mozilla/release-fennec-mozilla-beta.py", @@ -80,12 +77,21 @@ "aus_upload_base_dir": AUS_UPLOAD_BASE_DIR, "apk_base_name": APK_BASE_NAME, - "unsigned_apk_base_name": 'gecko-unsigned-unaligned.apk', + "unsigned_apk_base_name": APK_BASE_NAME, "download_base_url": DOWNLOAD_BASE_URL, "download_unsigned_base_subdir": "unsigned/%(platform)s/%(locale)s", "download_signed_base_subdir": "%(platform)s/%(locale)s", "buildid_base_url": BUILDID_BASE_URL, - "old_buildid_base_url": OLD_STYLE_BUILDID_BASE_URL, + "old_buildid_base_url": BUILDID_BASE_URL, + "actions": [ + "passphrase", + "clobber", + "pull", + "download-unsigned-bits", + "sign", + "verify-signatures", + "upload-signed-bits", + ], "keystore": KEYSTORE, "key_alias": KEY_ALIAS, diff --git a/configs/signing/android_mozilla-release.py b/configs/signing/android_mozilla-release.py index 9620e6c6..659959d9 100644 --- a/configs/signing/android_mozilla-release.py +++ b/configs/signing/android_mozilla-release.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -LOCALES = ["en-US", "multi"] +ADDITIONAL_LOCALES = ["en-US", "multi"] TAG = "default" #AUS_SERVER = "dev-stage01.build.mozilla.org" AUS_SERVER = "aus2-staging.mozilla.org" @@ -12,9 +12,7 @@ #DOWNLOAD_BASE_URL = "http://%s%s" % (FTP_SERVER, FTP_UPLOAD_BASE_DIR) DOWNLOAD_BASE_URL = "http://ftp.mozilla.org/pub/mozilla.org/mobile/candidates/%(version)s-candidates/build%(buildnum)d" APK_BASE_NAME = "fennec-%(version)s.%(locale)s.android-arm.apk" -# because sign_android-0.8.sh renamed these wrong :( -BUILDID_BASE_URL = DOWNLOAD_BASE_URL + "/%(platform)_info.txt" -OLD_STYLE_BUILDID_BASE_URL = DOWNLOAD_BASE_URL + "/linux-android_info.txt" +BUILDID_BASE_URL = DOWNLOAD_BASE_URL + "/%(platform)s_info.txt" FFXBLD_SSH_KEY = '~/.ssh/ffxbld_dsa' #FFXBLD_SSH_KEY = '~/staging_ssh/ffxbld_dsa' CLTBLD_SSH_KEY = '~/.ssh/cltbld_dsa' @@ -34,17 +32,16 @@ """ KEYSTORE = "/home/cltsign/.android/android-release.keystore" -WORK_DIR = "build" JAVA_HOME = "/tools/jdk-1.6.0_17" JARSIGNER = "%s/bin/jarsigner" % JAVA_HOME KEY_ALIAS = "release" config = { - "log_name": "sign_android", - "work_dir": WORK_DIR, + "log_name": "sign_android_release", + "work_dir": "release", - "locales": LOCALES, + "additional_locales": ADDITIONAL_LOCALES, "locales_file": "buildbot-configs/mozilla/l10n-changesets_mobile-release.json", "release_config_file": "buildbot-configs/mozilla/release-fennec-mozilla-release.py", @@ -80,12 +77,21 @@ "aus_upload_base_dir": AUS_UPLOAD_BASE_DIR, "apk_base_name": APK_BASE_NAME, - "unsigned_apk_base_name": 'gecko-unsigned-unaligned.apk', + "unsigned_apk_base_name": APK_BASE_NAME, "download_base_url": DOWNLOAD_BASE_URL, "download_unsigned_base_subdir": "unsigned/%(platform)s/%(locale)s", "download_signed_base_subdir": "%(platform)s/%(locale)s", "buildid_base_url": BUILDID_BASE_URL, - "old_buildid_base_url": OLD_STYLE_BUILDID_BASE_URL, + "old_buildid_base_url": BUILDID_BASE_URL, + "actions": [ + "passphrase", + "clobber", + "pull", + "download-unsigned-bits", + "sign", + "verify-signatures", + "upload-signed-bits", + ], "keystore": KEYSTORE, "key_alias": KEY_ALIAS, diff --git a/configs/signing/staging_android_mozilla-beta.py b/configs/signing/staging_android_mozilla-beta.py index 36676e6d..3cc6f1a1 100644 --- a/configs/signing/staging_android_mozilla-beta.py +++ b/configs/signing/staging_android_mozilla-beta.py @@ -2,19 +2,17 @@ import os -LOCALES = ["en-US", "multi"] +ADDITIONAL_LOCALES = ["en-US", "multi"] TAG = "default" AUS_SERVER = "dev-stage01.build.mozilla.org" FTP_SERVER = "dev-stage01.build.mozilla.org" AUS_UPLOAD_BASE_DIR = "/opt/aus2/snippets/staging" AUS_DIR_BASE_NAME = "Fennec-%(version)s-build%(buildnum)d" FTP_UPLOAD_BASE_DIR = "/pub/mozilla.org/mobile/candidates/%(version)s-candidates/build%(buildnum)d" -#DOWNLOAD_BASE_URL = "http://%s%s" % (FTP_SERVER, FTP_UPLOAD_BASE_DIR) -DOWNLOAD_BASE_URL = "http://ftp.mozilla.org/pub/mozilla.org/mobile/candidates/%(version)s-candidates/build%(buildnum)d" +DOWNLOAD_BASE_URL = "http://%s%s" % (FTP_SERVER, FTP_UPLOAD_BASE_DIR) +#DOWNLOAD_BASE_URL = "http://ftp.mozilla.org/pub/mozilla.org/mobile/candidates/%(version)s-candidates/build%(buildnum)d" APK_BASE_NAME = "fennec-%(version)s.%(locale)s.android-arm.apk" -# because sign_android-0.8.sh renamed these wrong :( -BUILDID_BASE_URL = DOWNLOAD_BASE_URL + "/%(platform)_info.txt" -OLD_STYLE_BUILDID_BASE_URL = DOWNLOAD_BASE_URL + "/linux-android_info.txt" +BUILDID_BASE_URL = DOWNLOAD_BASE_URL + "/%(platform)s_info.txt" FFXBLD_SSH_KEY = '~/.ssh/ffxbld_dsa' CLTBLD_SSH_KEY = '~/.ssh/id_rsa' @@ -32,20 +30,16 @@ """ KEYSTORE = "%s/.android/android.keystore" % os.environ['HOME'] -#BASE_WORK_DIR = "%s/signing-work/fennec-beta" % (os.environ['HOME']) -BASE_WORK_DIR = "%s/signing-work/fennec-beta" % os.getcwd() -WORK_DIR = "build" JAVA_HOME = "/tools/jdk6" JARSIGNER = "%s/bin/jarsigner" % JAVA_HOME KEY_ALIAS = "nightly" config = { - "log_name": "sign_android", - "base_work_dir": BASE_WORK_DIR, - "work_dir": WORK_DIR, + "log_name": "sign_android_staging_beta", + "work_dir": "staging_beta", - "locales": LOCALES, + "additional_locales": ADDITIONAL_LOCALES, "locales_file": "buildbot-configs/mozilla/l10n-changesets_mobile-beta.json", "release_config_file": "buildbot-configs/mozilla/staging_release-fennec-mozilla-beta.py", @@ -81,12 +75,12 @@ "aus_upload_base_dir": AUS_UPLOAD_BASE_DIR, "apk_base_name": APK_BASE_NAME, - "unsigned_apk_base_name": 'gecko-unsigned-unaligned.apk', + "unsigned_apk_base_name": APK_BASE_NAME, "download_base_url": DOWNLOAD_BASE_URL, "download_unsigned_base_subdir": "unsigned/%(platform)s/%(locale)s", "download_signed_base_subdir": "%(platform)s/%(locale)s", "buildid_base_url": BUILDID_BASE_URL, - "old_buildid_base_url": OLD_STYLE_BUILDID_BASE_URL, + "old_buildid_base_url": BUILDID_BASE_URL, "keystore": KEYSTORE, "key_alias": KEY_ALIAS, diff --git a/mozharness/base/config.py b/mozharness/base/config.py index 13b309a6..a5f70b68 100755 --- a/mozharness/base/config.py +++ b/mozharness/base/config.py @@ -177,7 +177,7 @@ class BaseConfig(object): """ def __init__(self, config=None, initial_config_file=None, config_options=None, all_actions=None, default_actions=None, - volatile_config_vars=None, + volatile_config=None, require_config_file=False, usage="usage: %prog [options]"): self._config = {} self.actions = [] @@ -192,10 +192,14 @@ def __init__(self, config=None, initial_config_file=None, config_options=None, self.default_actions = default_actions[:] else: self.default_actions = self.all_actions[:] - if volatile_config_vars is None: - self.volatile_config_vars = [] + if volatile_config is None: + self.volatile_config = { + 'actions': None, + 'add_actions': None, + 'no_actions': None, + } else: - self.volatile_config_vars = volatile_config_vars[:] + self.volatile_config = deepcopy(volatile_config) if config: self.set_config(config) @@ -271,7 +275,7 @@ def _create_config_parser(self, config_options, usage): ) action_option_group.add_option( "--action", action="extend", - dest="only_actions", metavar="ACTIONS", + dest="actions", metavar="ACTIONS", help="Do action %s" % self.all_actions ) action_option_group.add_option( @@ -287,7 +291,7 @@ def _create_config_parser(self, config_options, usage): for action in self.all_actions: action_option_group.add_option( "--only-%s" % action, "--%s" % action, action="append_const", - dest="only_actions", const=action, + dest="actions", const=action, help="Add %s to the limited list of actions" % action ) action_option_group.add_option( @@ -296,9 +300,6 @@ def _create_config_parser(self, config_options, usage): help="Remove %s from the list of actions to perform" % action ) self.config_parser.add_option_group(action_option_group) - self.volatile_config_vars.extend(['only_actions', 'add_actions', - 'no_actions', 'list_actions', - 'noop']) # Child-specified options # TODO error checking for overlapping options if config_options: @@ -351,7 +352,7 @@ def parse_args(self, args=None): if not options.config_file: if self.require_config_file: - print("Required config file not set!") + print("Required config file not set! (use --config-file option)") raise SystemExit(-1) else: self.set_config(parse_config_file(options.config_file)) @@ -364,32 +365,51 @@ def parse_args(self, args=None): continue self._config[key] = value + # The idea behind the volatile_config is we don't want to save this + # info over multiple runs. This defaults to the action-specific + # config options, but can be anything. + for key in self.volatile_config.keys(): + if self._config.get(key) is not None: + self.volatile_config[key] = self._config[key] + del(self._config[key]) + """Actions. Seems a little complex, but the logic goes: - If we specify --BLAH or --only-BLAH once or multiple times, - we want to override the default_actions list with the ones we list. + First, if default_actions is specified in the config, set our + default actions even if the script specifies other default actions. + + Without any other action-specific options, run with default actions. - Otherwise, if we specify --add-action, we want to add an action to - the default list. + If we specify --ACTION or --only-ACTION once or multiple times, + we want to override the default_actions list with the one(s) we list. - Finally, if we specify --no-BLAH, remove that from the list of + Otherwise, if we specify --add-action ACTION, we want to add an + action to the list. + + Finally, if we specify --no-ACTION, remove that from the list of actions to perform. """ + if self._config.get('default_actions'): + default_actions = self.verify_actions(self._config['default_actions']) + self.default_actions = default_actions self.actions = self.default_actions[:] - if options.only_actions: - actions = self.verify_actions(options.only_actions) + if self.volatile_config['actions']: + actions = self.verify_actions(self.volatile_config['actions']) self.actions = actions - elif options.add_actions: - actions = self.verify_actions(options.add_actions) + elif self.volatile_config['add_actions']: + actions = self.verify_actions(self.volatile_config['add_actions']) self.actions.extend(actions) - if options.no_actions: - actions = self.verify_actions(options.no_actions) + if self.volatile_config['no_actions']: + actions = self.verify_actions(self.volatile_config['no_actions']) for action in actions: if action in self.actions: self.actions.remove(action) + # Keep? This is for saving the volatile config in the dump_config + self._config['volatile_config'] = self.volatile_config + self.options = options self.args = args return (self.options, self.args) diff --git a/mozharness/l10n/locales.py b/mozharness/l10n/locales.py index 6f6f2e6c..334043eb 100755 --- a/mozharness/l10n/locales.py +++ b/mozharness/l10n/locales.py @@ -62,7 +62,8 @@ def query_locales(self): return self.locales c = self.config locales = c.get("locales", None) - ignore_locales = c.get("ignore_locales", None) + ignore_locales = c.get("ignore_locales", []) + additional_locales = c.get("additional_locales", []) if locales is None: locales = [] @@ -73,11 +74,14 @@ def query_locales(self): locales = self.parse_locales_file(locales_file) else: self.fatal("No way to determine locales!") - elif ignore_locales: - for locale in ignore_locales: - if locale in locales: - self.debug("Ignoring locale %s." % locale) - locales.remove(locale) + for locale in ignore_locales: + if locale in locales: + self.debug("Ignoring locale %s." % locale) + locales.remove(locale) + for locale in additional_locales: + if locale not in locales: + self.debug("Adding locale %s." % locale) + locales.append(locale) if locales is not None: self.locales = locales diff --git a/scripts/sign_android.py b/scripts/sign_android.py index 35f78117..9a560b2f 100755 --- a/scripts/sign_android.py +++ b/scripts/sign_android.py @@ -196,6 +196,7 @@ def __init__(self, require_config_file=True): self.store_passphrase = os.environ.get('android_storepass') self.key_passphrase = os.environ.get('android_keypass') self.release_config = {} + self.failures = [] LocalesMixin.__init__(self) MercurialScript.__init__(self, config_options=self.config_options, @@ -207,8 +208,8 @@ def __init__(self, require_config_file=True): "sign", "verify-signatures", "upload-signed-bits", -# "create-snippets", -# "upload-snippets", + "create-snippets", + "upload-snippets", ], require_config_file=require_config_file ) @@ -293,20 +294,32 @@ def query_buildid(self, platform, base_url, buildnum=None, version=None): self.warning("Can't get buildID from %s (try %d)" % (url, count)) self.critical("Can't get buildID from %s!" % url) - def _sign(self, apk, error_list=None): + def _sign(self, apk, remove_signature=True, error_list=None): c = self.config jarsigner = self.query_exe("jarsigner") + zip_bin = self.query_exe("zip") + if remove_signature: + # Get rid of previous signature. + # TODO error checking, but allow for no META-INF/ in the zipfile. + self.run_command([zip_bin, apk, '-d', 'META-INF/*']) if error_list is None: error_list = JARSIGNER_ERROR_LIST # This needs to run silently, so no run_command() or # get_output_from_command() (though I could add a # suppress_command_echo=True or something?) - p = subprocess.Popen([jarsigner, "-keystore", c['keystore'], - "-storepass", self.store_passphrase, - "-keypass", self.key_passphrase, - apk, c['key_alias']], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + try: + p = subprocess.Popen([jarsigner, "-keystore", c['keystore'], + "-storepass", self.store_passphrase, + "-keypass", self.key_passphrase, + apk, c['key_alias']], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + except OSError: + self.dump_exception("Error while signing %s (missing %s?):" % (apk, jarsigner)) + return -1 + except ValueError: + self.dump_exception("Popen called with invalid arguments during signing?") + return -2 parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=error_list) loop = True @@ -318,6 +331,20 @@ def _sign(self, apk, error_list=None): parser.add_lines(line) return parser.num_errors + def add_failure(self, platform, locale, + message="%(platform)s:%(locale)s failed.", + level=ERROR): + s = "%s:%s" % (platform, locale) + if s not in self.failures: + self.failures.append(s) + self.return_code += 1 + self.add_summary(message % {'platform': platform, 'locale': locale}, + level=level) + + def query_failure(self, platform, locale): + s = "%s:%s" % (platform, locale) + return s in self.failures + # Actions {{{2 def passphrase(self): if not self.store_passphrase: @@ -327,9 +354,12 @@ def passphrase(self): def verify_passphrases(self): self.info("Verifying passphrases...") - status = self._sign("NOTAREALAPK", error_list=TEST_JARSIGNER_ERROR_LIST) + status = self._sign("NOTAREALAPK", remove_signature=False, + error_list=TEST_JARSIGNER_ERROR_LIST) if status == 0: self.info("Passphrases are good.") + elif status < 0: + self.fatal("Encountered errors while trying to sign!") else: self.fatal("Unable to verify passphrases!") @@ -371,14 +401,14 @@ def download_unsigned_bits(self): for locale in locales: replace_dict['locale'] = locale url = base_url % replace_dict - parent_dir = '%s/%s/%s' % (dirs['abs_work_dir'], + parent_dir = '%s/unsigned/%s/%s' % (dirs['abs_work_dir'], platform, locale) - file_path = '%s/gecko_unsigned_unaligned.apk' % parent_dir + file_path = '%s/gecko.ap_' % parent_dir self.mkdir_p(parent_dir) total_count += 1 if not self.download_file(url, file_path): - self.add_summary("Unable to download %s:%s unsigned apk!" % (platform, locale), - level=ERROR) + self.add_failure(platform, locale, + message="Unable to download %(platform)s:%(locale)s unsigned apk!") else: successful_count += 1 level = INFO @@ -402,27 +432,32 @@ def sign(self): zipalign = self.query_exe("zipalign") for platform in c['platforms']: for locale in locales: - parent_dir = '%s/%s/%s' % (dirs['abs_work_dir'], - platform, locale) - unsigned_unaligned_path = '%s/gecko_unsigned_unaligned.apk' % parent_dir - unaligned_path = '%s/gecko_unaligned.apk' % parent_dir - signed_path = '%s/%s' % (parent_dir, - c['apk_base_name'] % {'version': rc['version'], - 'locale': locale}) - self.mkdir_p(parent_dir) + if self.query_failure(platform, locale): + self.warning("%s:%s had previous issues; skipping!" % (platform, locale)) + continue + unsigned_path = '%s/unsigned/%s/%s/gecko.ap_' % (dirs['abs_work_dir'], platform, locale) + signed_dir = '%s/%s/%s' % (dirs['abs_work_dir'], platform, locale) + signed_file_name = c['apk_base_name'] % {'version': rc['version'], + 'locale': locale} + signed_path = "%s/%s" % (signed_dir, signed_file_name) total_count += 1 self.info("Signing %s %s." % (platform, locale)) - self.copyfile(unsigned_unaligned_path, unaligned_path) - if self._sign(unaligned_path) != 0: + if not os.path.exists(unsigned_path): + self.error("Missing apk %s!" % unsigned_path) + continue + if self._sign(unsigned_path) != 0: self.add_summary("Unable to sign %s:%s apk!", level=FATAL) - elif self.run_command([zipalign, '-f', '4', - unaligned_path, signed_path], - error_list=BaseErrorList): - self.add_summary("Unable to align %s:%s apk!", - level=FATAL) else: - successful_count += 1 + self.mkdir_p(signed_dir) + if self.run_command([zipalign, '-f', '4', + unsigned_path, signed_path], + error_list=BaseErrorList): + self.add_failure(platform, locale, + message="Unable to align %(platform)s:%(locale)s apk!") + self.rmtree(signed_dir) + else: + successful_count += 1 level = INFO if successful_count < total_count: level = ERROR @@ -452,16 +487,29 @@ def verify_signatures(self): env = self.query_env(partial_env=c.get("env")) for platform in c['platforms']: for locale in locales: + if self.query_failure(platform, locale): + self.warning("%s:%s had previous issues; skipping!" % (platform, locale)) + continue signed_path = '%s/%s/%s' % (platform, locale, c['apk_base_name'] % {'version': rc['version'], 'locale': locale}) - self.run_command([c['signature_verification_script'], - '--tools-dir=tools/', - '--%s' % c['key_alias'], - '--apk=%s' % signed_path], - cwd=dirs['abs_work_dir'], - env=env, - error_list=verification_error_list) + if not os.path.exists(os.path.join(dirs['abs_work_dir'], + signed_path)): + self.add_failure(platform, locale, + message="Can't verify nonexistent %(platform)s:%(locale)s apk!") + continue + status = self.run_command( + [c['signature_verification_script'], + '--tools-dir=tools/', + '--%s' % c['key_alias'], + '--apk=%s' % signed_path], + cwd=dirs['abs_work_dir'], + env=env, + error_list=verification_error_list + ) + if status: + self.add_failure(platform, locale, + message="Errors verifying %(platform)s:%(locale)s apk!") def upload_signed_bits(self): c = self.config @@ -510,6 +558,9 @@ def create_snippets(self): replace_dict['platform'] = platform replace_dict['buildid'] = buildid for locale in locales: + if self.query_failure(platform, locale): + self.warning("%s:%s had previous issues; skipping!" % (platform, locale)) + continue replace_dict['locale'] = locale parent_dir = '%s/%s/%s' % (dirs['abs_work_dir'], platform, locale)