bug 659544 - android partner repacks. r=rail

mozilla · Mar 27, 2012 · d9e20c0 · d9e20c0
1 parent f772505
commit d9e20c0
Show file tree

Hide file tree

Showing 17 changed files with 865 additions and 150 deletions.
diff --git a/configs/partner_repacks/release_mozilla-release_android.py b/configs/partner_repacks/release_mozilla-release_android.py
@@ -0,0 +1,45 @@
+FTP_SERVER = "stage.mozilla.org"
+FTP_USER = "ffxbld"
+FTP_SSH_KEY = "~/.ssh/ffxbld_dsa"
+FTP_UPLOAD_BASE_DIR = "/pub/mozilla.org/mobile/candidates/%(version)s-candidates/build%(buildnum)d"
+DOWNLOAD_BASE_URL = "http://%s%s" % (FTP_SERVER, FTP_UPLOAD_BASE_DIR)
+APK_BASE_NAME = "fennec-%(version)s.%(locale)s.android-arm.apk"
+HG_SHARE_BASE_DIR = "/builds/hg-shared"
+KEYSTORE = "/home/cltsign/.android/android-release.keystore"
+KEY_ALIAS = "release"
+
+config = {
+    "log_name": "partner_repack",
+    "locales_file": "buildbot-configs/mozilla/l10n-changesets_mobile-release.json",
+    "additional_locales": ['en-US'],
+    "platforms": ["android"],
+    "repos": [{
+        "repo": "http://hg.mozilla.org/build/buildbot-configs",
+        "revision": "default",
+    }],
+    'vcs_share_base': HG_SHARE_BASE_DIR,
+    "ftp_upload_base_dir": FTP_UPLOAD_BASE_DIR,
+    "ftp_ssh_key": FTP_SSH_KEY,
+    "ftp_user": FTP_USER,
+    "ftp_server": FTP_SERVER,
+    "installer_base_names": {
+        "android": APK_BASE_NAME,
+    },
+    "partner_config": {
+        "android-market": {},
+    },
+    "download_unsigned_base_subdir": "unsigned/%(platform)s/%(locale)s",
+    "download_base_url": DOWNLOAD_BASE_URL,
+
+    "release_config_file": "buildbot-configs/mozilla/release-fennec-mozilla-release.py",
+
+    "default_actions": ["clobber", "pull", "download", "repack", "upload-unsigned-bits"],
+
+    # signing (optional)
+    "keystore": KEYSTORE,
+    "key_alias": KEY_ALIAS,
+    "exes": {
+        "jarsigner": "/tools/jdk-1.6.0_17/bin/jarsigner",
+        "zipalign": "/tools/android-sdk-r8/tools/zipalign",
+    },
+}
diff --git a/configs/partner_repacks/staging_release_mozilla-release_android.py b/configs/partner_repacks/staging_release_mozilla-release_android.py
@@ -0,0 +1,50 @@
+FTP_SERVER = "dev-stage01.build.sjc1.mozilla.com"
+FTP_USER = "ffxbld"
+FTP_SSH_KEY = "~/.ssh/ffxbld_dsa"
+FTP_UPLOAD_BASE_DIR = "/pub/mozilla.org/mobile/candidates/%(version)s-candidates/build%(buildnum)d"
+#DOWNLOAD_BASE_URL = "http://%s%s" % (FTP_SERVER, FTP_UPLOAD_BASE_DIR)
+DOWNLOAD_BASE_URL = "http://ftp.mozilla.org/pub/mozilla.org/mobile/candidates/%(version)s-candidates/build%(buildnum)d"
+#DOWNLOAD_BASE_URL = "http://dev-stage01.build.mozilla.org/pub/mozilla.org/mobile/candidates/11.0b1-candidates/build1/"
+APK_BASE_NAME = "fennec-%(version)s.%(locale)s.android-arm.apk"
+#APK_BASE_NAME = "fennec-11.0b1.%(locale)s.android-arm.apk"
+HG_SHARE_BASE_DIR = "/builds/hg-shared"
+#KEYSTORE = "/home/cltsign/.android/android-release.keystore"
+KEYSTORE = "/home/cltbld/.android/android.keystore"
+#KEY_ALIAS = "release"
+KEY_ALIAS = "nightly"
+
+config = {
+    "log_name": "partner_repack",
+    "locales_file": "buildbot-configs/mozilla/l10n-changesets_mobile-release.json",
+    "additional_locales": ['en-US'],
+    "platforms": ["android"],
+    "repos": [{
+        "repo": "http://hg.mozilla.org/build/buildbot-configs",
+        "revision": "default",
+    }],
+    'vcs_share_base': HG_SHARE_BASE_DIR,
+    "ftp_upload_base_dir": FTP_UPLOAD_BASE_DIR,
+    "ftp_ssh_key": FTP_SSH_KEY,
+    "ftp_user": FTP_USER,
+    "ftp_server": FTP_SERVER,
+    "installer_base_names": {
+        "android": APK_BASE_NAME,
+    },
+    "partner_config": {
+        "android-market": {},
+    },
+    "download_unsigned_base_subdir": "unsigned/%(platform)s/%(locale)s",
+    "download_base_url": DOWNLOAD_BASE_URL,
+
+    "release_config_file": "buildbot-configs/mozilla/release-fennec-mozilla-release.py",
+
+    "default_actions": ["clobber", "pull", "download", "repack", "upload-unsigned-bits"],
+
+    # signing (optional)
+    "keystore": KEYSTORE,
+    "key_alias": KEY_ALIAS,
+    "exes": {
+        "jarsigner": "/tools/jdk-1.6.0_17/bin/jarsigner",
+        "zipalign": "/tools/android-sdk-r8/tools/zipalign",
+    },
+}
diff --git a/configs/signing/android_mozilla-release.py b/configs/signing/android_mozilla-release.py
@@ -50,6 +50,9 @@
     "update_platform_map": {
         'android': 'Android_arm-eabi-gcc3',
     },
+    "enable_partner_repacks": False,
+    "partner_platforms": ['android'],
+    "partners": ['android-market'],
     "update_channels": {
         'release': {
             'url': RELEASE_UPDATE_URL,

diff --git a/configs/signing/staging_android_mozilla-beta.py b/configs/signing/staging_android_mozilla-beta.py
@@ -48,6 +48,9 @@
     "update_platform_map": {
         'android': 'Android_arm-eabi-gcc3',
     },
+#    "enable_partner_repacks": True,
+#    "partner_platforms": ['android'],
+#    "partners": ['android-market'],
     "update_channels": {
         'release': {
             'url': RELEASE_UPDATE_URL,

diff --git a/mozharness/base/errors.py b/mozharness/base/errors.py
@@ -101,6 +101,28 @@ class VCSException(Exception):
     'explanation': r'''The apk is missing!''',
 }]
 
+ZipErrorList = BaseErrorList + [{
+    'substr': r'''zip warning:''',
+    'level': WARNING,
+},{
+    'substr': r'''zip error:''',
+    'level': ERROR,
+},{
+    'substr': r'''Cannot open file: it does not appear to be a valid archive''',
+    'level': ERROR,
+}]
+
+ZipalignErrorList = BaseErrorList + [{
+    'regex': re.compile(r'''Unable to open .* as a zip archive'''),
+    'level': ERROR,
+},{
+    'regex': re.compile(r'''Output file .* exists'''),
+    'level': ERROR,
+},{
+    'substr': r'''Input and output can't be the same file''',
+    'level': ERROR,
+}]
+
 
 
 # __main__ {{{1

diff --git a/mozharness/base/script.py b/mozharness/base/script.py
@@ -21,10 +21,8 @@
 import shutil
 import subprocess
 import sys
-import tarfile
 import urllib2
 import urlparse
-import zipfile
 
 try:
     import simplejson as json
@@ -41,15 +39,26 @@ class OSMixin(object):
 
     Depends on LogMixin, ShellMixin, and a self.config of some sort.
     """
-    def mkdir_p(self, path):
+    def mkdir_p(self, path, error_level=ERROR):
+        """
+        Returns None for success, not None for failure
+        """
         if not os.path.exists(path):
             self.info("mkdir: %s" % path)
             if not self.config.get('noop'):
-                os.makedirs(path)
+                try:
+                    os.makedirs(path)
+                except OSError:
+                    self.log("Can't create directory %s!" % path,
+                             level=error_level)
+                    return -1
         else:
             self.debug("mkdir_p: %s Already exists." % path)
 
     def rmtree(self, path, log_level=INFO, error_level=ERROR, exit_code=-1):
+        """
+        Returns None for success, not None for failure
+        """
         self.log("rmtree: %s" % path, level=log_level)
         if os.path.exists(path):
             if not self.config.get('noop'):
@@ -66,7 +75,6 @@ def rmtree(self, path, log_level=INFO, error_level=ERROR, exit_code=-1):
                     return -1
         else:
             self.debug("%s doesn't exist." % path)
-        return 0
 
     def _is_windows(self):
         system = platform.system()
@@ -139,7 +147,7 @@ def download_file(self, url, file_name=None, parent_dir=None,
             self.info("Downloading %s%s" % (url, message))
             f = urllib2.urlopen(req)
             if create_parent_dir and parent_dir:
-                self.mkdir_p(parent_dir)
+                self.mkdir_p(parent_dir, error_level=error_level)
             local_file = open(file_name, 'wb')
             local_file.write(f.read())
             local_file.close()
@@ -179,6 +187,60 @@ def copyfile(self, src, dest, log_level=INFO, error_level=ERROR):
             except (IOError, shutil.Error):
                 self.dump_exception("Can't copy %s to %s!" % (src, dest),
                                     level=error_level)
+                return -1
+
+    def write_to_file(self, file_path, contents, verbose=True,
+                      open_mode='w', create_parent_dir=False,
+                      error_level=ERROR):
+        """
+        Write contents to file_path.
+
+        This doesn't currently create the parent_dir or translate into
+        abs_path; that needs to be done beforehand, since OSMixin doesn't
+        necessarily have access to query_abs_dirs().
+
+        Returns file_path if successful, None if not.
+        """
+        self.info("Writing to file %s" % file_path)
+        if verbose:
+            self.info("Contents:")
+            for line in contents.splitlines():
+                self.info(" %s" % line)
+        if create_parent_dir:
+            parent_dir = os.path.dirname(file_path)
+            self.mkdir_p(parent_dir, error_level=error_level)
+        try:
+            fh = open(file_path, open_mode)
+            fh.write(contents)
+            fh.close()
+            return file_path
+        except IOError:
+            self.log("%s can't be opened for writing!" % file_path,
+                     level=error_level)
+
+    def read_from_file(self, file_path, verbose=True, open_mode='r',
+                       error_level=ERROR):
+        """
+        Reads from file_path.
+
+        Returns contents if successful, None if not.
+        """
+        self.info("Reading from file %s" % file_path)
+        if not os.path.exists(file_path):
+            self.log("%s doesn't exist!" % file_path, level=error_level)
+            return
+        try:
+            fh = open(file_path, open_mode)
+            contents = fh.read()
+            fh.close()
+            if verbose:
+                self.info("Contents:")
+                for line in contents.splitlines():
+                    self.info(" %s" % line)
+            return contents
+        except IOError:
+            self.log("%s can't be opened for reading!" % file_path,
+                     level=error_level)
 
     def chdir(self, dir_name, ignore_if_noop=False):
         self.log("Changing directory to %s." % dir_name)
@@ -403,8 +465,8 @@ def get_output_from_command(self, command, cwd=None,
         return_level = DEBUG
         output = None
         if os.path.exists(tmp_stdout_filename) and os.path.getsize(tmp_stdout_filename):
-            fh = open(tmp_stdout_filename)
-            output = fh.read()
+            output = self.read_from_file(tmp_stdout_filename,
+                                         verbose=False)
             if not silent:
                 self.info("Output received:")
                 output_lines = output.rstrip().splitlines()
@@ -414,18 +476,16 @@ def get_output_from_command(self, command, cwd=None,
                     line = line.decode("utf-8")
                     self.info(' %s' % line)
                 output = '\n'.join(output_lines)
-            fh.close()
         if os.path.exists(tmp_stderr_filename) and os.path.getsize(tmp_stderr_filename):
             return_level = ERROR
             self.error("Errors received:")
-            fh = open(tmp_stderr_filename)
-            errors = fh.read()
+            errors = self.read_from_file(tmp_stderr_filename,
+                                         verbose=False)
             for line in errors.rstrip().splitlines():
                 if not line or line.isspace():
                     continue
                 line = line.decode("utf-8")
                 self.error(' %s' % line)
-            fh.close()
         elif p.returncode:
             return_level = ERROR
         # Clean up.