Bug 708656 - Use signing on demand for releases. r=bhearsum

--HG-- rename : scripts/l10n/release_repacks.sh => scripts/release/generate-sums.sh
mozilla · Jan 18, 2012 · dbeafbe · dbeafbe
1 parent 443ae69
commit dbeafbe
Show file tree

Hide file tree

Showing 13 changed files with 449 additions and 56 deletions.
diff --git a/lib/perl/Release/Patcher/Config.pm b/lib/perl/Release/Patcher/Config.pm
@@ -77,6 +77,10 @@ sub GetReleaseBlock {
       '/pub/mozilla.org/' . $product . '/nightly/' . $version . '-candidates/' .
       $buildStr . '/update/%platform%/%locale%/' . $product . '-' . $version .
       '.complete.mar';
+    $releaseBlock->{'checksumsurl'} = 'http://' . $stagingServer .
+      '/pub/mozilla.org/' . $product . '/nightly/' . $version . '-candidates/' .
+      $buildStr . '/%platform%/%locale%/' . $product . '-' . $version .
+      '.checksums';
 
     $releaseBlock->{'exceptions'} = {};
 

diff --git a/lib/python/build/l10n.py b/lib/python/build/l10n.py
@@ -5,7 +5,7 @@
 from urllib2 import urlopen
 from urlparse import urljoin
 
-from release.platforms import getPlatformLocales
+from release.platforms import getPlatformLocales, buildbot2ftp
 from util.commands import run_cmd
 from util.hg import mercurial, update
 from util.paths import windows2msys
@@ -59,7 +59,9 @@ def l10nRepackPrep(sourceRepoName, objdir, mozconfigPath,
                     env=env)
 
 def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir,
-                 l10nIni, compareLocalesRepo, env, merge=True):
+                 l10nIni, compareLocalesRepo, env, merge=True,
+                 prevMar=None, productName=None, platform=None,
+                 version=None, oldVersion=None):
     repo = "/".join([l10nBaseRepo, locale])
     localeDir = path.join(l10nRepoDir, locale)
     retry(mercurial, args=(repo, localeDir))
@@ -74,7 +76,44 @@ def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir,
     if sys.platform.startswith('darwin'):
         env["MOZ_PKG_PLATFORM"] = "mac"
     run_cmd(["make", "installers-%s" % locale], cwd=localeSrcDir, env=env)
-    retry(run_cmd, args=(["make", "upload", "AB_CD=%s" % locale],),
+    UPLOAD_EXTRA_FILES = []
+    if prevMar:
+        nativeDistDir = path.normpath(path.abspath(path.join(localeSrcDir,
+                                                             '../../dist')))
+        posixDistDir = windows2msys(nativeDistDir)
+        mar = '%s/host/bin/mar' % posixDistDir
+        mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir
+        current = '%s/current' % posixDistDir
+        previous = '%s/previous' % posixDistDir
+        updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale)
+        updateAbsDir = '%s/%s' % (posixDistDir, updateDir)
+        current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName, version)
+        partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion,
+                                                     version)
+        partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name)
+        UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name))
+        env['MAR'] = mar
+        env['MBSDIFF'] = mbsdiff
+        run_cmd(['rm', '-rf', previous, current])
+        run_cmd(['mkdir', previous, current])
+        run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl',
+                 '../../../../%s' % prevMar],
+                cwd=path.join(nativeDistDir, 'previous'), env=env)
+        run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl',
+                 current_mar], cwd=path.join(nativeDistDir, 'current'), env=env)
+        run_cmd(
+            ['bash', '../../tools/update-packaging/make_incremental_update.sh',
+             partial_mar, previous, current], cwd=nativeDistDir, env=env)
+        if os.environ.get('MOZ_SIGN_CMD'):
+            run_cmd(['bash', '-c',
+                     '%s -f gpg -f mar "%s"' %
+                     (os.environ['MOZ_SIGN_CMD'], partial_mar)],
+                     env=env)
+            UPLOAD_EXTRA_FILES.append('%s/%s.asc' % (updateDir, partial_mar_name))
+
+    retry(run_cmd,
+          args=(["make", "upload", "AB_CD=%s" % locale,
+                 'UPLOAD_EXTRA_FILES=%s' % ' '.join(UPLOAD_EXTRA_FILES)],),
           kwargs={'cwd': localeSrcDir, 'env': env})
 
 def getLocalesForChunk(possibleLocales, chunks, thisChunk):

diff --git a/lib/python/build/upload.py b/lib/python/build/upload.py
@@ -16,6 +16,7 @@ def postUploadCmdPrefix(upload_dir=None,
         to_candidates=False,
         to_mobile_candidates=False,
         nightly_dir=None,
+        signed=False,
         ):
     """Returns a post_upload.py command line for the given arguments.
     It is expected that the returned value is augmented with the list of files
@@ -60,5 +61,7 @@ def postUploadCmdPrefix(upload_dir=None,
         cmd.append("--release-to-mobile-candidates-dir")
     if nightly_dir:
         cmd.append("--nightly-dir=%s" % nightly_dir)
+    if signed:
+            cmd.append("--signed")
 
     return " ".join(cmd)
diff --git a/lib/python/release/download.py b/lib/python/release/download.py
@@ -2,12 +2,14 @@
 from os import path
 import urllib
 from urllib import urlretrieve
+from urllib2 import urlopen, HTTPError
 
-from release.platforms import ftp_platform_map
+from release.platforms import ftp_platform_map, buildbot2ftp
 from release.l10n import makeReleaseRepackUrls
 from release.paths import makeCandidatesDir
 from util.paths import windows2msys
 from util.file import directoryContains
+from util.commands import run_cmd
 
 import logging
 log = logging.getLogger(__name__)
@@ -27,11 +29,13 @@ def getInstallerExt(platform):
     return installer_ext_map[platform]
 
 def downloadReleaseBuilds(stageServer, productName, brandName, version,
-                          buildNumber, platform, candidatesDir=None):
+                          buildNumber, platform, candidatesDir=None,
+                          signed=False):
     if candidatesDir is None:
         candidatesDir = makeCandidatesDir(productName, version, buildNumber,
                                           protocol='http', server=stageServer)
-    files = makeReleaseRepackUrls(productName, brandName, version, platform)
+    files = makeReleaseRepackUrls(productName, brandName, version, platform,
+                                  signed=signed)
 
     env = {}
     for fileName, remoteFile in files.iteritems():
@@ -50,6 +54,34 @@ def downloadReleaseBuilds(stageServer, productName, brandName, version,
 
     return env
 
+def downloadUpdate(stageServer, productName, version, buildNumber,
+                   platform, locale, candidatesDir=None):
+    if candidatesDir is None:
+        candidatesDir = makeCandidatesDir(productName, version, buildNumber,
+                                          protocol='http', server=stageServer)
+    fileName = '%s-%s.complete.mar' % (productName, version)
+    destFileName = '%s-%s.%s.complete.mar' % (productName, version, locale)
+    platformDir = buildbot2ftp(platform)
+    url = '/'.join([p.strip('/') for p in [
+        candidatesDir, 'update', platformDir, locale, fileName]])
+    log.info("Downloading %s to %s", url, destFileName)
+    remote_f = urlopen(url)
+    local_f = open(destFileName, "wb")
+    local_f.write(remote_f.read())
+    local_f.close()
+    return destFileName
+
+def downloadUpdateIgnore404(*args, **kwargs):
+    try:
+        return downloadUpdate(*args, **kwargs)
+    except HTTPError, e:
+        if e.code == 404:
+            # New locale
+            log.warning('Got 404. Skipping %s' % e.geturl())
+            return None
+        else:
+            raise
+
 def expectedFiles(unsignedDir, locale, platform, signedPlatforms,
         firstLocale='en-US'):
     """ When checking a full set of downloaded release builds,
@@ -84,3 +116,17 @@ def expectedFiles(unsignedDir, locale, platform, signedPlatforms,
         langpack = False
 
     return update and packages and langpack
+
+def rsyncFilesByPattern(server, userName, sshKey, source_dir, target_dir,
+                        pattern):
+    cmd = ['rsync', '-e',
+           'ssh -l %s -oIdentityFile=%s' % (userName, sshKey),
+           '-av', '--include=%s' % pattern, '--include=*/', '--exclude=*',
+           '%s:%s' % (server, source_dir), target_dir]
+    run_cmd(cmd)
+
+def rsyncFiles(files, server, userName, sshKey, target_dir):
+    cmd = ['rsync', '-e',
+           'ssh -l %s -oIdentityFile=%s' % (userName, sshKey),
+           '-av'] + files + ['%s:%s' % (server, target_dir)]
+    run_cmd(cmd)
diff --git a/lib/python/release/l10n.py b/lib/python/release/l10n.py
@@ -70,7 +70,7 @@ def getL10nRepositories(fileName, l10nRepoPath, relbranch=None):
 
 
 def makeReleaseRepackUrls(productName, brandName, version, platform,
-                          locale='en-US'):
+                          locale='en-US', signed=False):
     longVersion = version
     builds = {}
     if productName not in ('fennec',):
@@ -86,12 +86,18 @@ def makeReleaseRepackUrls(productName, brandName, version, platform,
         elif platform.startswith('win'):
             filename = '%s.zip' % productName
             instname = '%s.exe' % productName
-            builds[filename] = '/'.join([p.strip('/') for p in [
-                'unsigned', platformDir, locale,
-                '%s-%s.zip' % (productName, version)]])
-            builds[instname] = '/'.join([p.strip('/') for p in [
-                'unsigned', platformDir, locale,
-                '%s Setup %s.exe' % (brandName, longVersion)]])
+            prefix = []
+            if not signed:
+                prefix.append('unsigned')
+            prefix.extend([platformDir, locale])
+            builds[filename] = '/'.join(
+                [p.strip('/') for p in 
+                 prefix + ['%s-%s.zip' % (productName, version)]]
+            )
+            builds[instname] = '/'.join(
+                [p.strip('/') for p in 
+                 prefix + ['%s Setup %s.exe' % (brandName, longVersion)]]
+            )
         else:
             raise "Unsupported platform"
     else:

diff --git a/lib/python/release/signing.py b/lib/python/release/signing.py
@@ -0,0 +1,42 @@
+import os
+from util.commands import run_cmd
+
+def generateChecksums(checksums_dir, sums_info):
+    """
+    Generates {MD5,SHA1,etc}SUMS files using *.checksums files.
+
+    @type  checksums_dir: string
+    @param checksums_dir: Directory name wich *.checksums files
+
+    @type  sums_info: dict
+    @param sums_info: A dictionary which contains hash type and output file
+                      pairs. Example: {'sha1': '/tmp/SHA1SUMS',
+                                       'md5': '/tmp/MD5SUMS'}
+    """
+    sums = {}
+    for hash_type in sums_info.keys():
+        sums[hash_type] = []
+    for top, dirs, files in os.walk(checksums_dir):
+        files = [f for f in files if f.endswith('.checksums')]
+        for f in files:
+            fd = open(os.path.join(top, f))
+            for line in fd:
+                line = line.rstrip()
+                try:
+                    hash, hash_type, size, file_name = line.split(None, 3)
+                except ValueError:
+                    print "Failed to parse the following line:"
+                    print line
+                    raise
+                if sums.has_key(hash_type):
+                    sums[hash_type].append((hash, file_name))
+    for hash_type in sums_info.keys():
+        sums_file = open(sums_info[hash_type], 'w')
+        # sort by file name
+        for hash, file_name in sorted(sums[hash_type], key=lambda x: x[1]):
+            sums_file.write('%s  %s\n' % (hash, file_name))
+        sums_file.close()
+
+def signFiles(files):
+    for f in files:
+        run_cmd(['bash', '-c', os.environ['MOZ_SIGN_CMD'] + ' -f gpg "%s"' % f])