bug 785965: adjust push to mirrors process for CDN. r=nthomas

--HG-- rename : scripts/release/push-to-mirrors.py => scripts/release/stage-tasks.py rename : scripts/release/push-to-mirrors.sh => scripts/release/stage-tasks.sh
mozilla · Oct 2, 2012 · 0f2fd4c · 0f2fd4c
1 parent 695635c
commit 0f2fd4c
Show file tree

Hide file tree

Showing 6 changed files with 100 additions and 4 deletions.
diff --git a/lib/python/mozilla_buildtools/test/test_release_paths.py b/lib/python/mozilla_buildtools/test/test_release_paths.py
@@ -0,0 +1,20 @@
+import unittest
+
+from release.paths import makeReleasesDir
+
+class TestReleasesDir(unittest.TestCase):
+    def testBaseReleases(self):
+        got = makeReleasesDir('bbb')
+        self.assertEquals('/pub/mozilla.org/bbb/releases/', got)
+
+    def testVersioned(self):
+        got = makeReleasesDir('aa', '15.1')
+        self.assertEquals('/pub/mozilla.org/aa/releases/15.1/', got)
+
+    def testRemote(self):
+        got = makeReleasesDir('yy', protocol='http', server='foo.bar')
+        self.assertEquals('http://foo.bar/pub/mozilla.org/yy/releases/', got)
+
+    def testRemoteAndVersioned(self):
+        got = makeReleasesDir('yx', '1.0', protocol='https', server='cee.dee')
+        self.assertEquals('https://cee.dee/pub/mozilla.org/yx/releases/1.0/', got)
diff --git a/lib/python/release/paths.py b/lib/python/release/paths.py
@@ -23,12 +23,14 @@ def makeCandidatesDir(product, version, buildNumber, nightlyDir=None,
     else:
         return directory
 
-def makeReleasesDir(product, version, protocol=None, server=None,
+def makeReleasesDir(product, version=None, protocol=None, server=None,
                     ftp_root='/pub/mozilla.org/'):
     if protocol:
         assert server is not None, "server is required with protocol"
 
-    directory = '%s%s/releases/%s/' % (ftp_root, product, version)
+    directory = '%s%s/releases/' % (ftp_root, product)
+    if version:
+        directory += '%s/' % version
 
     if protocol:
         return urlunsplit((protocol, server, directory, None, None))

diff --git a/lib/python/util/commands.py b/lib/python/util/commands.py
@@ -57,7 +57,7 @@ def run_remote_cmd(cmd, server, username=None, sshKey=None, ssh='ssh',
     cmd_prefix.append(server)
     if isinstance(cmd, basestring):
         cmd = [cmd]
-    run_cmd(cmd_prefix + cmd, **kwargs)
+    return run_cmd(cmd_prefix + cmd, **kwargs)
 
 def get_output(cmd, include_stderr=False, dont_log=False, **kwargs):
     """Run cmd (a list of arguments) and return the output.  If include_stderr

diff --git a/lib/python/util/transfer.py b/lib/python/util/transfer.py
@@ -0,0 +1,8 @@
+from util.commands import run_cmd
+
+def scp(src, dst, sshKey=None):
+    cmd = ['scp', '-o', 'BatchMode=yes']
+    if sshKey:
+        cmd.extend(['-o', 'IdentityFile=%s' % sshKey])
+    cmd.extend([src, dst])
+    return run_cmd(cmd)
diff --git a/scripts/release/push-to-mirrors.py → scripts/release/stage-tasks.py b/scripts/release/push-to-mirrors.py → scripts/release/stage-tasks.py
@@ -5,6 +5,7 @@
 import sys
 import logging
 from subprocess import CalledProcessError
+from tempfile import NamedTemporaryFile
 try:
     import json
 except ImportError:
@@ -18,6 +19,7 @@
 from release.paths import makeCandidatesDir, makeReleasesDir
 from util.hg import update, make_hg_url, mercurial
 from util.commands import run_remote_cmd
+from util.transfer import scp
 
 
 DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url('hg.mozilla.org',
@@ -132,6 +134,45 @@ def pushToMirrors(productName, version, buildNumber, stageServer,
                    server=stageServer, username=stageUsername,
                    sshKey=stageSshKey)
 
+
+indexFileTemplate = """\
+<!DOCTYPE html>
+<html><head>
+<meta http-equiv="content-type" content="text/html; charset=UTF-8">
+<style media="all">@import "http://www.mozilla.org/style/firefox/4.0beta/details.css";</style>
+<title>Thanks for your interest in Firefox %(version)s</title>
+</head>
+<body>
+<h1>Thanks for your interest in Firefox %(version)s</h1>
+<p>We aren't quite finished qualifying Firefox %(version)s yet. You should check out the latest <a href="http://www.mozilla.org/firefox/channel">Beta</a>.</p>
+<p>When we're all done with Firefox %(version)s it will show up on <a href="http://firefox.com?ref=ftp">Firefox.com</a>.</p>
+</body>
+</html>"""
+
+def makeIndexFiles(productName, version, buildNumber, stageServer,
+                     stageUsername, stageSshKey):
+    candidates_dir = makeCandidatesDir(productName, version, buildNumber)
+    indexFile = NamedTemporaryFile()
+    indexFile.write(indexFileTemplate % {'version': version})
+    indexFile.flush()
+
+    scp(indexFile.name, '%s@%s:%s/index.html' % (stageUsername, stageServer, candidates_dir))
+    run_remote_cmd(['find', candidates_dir, '-mindepth', '1', '-type', 'd', '-exec', 'cp', '-pv', '%s/index.html' % candidates_dir, '{}', '\\;'],
+                   server=stageServer, username=stageUsername, sshKey=stageSshKey)
+
+def deleteIndexFiles(cleanup_dir, stageServer, stageUsername,
+                     stageSshKey):
+    run_remote_cmd(['find', cleanup_dir, '-name', 'index.html', '-exec', 'rm', '-v', '{}', '\\;'],
+                   server=stageServer, username=stageUsername, sshKey=stageSshKey)
+
+def updateSymlink(productName, version, stageServer, stageUsername,
+                  stageSshKey):
+    releases_dir = makeReleasesDir(productName)
+
+    run_remote_cmd(['cd %s && rm latest && ln -s %s latest' % (releases_dir, version)],
+                   server=stageServer, username=stageUsername, sshKey=stageSshKey)
+
+
 if __name__ == '__main__':
     from optparse import OptionParser
     parser = OptionParser("")
@@ -165,6 +206,7 @@ def pushToMirrors(productName, version, buildNumber, stageServer,
     stageUsername = options.ssh_username or branchConfig['stage_username']
     stageSshKey = options.ssh_key or branchConfig["stage_ssh_key"]
     stageSshKey = path.join(os.path.expanduser("~"), ".ssh", stageSshKey)
+    createIndexFiles = releaseConfig.get('makeIndexFiles', False)
 
     if 'permissions' in args:
         checkStagePermissions(stageServer=stageServer,
@@ -193,10 +235,34 @@ def pushToMirrors(productName, version, buildNumber, stageServer,
                       dryRun=True)
 
     if 'push' in args:
+        if createIndexFiles:
+            makeIndexFiles(stageServer=stageServer,
+                           stageUsername=stageUsername,
+                           stageSshKey=stageSshKey,
+                           productName=productName,
+                           version=version,
+                           buildNumber=buildNumber)
         pushToMirrors(stageServer=stageServer,
                       stageUsername=stageUsername,
                       stageSshKey=stageSshKey,
                       productName=productName,
                       version=version,
                       extra_excludes=options.extra_excludes,
                       buildNumber=buildNumber)
+        if createIndexFiles:
+            deleteIndexFiles(stageServer=stageServer,
+                             stageUsername=stageUsername,
+                             stageSshKey=stageSshKey,
+                             cleanup_dir=makeCandidatesDir(productName, version, buildNumber))
+
+    if 'postrelease' in args:
+        if createIndexFiles:
+            deleteIndexFiles(stageServer=stageServer,
+                             stageUsername=stageUsername,
+                             stageSshKey=stageSshKey,
+                             cleanup_dir=makeReleasesDir(productName, version))
+        updateSymlink(stageServer=stageServer,
+                      stageUsername=stageUsername,
+                      stageSshKey=stageSshKey,
+                      productName=productName,
+                      version=version)
diff --git a/scripts/release/push-to-mirrors.sh → scripts/release/stage-tasks.sh b/scripts/release/push-to-mirrors.sh → scripts/release/stage-tasks.sh
@@ -42,5 +42,5 @@ $PYTHON $SCRIPTS_DIR/buildfarm/maintenance/purge_builds.py \
   -s 0.3 -n info -n 'rel-*' -n $slavebuilddir
 cd $workdir
 
-$PYTHON $MY_DIR/push-to-mirrors.py -c $branchConfig -r $releaseConfig \
+$PYTHON $MY_DIR/stage-tasks.py -c $branchConfig -r $releaseConfig \
   -b $BUILDBOT_CONFIGS -t $releaseTag $action $@