bug 372746: Create command-line API for creating Bouncer/MirrorBrain …

…entries - add bouncer entries with release automation. r=bhearsum
mozilla · Jun 9, 2010 · d889e6f · d889e6f
1 parent 79f5023
commit d889e6f
Show file tree

Hide file tree

Showing 5 changed files with 369 additions and 0 deletions.
diff --git a/lib/python/release/__init__.py b/lib/python/release/__init__.py
@@ -0,0 +1 @@
+
diff --git a/lib/python/release/platforms.py b/lib/python/release/platforms.py
@@ -0,0 +1,53 @@
+# buildbot -> bouncer platform mapping
+bouncer_platform_map = {'win32': 'win', 'macosx': 'osx', 'linux': 'linux'}
+# buildbot -> ftp platform mapping
+ftp_platform_map = {'win32': 'win32', 'macosx': 'mac', 'linux': 'linux-i686'}
+# buildbot -> shipped-locales platform mapping
+sl_platform_map = {'win32': 'win32', 'macosx': 'osx', 'linux': 'linux'}
+
+def buildbot2bouncer(platform):
+    return bouncer_platform_map.get(platform, platform)
+
+def buildbot2ftp(platform):
+    return ftp_platform_map.get(platform, platform)
+
+def buildbot2shippedlocales(platform):
+    return sl_platform_map.get(platform, platform)
+
+def shippedlocales2buildbot(platform):
+    try:
+        return [k for k, v in sl_platform_map.iteritems() if v == platform][0]
+    except IndexError:
+        return platform
+
+def getPlatformLocales(shipped_locales, platforms):
+    platform_locales = {}
+    for platform in platforms:
+        platform_locales[platform] = []
+    f = open(shipped_locales)
+    for line in open(shipped_locales).readlines():
+        entry = line.split()
+        locale = entry[0]
+        if len(entry)>1:
+            for platform in entry[1:]:
+                if shippedlocales2buildbot(platform) in platforms:
+                    platform_locales[shippedlocales2buildbot(platform)].append(locale)
+        else:
+            for platform in platforms:
+                platform_locales[platform].append(locale)
+    f.close()
+    return platform_locales
+
+def getAllLocales(shipped_locales):
+    locales = []
+    f = open(shipped_locales)
+    for line in f.readlines():
+        entry = line.split()
+        locale = entry[0]
+        if locale:
+            locales.append(locale)
+    f.close()
+    return locales
+
+def getPlatforms():
+    return bouncer_platform_map.keys()
diff --git a/lib/python/release/versions.py b/lib/python/release/versions.py
@@ -0,0 +1,7 @@
+import re
+
+def getPrettyVersion(version):
+    version = re.sub(r'a([0-9]+)$', r' Alpha \1', version)
+    version = re.sub(r'b([0-9]+)$', r' Beta \1', version)
+    version = re.sub(r'rc([0-9]+)$', r' RC \1', version)
+    return version
diff --git a/release/firefox-tuxedo.ini b/release/firefox-tuxedo.ini
@@ -0,0 +1,12 @@
+[DEFAULT]
+complete_mar_template = /%(product)s/releases/%(version)s/update/%(ftp_platform)s/%(locale)s/%(product)s-%(version)s.complete.mar
+partial_mar_template = /%(product)s/releases/%(version)s/update/%(ftp_platform)s/%(locale)s/%(product)s-%(old_version)s-%(version)s.partial.mar
+
+[win32]
+full_product_template = /%(product)s/releases/%(version)s/%(ftp_platform)s/%(locale)s/%(brandName)s Setup %(prettyVersion)s.exe
+
+[linux]
+full_product_template = /%(product)s/releases/%(version)s/%(ftp_platform)s/%(locale)s/%(product)s-%(version)s.tar.bz2
+
+[macosx]
+full_product_template = /%(product)s/releases/%(version)s/%(ftp_platform)s/%(locale)s/%(brandName)s %(prettyVersion)s.dmg
diff --git a/release/tuxedo-add.py b/release/tuxedo-add.py
@@ -0,0 +1,296 @@
+#!/usr/bin/env python
+
+import optparse
+from optparse import Option, OptionParser, OptionError
+import urllib2
+from urllib import urlencode, quote
+import base64
+import sys
+import os
+import traceback
+from release.platforms import buildbot2bouncer, buildbot2ftp, \
+        getAllLocales, getPlatforms
+from release.versions import getPrettyVersion
+
+class TuxedoOption(Option):
+    ATTRS = Option.ATTRS + ['required']
+
+    def _check_required(self):
+        if self.required and not self.takes_value():
+            raise OptionError(
+                "required flag set for option that doesn't take a value",
+                 self)
+
+    # Make sure _check_required() is called from the constructor!
+    CHECK_METHODS = Option.CHECK_METHODS + [_check_required]
+
+    def process (self, opt, value, values, parser):
+        Option.process(self, opt, value, values, parser)
+        parser.option_seen[self] = 1
+
+class TuxedoOptionParser(OptionParser):
+
+    def _init_parsing_state(self):
+        OptionParser._init_parsing_state(self)
+        self.option_seen = {}
+
+    def check_values(self, values, args):
+        for option in self.option_list:
+            if (isinstance(option, Option) and
+                option.required and
+                not self.option_seen.has_key(option)):
+                self.print_help()
+                self.set_usage(optparse.SUPPRESS_USAGE)
+                self.error("%s not supplied" % option)
+        return (values, args)
+
+class TuxedoEntrySubmitter(object):
+
+    full_product_template = {}
+    complete_mar_template = {}
+    partial_mar_template = {}
+
+    def __init__(self, config, productName, version, tuxedoServerUrl,
+                 brandName=None, bouncerProductName=None, shippedLocales=None,
+                 addMARs=True, oldVersion=None, username=None, password=None,
+                 verbose=True, dryRun=False, platforms=None):
+        self.config = config
+        self.productName = productName
+        self.version = version
+        self.tuxedoServerUrl = tuxedoServerUrl
+        self.brandName = brandName or productName.capitalize()
+        self.bouncerProductName = bouncerProductName or productName.capitalize()
+        self.shippedLocales = shippedLocales
+        self.addMARs = addMARs
+        self.oldVersion = oldVersion
+        self.username = username
+        self.password = password
+        self.verbose = verbose
+        self.dryRun = dryRun
+        self.platforms = platforms
+
+        if not self.platforms:
+            self.platforms = getPlatforms()
+
+        if self.shippedLocales:
+            self.locales = getAllLocales(self.shippedLocales)
+        else:
+            self.locales = ('en-US',)
+
+        self.bouncer_product_name = '%s-%s' % (self.bouncerProductName,
+                                               self.version)
+        self.complete_mar_bouncer_product_name = '%s-%s-Complete' % \
+            (self.bouncerProductName, self.version)
+        self.partial_mar_bouncer_product_name = '%s-%s-Partial-%s' % \
+            (self.bouncerProductName, self.version,
+             self.oldVersion)
+        self.read_config()
+
+    def read_config(self):
+        """Example config file
+
+
+[DEFAULT]
+complete_mar_template = /%(product)s/releases/%(version)s/update/%(ftp_platform)s/%(locale)s/%(product)s-%(version)s.complete.mar
+partial_mar_template = /%(product)s/releases/%(version)s/update/%(ftp_platform)s/%(locale)s/%(product)s-%(old_version)s-%(version)s.partial.mar
+
+[win32]
+full_product_template = /%(product)s/releases/%(version)s/%(ftp_platform)s/%(locale)s/%(brandName)s Setup %(prettyVersion)s.exe
+
+[linux]
+full_product_template = /%(product)s/releases/%(version)s/%(ftp_platform)s/%(locale)s/%(product)s-%(version)s.tar.bz2
+
+[macosx]
+full_product_template = /%(product)s/releases/%(version)s/%(ftp_platform)s/%(locale)s/%(brandName)s %(prettyVersion)s.dmg
+
+"""
+        try:
+            import ConfigParser
+        except ImportError:
+            import configparser as ConfigParser
+
+        cfg = ConfigParser.RawConfigParser()
+        if not cfg.read(self.config):
+            print >> sys.stderr, "Cannot read %s" % self.config
+            sys.exit(2)
+
+        for platform in self.platforms:
+            self.full_product_template[platform] = cfg.get(platform,
+                                                           'full_product_template')
+            self.complete_mar_template[platform] = cfg.get(platform,
+                                                           'complete_mar_template')
+            self.partial_mar_template[platform] = cfg.get(platform,
+                                                          'partial_mar_template')
+
+
+    def tuxedoRequest(self, url, postdata=None):
+        full_url = self.tuxedoServerUrl + url
+        request = urllib2.Request(full_url)
+        if self.username and self.password:
+            basicAuth = base64.encodestring('%s:%s' % (self.username, self.password))
+            request.add_header("Authorization", "Basic %s" % basicAuth.strip())
+        if postdata:
+            if isinstance(postdata, dict):
+                postdata = urlencode(postdata)
+            request.add_data(postdata)
+        if self.dryRun:
+            print >> sys.stderr, "Tuxedo API URL: %s" % full_url
+            if postdata:
+                print >> sys.stderr, "POST data: %s" % postdata
+            return
+        try:
+            return urllib2.urlopen(request).read()
+        except urllib2.URLError:
+            print >> sys.stderr, "FAILED: Tuxedo API error. URL: %s" % full_url
+            if postdata:
+                print >> sys.stderr, "POST data: %s" % postdata
+            traceback.print_exc(file=sys.stdout)
+            sys.exit(1)
+
+    def product_add(self, product):
+        if self.verbose:
+            print "Adding product: %s" % product
+            print "Locales: %s" % ", ".join(self.locales)
+        locales_post_data = ["languages=%s" % l for l in self.locales]
+        locales_post_data = "&".join(locales_post_data)
+        response = self.tuxedoRequest("product_add/",
+                                      "product=" + quote(product) + "&" +
+                                      locales_post_data)
+        if self.verbose:
+            print "Server response:"
+            print response
+
+    def location_add(self, product, platform, path):
+        path = path.replace(' ', '%20')
+        if self.verbose:
+            print "Adding location for %s, %s: %s" % \
+                    (product, buildbot2bouncer(platform), path)
+        self.tuxedoRequest("location_add/",
+                           {'product': product,
+                            'os': buildbot2bouncer(platform),
+                            'path': path})
+
+    def add_products(self):
+        self.product_add(self.bouncer_product_name)
+        if self.addMARs:
+            self.product_add(self.complete_mar_bouncer_product_name)
+            if self.oldVersion:
+                self.product_add(self.partial_mar_bouncer_product_name)
+
+    def add_locations(self):
+        for platform in self.platforms:
+            template_dict = {'product': self.productName,
+                             'brandName': self.brandName,
+                             'bouncer_product': self.bouncerProductName,
+                             'version': self.version,
+                             'prettyVersion': getPrettyVersion(self.version),
+                             'old_version': self.oldVersion,
+                             'ftp_platform': buildbot2ftp(platform),
+                             'locale': ':lang'}
+            # Full product
+            path = self.full_product_template[platform] % template_dict
+            self.location_add(self.bouncer_product_name, platform, path)
+
+            # Complete MAR product
+            if self.addMARs:
+                path = self.complete_mar_template[platform] % template_dict
+                self.location_add(self.complete_mar_bouncer_product_name,
+                                  platform, path)
+
+                # Partial MAR product
+                if self.oldVersion:
+                    path = self.partial_mar_template[platform] % template_dict
+                    self.location_add(self.partial_mar_bouncer_product_name,
+                                      platform, path)
+
+    def submit(self):
+        self.add_products()
+        self.add_locations()
+
+def getOptions():
+    parser = TuxedoOptionParser(option_class=TuxedoOption)
+    parser.add_option("--config", dest="config",
+                      required=True,
+                      help="Configuration file")
+    parser.add_option("-p", "--product-name", dest="productName",
+                      required=True,
+                      help="Product name")
+    parser.add_option("-v", "--version", dest="version",
+                      required=True,
+                      help="Product version")
+    parser.add_option("-t", "--tuxedo-server-url", dest="tuxedoServerUrl",
+                      required=True,
+                      help="Bouncer/Tuxedo API URL")
+    parser.add_option("-r", "--brand-name", dest="brandName",
+                      help="Brand name")
+    parser.add_option("-b", "--bouncer-product-name", dest="bouncerProductName",
+                      help="Bouncer product name")
+    parser.add_option("-l", "--shipped-locales", dest="shippedLocales",
+                      help="shipped-locales file location")
+    parser.add_option("-m", "--add-mars", action="store_true",
+                      dest="addMARs", default=False,
+                      help="Add MAR entries")
+    parser.add_option("-o", "--old-version", dest="oldVersion",
+                      help="Old product version")
+    parser.add_option("--platform", action="append",
+                      dest="platforms",
+                      help="Platform(s) to be processed")
+    parser.add_option("--dry-run", action="store_true", dest="dryRun",
+                      default=False,
+                      help="Print debug information and exit")
+    parser.add_option("--username", dest="username",
+                      help="Tuxedo user name")
+    parser.add_option("--password", dest="password",
+                      help="Tuxedo password")
+    parser.add_option("--credentials-file", dest="credentials",
+                      help="Get Tuxedo username/password from file")
+    parser.add_option("-q", "--quiet", action="store_false", dest="verbose",
+                      default=True,
+                      help="Don't print status messages to stdout")
+    return parser.parse_args()
+
+def credentials_from_file(credentials_file):
+    try:
+        module, _ = os.path.splitext(credentials_file)
+        credentials = __import__(module)
+        username = credentials.tuxedoUsername
+        password = credentials.tuxedoPassword
+        return (username, password)
+    except ImportError:
+        print >> sys.stderr, \
+                "Cannot open credentials file: %s" % credentials_file
+        sys.exit(3)
+    except AttributeError:
+        print >> sys.stderr, \
+                "Cannot retrieve credentials file: %s" % credentials_file
+        sys.exit(4)
+
+def main():
+    (options, args) = getOptions()
+
+    username = options.username
+    password = options.password
+    if options.credentials:
+        (username, password) = credentials_from_file(options.credentials)
+
+    tuxedo = TuxedoEntrySubmitter(config=options.config,
+                                  productName=options.productName,
+                                  version=options.version,
+                                  tuxedoServerUrl=options.tuxedoServerUrl,
+                                  brandName=options.brandName,
+                                  bouncerProductName=options.bouncerProductName,
+                                  shippedLocales=options.shippedLocales,
+                                  addMARs=options.addMARs,
+                                  oldVersion=options.oldVersion,
+                                  username=username,
+                                  password=password,
+                                  verbose=options.verbose,
+                                  dryRun=options.dryRun,
+                                  platforms=options.platforms,
+                                 )
+    tuxedo.submit()
+
+
+if __name__ == '__main__':
+    main()
+