makebumpver

#!/usr/bin/python2
#
# makebumpver - Increment version number and add in RPM spec file changelog
#               block.  Ensures rhel*-branch commits reference RHEL bugs.
#
# Copyright (C) 2009-2015  Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Author: David Cantrell <dcantrell@redhat.com>

import logging
logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger("bugzilla")
log.setLevel(logging.INFO)

import bugzilla
from contextlib import closing
import datetime
import getopt
import getpass
import json
import os
import re
import subprocess
import sys
import textwrap
import urllib

class MakeBumpVer:
    def __init__(self, *args, **kwargs):
        log.debug("%s", kwargs)
        self.bzserver = 'bugzilla.redhat.com'
        self.bzurl = "https://%s/xmlrpc.cgi" % self.bzserver
        self.jenkins = None
        self.jenkins_proxy = None
        self.username = None
        self.password = None
        self.bz = None
        self._bz_cache = {}

        authfile = os.path.realpath(os.getenv('HOME') + '/.rhbzauth')
        if os.path.isfile(authfile):
            f = open(authfile, 'r')
            lines = map(lambda x: x.strip(), f.readlines())
            f.close()

            for line in lines:
                if line.startswith('RHBZ_USER='):
                    self.username = line[10:].strip('"\'')
                elif line.startswith('RHBZ_PASSWORD='):
                    self.password = line[14:].strip('"\'')
                elif line.startswith('JENKINS='):
                    self.jenkins = line[8:].strip('"\'')
                elif line.startswith('JENKINS_PROXY='):
                    self.jenkins_proxy = line[14:].strip('"\'')

        self.gituser = self._gitConfig('user.name')
        self.gitemail = self._gitConfig('user.email')

        self.name = kwargs.get('name')
        self.version = kwargs.get('version')
        self.release = kwargs.get('release')
        self.new_release = kwargs.get('new_release') or self.release
        self.bugreport = kwargs.get('bugreport')
        self.ignore = kwargs.get('ignore')

        self.bugmap = {}
        bugmap = kwargs.get('bugmap')
        if bugmap and bugmap != '':
            maps = bugmap.split(',')
            for mapping in maps:
                bugs = mapping.split('=')
                if len(bugs) == 2:
                    self.bugmap[bugs[0]] = bugs[1]

        self.configure = kwargs.get('configure')
        self.spec = kwargs.get('spec')
        self.skip_acks = kwargs.get('skip_acks', False)
        self.skip_all = kwargs.get('skip_all', False)
        self.zanata_config = kwargs.get('zanata_config')
        self.skip_zanata = kwargs.get("skip_zanata", False)
        self.skip_jenkins = kwargs.get("skip_jenkins", False)

        if self.skip_all:
            self.skip_acks = True
            self.skip_jenkins = True
            self.skip_zanata = True

        self.git_branch = None

        # RHEL release number or None (also fills in self.git_branch)
        self.rhel = self._isRHEL()

    def _gitConfig(self, field):
        proc = subprocess.Popen(['git', 'config', field],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE).communicate()
        return proc[0].strip('\n')

    def _incrementVersion(self):
        fields = self.version.split('.')
        fields[-1] = str(int(fields[-1]) + 1)
        new = ".".join(fields)
        return new

    def _isRHEL(self):
        proc = subprocess.Popen(['git', 'branch'],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE).communicate()
        lines = filter(lambda x: x.startswith('*'),
                       proc[0].strip('\n').split('\n'))

        if lines == [] or len(lines) > 1:
            return False

        fields = lines[0].split(' ')

        if len(fields) == 2:
            self.git_branch = fields[1]

        if len(fields) == 2 and fields[1].startswith('rhel'):
            branch_pattern=r"^rhel(\d+)-(.*)"
            m = re.match(branch_pattern, fields[1])
            if m:
                return m.group(1)
        return False

    def _getCommitDetail(self, commit, field):
        proc = subprocess.Popen(['git', 'log', '-1',
                                 "--pretty=format:%s" % field, commit],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE).communicate()

        ret = proc[0].strip('\n').split('\n')

        if len(ret) == 1 and ret[0].find('@') != -1:
            ret = ret[0].split('@')[0]
        elif len(ret) == 1:
            ret = ret[0]
        else:
            ret = filter(lambda x: x != '', ret)

        return ret

    def _queryBug(self, bugid):
        if not self.bz:
            sys.stdout.write("Connecting to %s...\n" % self.bzserver)

            if not self.username:
                sys.stdout.write('Username: ')
                self.username = sys.stdin.readline()
                self.username = self.username.strip()

            if not self.password:
                self.password = getpass.getpass()

            bzclass = bugzilla.Bugzilla
            self.bz = bzclass(url=self.bzurl)
            print

        if not self.bz.logged_in:
            rc = self.bz.login(self.username, self.password)
            log.debug("login rc = %s", rc)

        if bugid in self._bz_cache:
            return self._bz_cache[bugid]

        bug = self.bz.getbug(bugid, extra_fields="flags")
        log.debug("bug = %s", bug)

        if not bug:
            return None
        else:
            self._bz_cache[bugid] = bug
            return bug

    def _isRHELBug(self, bug, commit, summary):
        bzentry = self._queryBug(bug)

        if not bzentry:
            print("*** Bugzilla query for %s failed.\n" % bug)
            return False

        if bzentry.product.startswith('Red Hat Enterprise Linux'):
            return True
        else:
            print("*** Bug %s is not a RHEL bug." % bug)
            print("***     Commit: %s" % commit)
            print("***     %s\n" % summary)
            return False

    def _isRHELBugInCorrectState(self, bug, commit, summary):
        bzentry = self._queryBug(bug)

        if not bzentry:
            print("*** Bugzilla query for %s failed.\n" % bug)
            return False

        if bzentry.bug_status in ['MODIFIED', 'ON_QA']:
            return True
        else:
            print("*** Bug %s is not in MODIFIED or ON_QA." % bug)
            print("***     Commit: %s" % commit)
            print("***     %s\n" % summary)
            return False

    def _isRHELBugFixedInVersion(self, bug, commit, summary, fixedIn):
        bzentry = self._queryBug(bug)

        if not bzentry:
            print("*** Bugzilla query for %s failed.\n" % bug)
            return False

        if bzentry.fixed_in == fixedIn:
            return True
        else:
            print("*** Bug %s does not have correct Fixed In Version." % bug)
            print("***     Found:     %s" % bzentry.fixed_in)
            print("***     Expected:  %s" % fixedIn)
            print("***     Commit:    %s" % commit)
            print("***     %s\n" % summary)
            return False

    def _isRHELBugAcked(self, bug, commit, summary):
        """ Check the bug's ack state
        """
        if not self.rhel or self.skip_acks:
            return True

        bzentry = self._queryBug(bug)
        ack_pattern=r"rhel-%s\.\d+\.\d+" % self.rhel
        for f in bzentry.flags:
            if re.match(ack_pattern, f['name']) and f['status'] == '+':
                return True

        print("*** Bug %s does not have ACK" % bug)
        print("***     Commit: %s" % commit)
        print("***     %s\n" % summary)
        return False

    def _rpmLog(self, fixedIn):
        git_range = "%s-%s-%s.." % (self.name, self.version, self.release)
        proc = subprocess.Popen(['git', 'log', '--pretty=oneline', git_range],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE).communicate()
        lines = filter(lambda x: x.find('l10n: ') != 41 and \
                                 x.find('Merge commit') != 41 and \
                                 x.find('Merge branch') != 41,
                       proc[0].strip('\n').split('\n'))

        if self.ignore and self.ignore != '':
            for commit in self.ignore.split(','):
                lines = filter(lambda x, c=commit: not x.startswith(c), lines)

        rpm_log = []
        bad_bump = False
        bad = False

        for line in lines:
            if not line:
                continue
            fields = line.split(' ')
            commit = fields[0]

            summary = self._getCommitDetail(commit, "%s")
            body = self._getCommitDetail(commit, "%b")
            if type(body) is not list:
                body = [body]
            author = self._getCommitDetail(commit, "%aE")

            if self.rhel:
                rhbz = set()
                bad = False

                # look for a bug in the summary line, validate if found
                m = re.search(r"\(#\d+(\,.*)*\)", summary)
                if m:
                    fullbug = summary[m.start():m.end()]
                    bugstr = summary[m.start()+2:m.end()-1]

                    bug = ''
                    for c in bugstr:
                        if c.isdigit():
                            bug += c
                        else:
                            break

                    if len(bugstr) > len(bug):
                        tmp = bugstr[len(bug):]

                        for c in tmp:
                            if not c.isalpha():
                                tmp = tmp[1:]
                            else:
                                break

                        if len(tmp) > 0:
                            author = tmp

                    ckbug = self.bugmap.get(bug, bug)

                    valid = self.skip_all or self._isRHELBug(ckbug, commit, summary)

                    if valid:
                        summary = summary.replace(fullbug, "(%s)" % author)
                        rhbz.add("Resolves: rhbz#%s" % ckbug)

                        if not self.skip_all:
                            if not self._isRHELBugInCorrectState(ckbug, commit,
                                                                 summary):
                                bad = True

                            if not self._isRHELBugFixedInVersion(ckbug, commit,
                                                                 summary, fixedIn):
                                bad = True

                            if not self._isRHELBugAcked(ckbug, commit, summary):
                                bad = True
                    else:
                        bad = True
                    summary_bug = ckbug
                else:
                    summary = summary.strip()
                    summary += " (%s)" % author
                    summary_bug = None

                for bodyline in body:
                    m = re.match(r"^(Resolves|Related|Conflicts):\ +rhbz#\d+.*$",
                                 bodyline)
                    if not m:
                        continue

                    actionre = re.search("(Resolves|Related|Conflicts)",
                                         bodyline)
                    bugre = re.search(r"\d+", bodyline)
                    if actionre and bugre:
                        action = actionre.group()
                        bug = bugre.group()
                        ckbug = self.bugmap.get(bug, bug)

                        valid = self.skip_all or self._isRHELBug(ckbug, commit, summary)

                        if valid:
                            rhbz.add("%s: rhbz#%s" % (action, ckbug))

                            # Remove the summary bug's Resolves action if it is for the same bug
                            if action != 'Resolves':
                                summary_str = "Resolves: rhbz#%s" % summary_bug
                                if summary_bug and ckbug == summary_bug and summary_str in rhbz:
                                    rhbz.remove(summary_str)
                        else:
                            bad = True

                        if self.skip_all:
                            print "*** Bug %s Related commit %s is allowed\n" % (bug, commit)
                            continue

                        if valid and action == 'Resolves' and \
                           (not self._isRHELBugInCorrectState(ckbug, commit,
                                                              summary) or \
                            not self._isRHELBugFixedInVersion(ckbug, commit,
                                                              summary,
                                                              fixedIn) or \
                            not self._isRHELBugAcked(ckbug, commit, summary)):
                            bad = True
                        elif valid and action == 'Related':
                            # A related bug needs to have acks, and if it is the same as the summary
                            # It overrides the summary having different fixed-in or state
                            if self._isRHELBugAcked(ckbug, commit, summary):
                                print("*** Bug %s Related commit %s is allowed\n" % (bug, commit))
                                if ckbug == summary_bug:
                                    bad = False
                            else:
                                bad = True

                if len(rhbz) == 0 and not self.skip_all:
                    print("*** No bugs referenced in commit %s\n" % commit)
                    bad = True

                rpm_log.append((summary.strip(), list(rhbz)))
            else:
                rpm_log.append(("%s (%s)" % (summary.strip(), author), None))

            if bad:
                bad_bump = True

        if bad_bump:
            sys.exit(1)

        return rpm_log

    def _writeNewConfigure(self, newVersion):
        f = open(self.configure, 'r')
        l = f.readlines()
        f.close()

        i = l.index("AC_INIT([%s], [%s], [%s])\n" % (self.name,
                                                     self.version,
                                                     self.bugreport))
        l[i] = "AC_INIT([%s], [%s], [%s])\n" % (self.name,
                                                newVersion,
                                                self.bugreport)

        i = l.index("AC_SUBST(PACKAGE_RELEASE, [1])\n")
        l[i] = "AC_SUBST(PACKAGE_RELEASE, [%s])\n" % self.new_release

        f = open(self.configure, 'w')
        f.writelines(l)
        f.close()

    def _writeNewSpec(self, newVersion, rpmlog):
        f = open(self.spec, 'r')
        l = f.readlines()
        f.close()

        i = l.index('%changelog\n')
        top = l[:i]
        bottom = l[i+1:]

        f = open(self.spec, 'w')
        f.writelines(top)

        f.write("%changelog\n")
        today = datetime.date.today()
        stamp = today.strftime("%a %b %d %Y")
        f.write("* %s %s <%s> - %s-%s\n" % (stamp, self.gituser, self.gitemail,
                                            newVersion, self.new_release))

        for msg, rhbz in rpmlog:
            msg = re.sub('(?<!%)%%(?!%)|(?<!%%)%(?!%%)', '%%', msg)
            sublines = textwrap.wrap(msg, 77)
            f.write("- %s\n" % sublines[0])

            if len(sublines) > 1:
                for subline in sublines[1:]:
                    f.write("  %s\n" % subline)

            if rhbz:
                for entry in rhbz:
                    f.write("  %s\n" % entry)

        f.write("\n")
        f.writelines(bottom)
        f.close()

    def check_jenkins(self):
        if not self.git_branch:
            log.error("No git branch, cannot check jenkins")
            return False

        if not self.jenkins:
            log.warning("No jenkins defined, skipping")
            return True

        ret = False

        if self.jenkins_proxy:
            proxies = { "http": self.jenkins_proxy }
        else:
            proxies = {}

        try:
            with closing(urllib.urlopen(self.jenkins, proxies=proxies)) as f:
                contents = f.readlines()

                try:
                    j = json.loads(contents[0])
                except (TypeError, ValueError):
                    pass
                else:
                    # Get the name of the job we should be looking for in jenkins.
                    # This name is composed from the name of the project, plus the
                    # branch of the project without any "-branch" stuff at the end.
                    targetJob = self.name + "-" + self.git_branch
                    if targetJob.endswith("-branch"):
                        targetJob = targetJob[:-7]

                    for job in j["jobs"]:
                        if job["name"] == targetJob:
                            ret = job["color"] == "blue"
                            break
        except IOError as e:
            log.error("Jenkins check failed: %s", e)
            return False

        return ret

    def check_zanata(self):
        """
        Make sure that the zanata project-version matches the current git branch

        This is to prevent accidentally pushing translations to the wrong branch,
        eg. when branching for a new release and zanata.xml hasn't been updated
        """
        if not self.git_branch:
            log.error("No git branch, cannot check zanata config")
            return False

        version_re = re.compile("<project-version>(.*)</project-version>")
        ret = False
        with open(self.zanata_config, "r") as f:
            for line in f:
                m = version_re.match(line.strip())
                if m and m.group(1) == self.git_branch:
                    ret = True
                    break
                elif m:
                    log.error("zanata.xml branch (%s) does not match current branch: %s", m.group(1), self.git_branch)
                    break
            else:
                log.error("zanata.xml does not have a project-version")

        return ret

    def run(self):
        if not self.skip_zanata and not self.check_zanata():
            sys.exit(1)

        # For now, jenkins is in an advisory role so do not require it to pass.
        if not self.skip_jenkins and not self.check_jenkins():
            log.warning("jenkins test results do not pass; ignoring for now")

        newVersion = self._incrementVersion()
        fixedIn = "%s-%s-%s" % (self.name, newVersion, self.new_release)
        rpmlog = self._rpmLog(fixedIn)

        self._writeNewConfigure(newVersion)
        self._writeNewSpec(newVersion, rpmlog)

def usage(cmd):
    sys.stdout.write("Usage: %s [OPTION]...\n" % (cmd,))
    sys.stdout.write("Options:\n")
    sys.stdout.write("    -n, --name       Package name.\n")
    sys.stdout.write("    -v, --version    Current package version number.\n")
    sys.stdout.write("    -r, --release    Package release number.\n")
    sys.stdout.write("        --newrelease Value for release in the .spec file.\n")
    sys.stdout.write("    -b, --bugreport  Bug reporting email address.\n")
    sys.stdout.write("    -i, --ignore     Comma separated list of git commits to ignore.\n")
    sys.stdout.write("    -m, --map        Comma separated list of FEDORA_BZ=RHEL_BZ mappings.\n")
    sys.stdout.write("    -s, --skip-acks  Skip checking for rhel-X.X.X ack flag\n")
    sys.stdout.write("    -S, --skip-all   Skip all checks\n")
    sys.stdout.write("    -d, --debug      Turn on debug logging to stdout\n")
    sys.stdout.write("    --skip-zanata    Skip checking Zanata config for branch name\n")
    sys.stdout.write("    --skip-jenkins   Skip checking Jenkins for test results\n")
    sys.stdout.write("\nThe -i switch is intended for use with utility commits that we do not need to\n")
    sys.stdout.write("reference in the spec file changelog.  The -m switch is used to map a Fedora\n")
    sys.stdout.write("BZ number to a RHEL BZ number for the spec file changelog.  Use -m if you have\n")
    sys.stdout.write("a commit that needs to reference a RHEL bug and have cloned the bug, but the\n")
    sys.stdout.write("original commit was already pushed to the central repo.\n")

def main(argv):
    prog = os.path.basename(sys.argv[0])
    cwd = os.getcwd()
    configure = os.path.realpath(cwd + '/configure.ac')
    spec = os.path.realpath(cwd + '/anaconda.spec.in')
    zanata_config = os.path.realpath(cwd + '/zanata.xml')
    name, version, release, new_release, bugreport = None, None, None, None, None
    ignore, bugmap = None, None
    show_help, unknown, skip_acks, skip_all, skip_zanata, skip_jenkins = False, False, False, False, False, False
    opts = []

    try:
        opts, _args = getopt.getopt(sys.argv[1:], 'n:v:r:b:i:m:sSd?',
                                   ['name=', 'version=', 'release=', "newrelease=",
                                    'bugreport=', 'ignore=', 'map=',
                                    'debug', 'help', 'skip-zanata', 'skip-jenkins'])
    except getopt.GetoptError:
        show_help = True

    for o, a in opts:
        if o in ('-n', '--name'):
            name = a
        elif o in ('-v', '--version'):
            version = a
        elif o in ('-r', '--release'):
            release = a
        elif o in ('--newrelease'):
            new_release = a
        elif o in ('-b', '--bugreport'):
            bugreport = a
        elif o in ('-i', '--ignore'):
            ignore = a
        elif o in ('-m', '--map'):
            bugmap = a
        elif o in ('-s', '--skip-acks'):
            skip_acks = True
        elif o in ('-S', '--skip-all'):
            skip_all = True
        elif o in ('-d', '--debug'):
            log.setLevel(logging.DEBUG)
        elif o in ('--skip-zanata'):
            skip_zanata = True
        elif o in ('--skip-jenkins'):
            skip_jenkins = True
        elif o in ('-?', '--help'):
            show_help = True
        else:
            unknown = True

    if show_help:
        usage(prog)
        sys.exit(0)
    elif unknown:
        sys.stderr.write("%s: extra operand `%s'" % (prog, sys.argv[1],))
        sys.stderr.write("Try `%s --help' for more information." % (prog,))
        sys.exit(1)

    if not name:
        sys.stderr.write("Missing required -n/--name option\n")
        sys.exit(1)

    if not version:
        sys.stderr.write("Missing required -v/--version option\n")
        sys.exit(1)

    if not release:
        sys.stderr.write("Missing required -r/--release option\n")
        sys.exit(1)

    if not bugreport:
        sys.stderr.write("Missing required -b/--bugreport option\n")
        sys.exit(1)

    if not os.path.isfile(configure) and not os.path.isfile(spec):
        sys.stderr.write("You must be at the top level of the anaconda source tree.\n")
        sys.exit(1)

    mbv = MakeBumpVer(name=name, version=version, release=release,
                      bugreport=bugreport, ignore=ignore, bugmap=bugmap,
                      configure=configure, spec=spec, skip_acks=skip_acks,
                      skip_all=skip_all, zanata_config=zanata_config, skip_zanata=skip_zanata,
                      skip_jenkins=skip_jenkins, new_release=new_release)
    mbv.run()

if __name__ == "__main__":
    main(sys.argv)