ipatool

#!/usr/bin/python3

"""Tool helping the IPA project processes

Usage:
  ipatool --help
  ipatool [options] [-v...] sample-config
  ipatool [options] [-v...] push [--branch=BRANCH...] [--reviewer=NAME...] [--] [PATCH ...]
  ipatool [options] [-v...] start-review [-f] [--am] [--ticket=NUMBER...] [--] [PATCH ...]
  ipatool [options] [-v...] am [--] [PATCH ...]
  ipatool [options] [-v...] pr-ack PR_ID [--comment=TEXT]
  ipatool [options] [-v...] pr-list [--state=(open|closed|all)]... [--label=NAME...]
  ipatool [options] [-v...] pr-push PR_ID [--reviewer=NAME...] [--backport=BRANCH...] [--autobackport]
  ipatool [options] [-v...] pr-reject PR_ID --comment=TEXT
  ipatool [options] [-v...] backport PR_ID --branch=BRANCH...

Common Options:
  -h, --help           Display this help and exit
  -v, --verbose        Increase verbosity
  --config FILE        Configuration file [default: ~/.ipa/toolconf.yaml]
  --no-reviewer        Do not add a Reviewed-By: line
  -n, --dry-run        Do not push
  --no-pagure          Do not contact Pagure.io
  --no-fetch           Do not synchronize before pushing
  --color=(auto|always|never)  Colorize output [default: auto]
  PATCH                Patch to push, or directory with *.patch files

Configuration is specified in the file given by --config.
Run ipatool sample-config to see what needs to go in it.

ipatool push:
  The given patches are applied on top of the given upstream branches,
  and pushed upstream.
  A "Reviewed-By" line with the name(s) given by --reviewer is added
  to all patches unless --no-reviewer is given).
  If neither --reviewer nor --no-reviewer is given, a reviewer is looked up in
  the "Reviewed by" field of the Pagure ticket(s). If that does not yield one
  reviewer, the pushpatches command fails.
  If the reviewer name is not in the
  form "Name Last <mail@address.example>", it is looked up in the contributors
  as listed in `git shortlog -se`.

  -b, --branch=BRANCH  Branch to push to (detected from ticket if not given here)
  -r, --reviewer=NAME  Reviewer(s) of the patches

ipatool start-review:
  Sets yourself as the reviewer for given tickets, and also adds you to CC.
  Tickets may be mentioned by number with the --ticket option,
  and are searched in any patches given on the command line.
  If no tickets and no patches are specified, all patches from the configured
  patchdir are searched.

  -t, --ticket NUMBER  Number of ticket to update
  -f, --force          Force setting reviewer even if already set
  --am                 Also apply the patches as in `ipatool am`

ipatool am:
  Applies the given patches to the working tree, using a configured command.
  Uses the configured patchdir if no patches are given.

ipatool pr-list:
  List pull requests. By default shows all opened pull requests.

  -s, --state (open|closed|all)  List pull requests in given state
  -l, --label NAME               List pull requests with given label

ipatool pr-ack:
  ACK pull request. Add `ack` label and comment.

ipatool pr-reject:
  Reject pull request. Add `rejected` label, comment with reject reason. Also
  close the pull request.

  -c, --comment TEXT    Reason for ACKing the PR

iptool pr-push:
  Fetch all patches from pull request, store them in `patchdir` and call push.

  -B, --backport BRANCH Rebase patches from PR and open new PR against BRANCH
  --autobackport        Automatically backport to branches indicated by PR labels

ipatool backport:
  Fetch all patches from a pull request, store them in `patchdir` and backport
  them to a specific branch. PR can be in pushed and closed state but it has
  to be 'ack'ed.

"""

SAMPLE_CONFIG = """
# Local "clean" repository
clean-repo-path: ~/dev/freeipa-clean
remote: origin

# Default directory where patches to push are stored
patchdir: ~/patches/to-apply

# URLs to use in reports & messages
ticket-url: https://pagure.io/freeipa/issue/
commit-url: https://pagure.io/freeipa/c/
bugzilla-bug-url: https://bugzilla.redhat.com/show_bug.cgi?id=
jira-ticket-url: https://issues.redhat.com/browse/RHEL-

# Pagure login details
pagure-repository: freeipa
# Create the token in https://pagure.io/freeipa/settings
# For token you need:
#   * Assign issue to someone
#   * Change the status of a ticket
#   * Comment on a ticket
#   * Create a new ticket
#   * Subscribe the user with this token to an issue
#   * Update an issue, status, comments, custom fields...
#   * Update the custom fields of an issue
#   * Update the milestone of an issue
pagure-token: "0123456789abcdef0123456789abcdef01234567"
# workaround: pagure doesn't provide the tokens for users so we cannot
# dynamically detect username
username: username

# Pagure issues operations
# update-issue options: yes/no/ask
update-issue: ask
# close-issue options: no/ask
close-issue: ask

# Mapping of logins to Git-style author lines
trac-username-map:
    abbra: Alexander Bokovoy <abokovoy@redhat.com>

# Command to run "git am" on the development tree (as argv list)
am-command: ["ssh", "ipa-devel-vm.local", "cd ~/freeipa/ ; git am -3"]

# Currently unused :(
browser: firefox

# GitHub configuration
# for the token, we require at least the 'repo', 'admin:org' group permissions
# and the 'user:email' and 'read:user' permissions
gh-token: "0123456789abcdef0123456789abcdef01234567"
gh-repo: "freeipa/freeipa"
gh-fork-remote: "mygh"

# To create gh-fork-remote do the following:
#
# NOTE: also set clean-repo-path to ~/redhat/freeipa-clean or
#       the path to your checkout.
#
# $ git clone ssh://git@pagure.io/freeipa.git ~/redhat/freeipa-clean
# $ cd ~/redhat/freeipa-clean
# $ git remote add mygh git@github.com:<yourname>/freeipa.git
#
# $ git remote -v
# mygh    git@github.com:<yourname>/freeipa.git (fetch)
# mygh    git@github.com:<yourname>/freeipa.git (push)
# origin  ssh://git@pagure.io/freeipa.git (fetch)
# origin  ssh://git@pagure.io/freeipa.git (push)
"""

import glob
import sys
import os
import string
import subprocess
import re
import collections
import pprint
from itertools import groupby

import yaml       # yum install python3-PyYAML
import blessings  # yum install python3-blessings
import github3    # yum install python3-github3py
import unidecode  # yum install python3-unidecode
import docopt     # yum install python3-docopt
import xtermcolor # yum install python3-xtermcolor
import libpagure  # yum install python3-libpagure


MILESTONES = {
    r"^FreeIPA 3\.3\..*": ['master', 'ipa-4-1', 'ipa-4-0', 'ipa-3-3'],
    r"^FreeIPA 4\.4.*": ['master', 'ipa-4-5', 'ipa-4-4'],
    r"^FreeIPA 4\.5.*": ['master', 'ipa-4-5'],
    r"^FreeIPA 4\.6.*": ['master'],
    r"^FreeIPA 4\.7.*": ['master'],
}

COLOR_OPT_MAP = {'auto': False, 'always': True, 'never': None}

SUBJECT_RE = re.compile('^Subject:( *\[PATCH( [^]*])?\])*(?P<subj>.*)')

GIT_REMOTE_SERVER = 'pagure.io'

SubprocessResult = collections.namedtuple(
    'SubprocessResult', 'stdout stderr returncode')

def cleanpath(path):
    """Return absolute path with leading ~ expanded"""
    path = os.path.expanduser(path)
    path = os.path.abspath(path)
    return path


def shellquote(arg):
    """Quote an argument for the shell"""
    if re.match('^[-_.:/=a-zA-Z0-9]*$', arg):
        return arg
    else:
        return "'%s'" % arg.replace("'", r"'\''")


class Patch(object):
    """Represents a sanitized patch

    - Removes ">" from From lines in the metadata/message
    - Adds a Reviewed-By tag

    Attributes:
    * subject - name of the patch
    * lines - iterator of lines with the patch
    * ticket_numbers - numbers of referenced tickets
    """
    def __init__(self, config, filename):
        if filename:
            self.filename = filename
            with open(filename) as file:
                lines = list(file)
        else:
            self.filename = '(patch)'
        assert lines
        lines_iter = iter(lines)
        self.head_lines = []
        self.patch_lines = []
        in_subject = False
        self.subject = ''
        for line in lines_iter:
            if not line.startswith(' '):
                in_subject = False
            if in_subject:
                self.subject += line.rstrip()
            match = SUBJECT_RE.match(line)
            if match:
                self.subject = match.group('subj').strip()
                in_subject = True

            if line.startswith('>From'):
                self.head_lines.append(line[1:])
            elif any([
                    line == '---\n',
                    line.startswith('diff -'),
                    line.startswith('Index: ')]):
                self.patch_lines.append(line)
                break
            else:
                self.head_lines.append(line)
        self.patch_lines.extend(lines_iter)

        self.ticket_numbers = []
        for line in self.lines:
            if line.startswith('-') or line.startswith(' '):
                # ignore ticket links in removed or context diff lines
                continue
            regex = '%s(\d+)' % re.escape(config['ticket-url'])
            for match in re.finditer(regex, line):
                self.ticket_numbers.append(int(match.group(1)))

    def add_reviewer(self, reviewer):
        if not re.match('^[-_a-zA-Z0-9]+: .*$', self.head_lines[-1]):
            self.head_lines.append('\n')
        self.head_lines.append('Reviewed-By: %s\n' % reviewer)

    @property
    def lines(self):
        yield from self.head_lines
        yield from self.patch_lines


class Ticket(object):
    """Pagure ticket with lazily fetched information"""
    def __init__(self, pagure, number):
        self.pagure = pagure
        self.number = number
        self._data = None

    def get_custom_data(self, name, default=None):
        for field in self.data.get('custom_fields', ()):
            if field['name'] == name:
                return field['value']
        return default

    def is_closed(self):
        return self.status == 'Closed'

    def is_fixed(self):
        return self.is_closed() and self.close_status == 'fixed'

    @property
    def data(self):
        if self._data is not None:
            return self._data
        print('Retrieving ticket %s' % self.number)
        self._data = self.pagure.issue_info(self.number)
        return self._data

    @property
    def reviewer(self):
        return self.get_custom_data('reviewer')

    @property
    def rhbz(self):
        return self.get_custom_data('rhbz')

    @property
    def milestone(self):
        return self.data['milestone']

    @property
    def summary(self):
        return self.data['content']

    @property
    def title(self):
        return self.data['title']

    @property
    def status(self):
        return self.data['status']

    @property
    def close_status(self):
        return self.data['close_status']


class Context(object):
    """Holds options, configuration, and helpers for a tool"""
    def __init__(self, options):
        self.options = options
        self.config = None
        try:
            with open(os.path.expanduser(options['--config'])) as conf_file:
                self.config = yaml.safe_load(conf_file)
        except:
            pass
        self.term = blessings.Terminal(
            force_styling=COLOR_OPT_MAP[options['--color']])
        self.verbosity = self.options['--verbose']
        if self.verbosity:
            print('Options:')
            pprint.pprint(self.options)
            print('Config:')
            self.print_sanitized_config()
        if self.options['--no-pagure'] or self.config == None:
            self.pagure = None
        else:
            try:
                self.pagure = libpagure.Pagure(
                    pagure_token=self.config['pagure-token'],
                    pagure_repository=self.config['pagure-repository']
                )
            except TypeError:
                self.pagure = libpagure.Pagure(
                    pagure_token=self.config['pagure-token'],
                    repo_to=self.config['pagure-repository']
                )


        self.color_arg = self.options['--color']
        if self.color_arg == 'auto':
            if self.term.is_a_tty:
                self.color_arg = 'always'
            else:
                self.color_arg = 'never'

        # ipatool sets GIT_COMMITTER_DATE to a fixed value, so
        # commits to parallel branches hopefully end up identical.
        # Getting the current date with timezone info is a pain in Python
        # (needs extra dependencies), so just run date
        self.isodate_now = ''
        date_result = self.runprocess(['date', '-Iseconds'],
                                      env={'GIT_COMMIT_DATE': ''})
        self.isodate_now = date_result.stdout.strip()

    def print_sanitized_config(self):
        # prints the config dictionary with sensitive informations starred
        sensitives = ('gh-token', 'pagure-token')
        pprint.pprint(dict((attr, val)
                        if attr not in sensitives else (attr, '***')
                        for attr, val in self.config.items()))

    commands = {}
    @classmethod
    def command(cls, name):
        def decorator(func):
            cls.commands[name] = func
            return func
        return decorator

    def run(self):
        for name, func in self.commands.items():
            if self.options[name]:
                return func(self)
        else:
            print('Registered commands: %s' % ', '.join(self.commands))
            self.die('Internal error: No command found')

    def die(self, message):
        print(self.term.red(message))
        exit(1)

    def get_patches(self):
        paths = self.options['PATCH'] or [self.config['patchdir']]
        for path in paths:
            path = cleanpath(path)
            if os.path.isdir(path):
                filenames = glob.glob(os.path.join(path, '*.patch'))
                for filename in sorted(filenames):
                    yield Patch(self.config, filename)
            else:
                yield Patch(self.config, path)

    def runprocess(self, argv, check_stdout=None, check_stderr=None,
                   check_returncode=0, stdin_string='', fail_message=None,
                   timeout=5, verbosity=None, env=None):
        """Run a command in a subprocess, check & return result"""
        if env is None:
            env = os.environ
        env.setdefault('GIT_COMMITTER_DATE', self.isodate_now)
        argv_repr = ' '.join(shellquote(a) for a in argv)
        if verbosity is None:
            verbosity = self.verbosity
        if verbosity:
            print(self.term.blue(argv_repr))
        if verbosity > 2:
            print(self.term.yellow(stdin_string.rstrip()))
        PIPE = subprocess.PIPE
        proc = subprocess.Popen(argv, stdout=PIPE, stderr=PIPE, stdin=PIPE,
                                env=env)
        try:
            stdout, stderr = proc.communicate(stdin_string.encode('utf-8'),
                                              timeout=timeout)
            timeout_expired = False
        except subprocess.TimeoutExpired:
            proc.kill()
            stdout = stderr = b''
            timeout_expired = True
        stdout = stdout.decode('utf-8')
        stderr = stderr.decode('utf-8')
        returncode = proc.returncode
        failed = any([
            timeout_expired,
            (check_stdout is not None and check_stdout != stdout),
            (check_stderr is not None and check_stderr != stderr),
            (check_returncode is not None and check_returncode != returncode),
        ])
        if failed and not verbosity:
            print(self.term.blue(argv_repr))
        if failed or verbosity >= 2:
            if stdout:
                print(stdout.rstrip())
            if stderr:
                print(self.term.yellow(stderr.rstrip()))
            print('→ %s' % self.term.blue(str(proc.returncode)))
        if failed:
            if timeout_expired:
                self.die('Command timeout expired')
            elif fail_message:
                self.die(fail_message)
            else:
                self.die('Command failed')
        return SubprocessResult(stdout, stderr, returncode)

@Context.command('sample-config')
def sample_config_command(ctx):
    print(ctx.term.cyan('Copy the following to %s, and modify to taste:' %
                        ctx.options['--config']),
          file=sys.stderr)
    print(ctx.term.cyan('---8<---'.ljust(70, '-')), file=sys.stderr)
    print(SAMPLE_CONFIG.strip())
    print(ctx.term.cyan('--->8---'.rjust(70, '-')), file=sys.stderr)

def ensure_clean_repo(ctx):
    """Make sure the working tree matches the git index"""
    ctx.runprocess(['git', 'status', '--porcelain'],
                   check_stdout='',
                   check_stderr='',
                   fail_message='Repository %s not clean' % os.getcwd())

def get_reviewers(ctx, tickets):
    """Get name & address of reviewers, or empty list for --no-reviewer

    Raises if a suitable reviewer is not found
    """
    if ctx.options['--no-reviewer']:
        return []
    reviewers = ctx.options['--reviewer']
    if ctx.pagure and not reviewers:
        reviewers = set()
        for ticket in tickets:
            if ticket.reviewer:
                reviewers.add(ticket.reviewer)
        if len(reviewers) > 1:
            print('Reviewers found: %s' % ', '.join(reviewers))
            ctx.die('Too many reviewers found in ticket(s), '
                        'specify --reviewer explicitly')
        if not reviewers:
            ctx.die('No reviewer found in ticket(s), '
                        'specify --reviewer explicitly')
        username_map = ctx.config.get('trac-username-map', {})
        reviewers = [username_map.get(r, r) for r in reviewers]
    if not reviewers:
        ctx.die('No reviewer found, please specify --reviewer')
    return [normalize_reviewer(ctx, r) for r in reviewers]

def normalize_reviewer(ctx, reviewer):
    """Expand a partial reviewer name to a full name + address

    Uses the list of contributors from git's mailmap
    """
    name_re = re.compile(r'^\w+ [^<]+ <.*@.*\..*>$')
    if name_re.match(reviewer):
        return reviewer
    rbranch = '%s/master' % ctx.config['remote']
    cmd = ['git',
           '-c', 'mailmap.blob=origin/master:.mailmap',
           'shortlog', '-sen', rbranch]
    names = ctx.runprocess(cmd).stdout.splitlines()
    names = (name.split('\t', 1)[-1] for name in names)
    names = (name for name in names if name_re.match(name))
    names = [name for name in names if reviewer.lower() in name.lower()]
    if not names:
        ctx.die('Reviewer %s not found' % reviewer)
    elif len(names) > 1:
        print(ctx.term.red('Reviewer %s could be:' % reviewer))
        for name in names:
            print('- %s' % name)
        ctx.die('Multiple matches found for reviewer')
    else:
        name = unidecode.unidecode(names[0])
        return name

def apply_patches(ctx, patches, branch, die_on_fail=True):
    """Apply patches to the given branch

    Checks out the branch
    """
    ctx.runprocess(['git', 'checkout',
                    '%s/%s' % (ctx.config['remote'], branch)])
    for patch in patches:
        print('Applying to %s: %s' % (branch, patch.subject))
        res = ctx.runprocess(
            ['git', 'am', '--3way'],
            stdin_string=''.join(patch.lines),
            check_returncode=0 if die_on_fail else None,
        )
        if not die_on_fail and res.returncode:
            raise RuntimeError(res.stderr)
    sha1 = ctx.runprocess(['git', 'rev-parse', 'HEAD']).stdout.strip()
    if ctx.verbosity:
        print('Resulting hash: %s' % sha1)
    return sha1

def print_push_info(ctx, patches, sha1s, ticket_numbers, tickets):
    """Print lots of info about the to-be-pushed commits"""
    remote = ctx.config['remote']
    branches = sha1s.keys()

    ctx.push_info = {}
    pagure_log = []
    bugzilla_log = ['Fixed upstream']
    for branch in branches:
        pagure_log.append('%s:\n' % branch)  # we need extra newline for pagure
        bugzilla_log.append('%s:' % branch)
        log_result = ctx.runprocess(
            ['git', 'log', '--graph', '--oneline', '--abbrev=99',
             '--color=never', '%s/%s..%s' % (remote, branch, sha1s[branch])])
        pagure_log.extend(
            line.rstrip()
            for line in reversed(log_result.stdout.splitlines()))
        pagure_log.append('\n')  # add newline to fix github/pagure formatting

        log_result = ctx.runprocess(
            ['git', 'log', '--pretty=format:%H',
             '%s/%s..%s' % (remote, branch, sha1s[branch])])
        bugzilla_log.extend(
            ctx.config['commit-url'] + line.strip()
            for line in reversed(log_result.stdout.splitlines()))

    bugzilla_urls = []
    bugzilla_re = re.compile('(%s\d+)' %
                                re.escape(ctx.config['bugzilla-bug-url']))
    jira_urls = []
    jira_re = re.compile('(%s\d+)' % re.escape(ctx.config['jira-ticket-url']))

    for ticket in tickets:
        if ticket.rhbz:
            for match in bugzilla_re.finditer(ticket.rhbz):
                bugzilla_urls.append(match.group(0))
            for match in jira_re.finditer(ticket.rhbz):
                jira_urls.append(match.group(0))

    for branch in branches:
        print(ctx.term.cyan('=== Diffstat for %s ===' % branch))
        log_result = ctx.runprocess(
            ['git', 'diff', '--stat', '--color=%s' % ctx.color_arg,
             '%s/%s..%s' % (remote, branch, sha1s[branch])],
            verbosity=2)
        print(ctx.term.cyan('=== Log for %s ===' % branch))
        log_result = ctx.runprocess(
            ['git', 'log', '--reverse', '--color=%s' % ctx.color_arg,
             '%s/%s..%s' % (remote, branch, sha1s[branch])],
            verbosity=2)

    print(ctx.term.cyan('=== Patches pushed ==='))
    for patch in patches:
        print(patch.filename)

    print(ctx.term.cyan('=== Mail summary ==='))
    if len(branches) == 1:
        print('Pushed to ', end='')
    else:
        print('Pushed to:')
    for branch in branches:
        print('%s: %s' % (branch, sha1s[branch]))

    print(ctx.term.cyan('=== Ticket comment ==='))
    pagure_msg = '\n'.join(pagure_log)
    print(pagure_msg)
    ctx.push_info['pagure_comment'] = pagure_msg

    print(ctx.term.cyan('=== Bugzilla/JIRA comment ==='))
    bugzilla_msg = '\n'.join(bugzilla_log)
    print(bugzilla_msg)
    ctx.push_info['bugzilla_comment'] = bugzilla_msg

    if ticket_numbers:
        print(ctx.term.cyan('=== Tickets fixed ==='))
        for number in sorted(ticket_numbers):
            print('%s%s' % (ctx.config['ticket-url'], number))

    if bugzilla_urls:
        print(ctx.term.cyan('=== Bugzillas fixed ==='))
        print('\n'.join(bugzilla_urls))
    
    if jira_urls:
        print(ctx.term.cyan('=== Jira tickets fixed ==='))
        print('\n'.join(jira_urls))

    print(ctx.term.cyan('=== Ready to push ==='))


def _update_issue(ctx, ticket):
    update_issue = ctx.config.get('update-issue', 'ask')
    if update_issue == 'no':
        do_comment = False
    elif update_issue == 'yes':
        do_comment = True
    else:
        if update_issue != 'ask':
            print(ctx.term.red(
                'Invalid value for "update-issue" in config file'))
        response = input(
            'Update issue "#{}: {}" with commit info? [y/n] '.format(
                ticket.number,
                ticket.title))
        if response.lower() == 'y':
            do_comment = True
        else:
            do_comment = False

    if do_comment:
        try:
            ctx.pagure.comment_issue(ticket.number,
                ctx.push_info['pagure_comment'])
        except Exception as e:
            print(ctx.term.red('Comment failed: {}'.format(e)))
            print(ctx.term.yellow('Please update issue manually'))
        else:
            print(ctx.term.green('Comment added'))

def verify_remote_url(ctx):
    remote = ctx.config['remote']
    stdout = ctx.runprocess(['git', 'remote', 'get-url', '--push', remote]).stdout
    remote_url = stdout.splitlines()[0]

    if GIT_REMOTE_SERVER not in remote_url:
        print(ctx.term.red(
            '!!! WARNING !!! not pushing to {} git repo').format(
                GIT_REMOTE_SERVER))
        response = input(
            'Push to "{}"? [y/n] '.format(
                remote_url))
        if response.lower() != 'y':
            ctx.die('Aborting push')

def _close_issue(ctx, ticket):
    if ticket.is_closed():
        # nothing to do
        print("Issue already closed.")
        return
    if ctx.options.get('--backport') or ctx.options.get('--autobackport'):
        # Backports must be pushed, thus ticket must remain opened.
        return

    close_ticket = ctx.config.get('close-issue', 'ask')
    if close_ticket == 'no':
        do_close = False
    else:
        if close_ticket != 'ask':
            print(ctx.term.red(
                'Invalid value for "close-issue" in config file'))
        response = input(
            'Close issue "#{}: {}"? [y/n] '.format(
                ticket.number,
                ticket.title))
        if response.lower() == 'y':
            do_close = True
        else:
            do_close = False

    if do_close:
        try:
            # workaround https://pagure.io/libpagure/issue/22
            try:
                ctx.pagure.change_issue_status(
                    ticket.number,
                    new_status='Closed',
                    close_status='fixed'
                )
            except AttributeError:
                updated_ticket = Ticket(ctx.pagure, ticket.number)
                if not updated_ticket.is_fixed():
                    raise
        except Exception as e:
            print(ctx.term.red('Failed to close the issue: {}'.format(e)))
            print(ctx.term.yellow('Please close the issue manually'))
        else:
            print(ctx.term.green('Issue closed'))


@Context.command('push')
def push_command(ctx):
    patches = list(ctx.get_patches())
    if not patches:
        ctx.die('No patches to push')

    os.chdir(cleanpath(ctx.config['clean-repo-path']))
    ensure_clean_repo(ctx)

    ticket_numbers = set()
    for patch in patches:
        ticket_numbers.update(patch.ticket_numbers)
    if ctx.pagure:
        tickets = [Ticket(ctx.pagure, n) for n in ticket_numbers]
    else:
        tickets = []

    reviewers = get_reviewers(ctx, tickets)
    if reviewers:
        for reviewer in reviewers:
            print('Reviewer: %s' % reviewer)
            for patch in patches:
                patch.add_reviewer(reviewer)
    else:
        print('No reviewer')

    branches = ctx.options['--branch']
    if not branches:
        if not tickets:
            if ctx.pagure:
                ctx.die('No branches specified and no tickets found')
            else:
                ctx.die('No branches specified and Pagure disabled')
        if ctx.verbosity:
            print('Divining branches from tickets: %s' %
                    ', '.join(str(t.number) for t in tickets))
        milestones = set(t.milestone for t in tickets)
        if not milestones:
            ctx.die('No milestones found in tickets')
        elif len(milestones) > 1:
            ctx.die('Tickets belong to disparate milestones; '
                        'fix them in Pagure or specify branches explicitly')
        [milestone] = milestones
        for template, templ_branches in MILESTONES.items():
            if re.match(template, milestone):
                branches = templ_branches
                break
        else:
            ctx.die('No branches correspond to `%s`. ' % milestone +
                        'Update MILESTONES in the ipatool script.')
    print('Will apply %s patches to: %s' %
            (len(patches), ', '.join(branches)))

    remote = ctx.config['remote']
    verify_remote_url(ctx)

    if not ctx.options['--no-fetch']:
        print('Fetching...')
        ctx.runprocess(['git', 'fetch', remote], timeout=60)

    rev_parse = ctx.runprocess(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
    old_branch = rev_parse.stdout.strip()
    if ctx.verbosity:
        print('Old branch: %s' % old_branch)
    try:
        sha1s = collections.OrderedDict()
        for branch in branches:
            sha1s[branch] = apply_patches(ctx, patches, branch)

        push_args = ['%s:%s' % (sha1, branch)
                        for branch, sha1 in sha1s.items()]
        print('Trying push...')
        ctx.runprocess(['git', 'push', '--dry-run', remote] + push_args,
                       timeout=60, verbosity=2)

        print('Generating info...')
        print_push_info(ctx, patches, sha1s, ticket_numbers, tickets)

        if ctx.options['--dry-run']:
            print('Exiting, --dry-run specified')
            ctx.push_info['pushed'] = False
        else:
            while True:
                print('(k will start `gitk`)')
                branchesrepr = ', '.join(branches)
                response = input('Push to %s? [y/n/k] ' % branchesrepr)
                if response.lower() == 'n':
                    break
                elif response.lower() == 'k':
                    ctx.runprocess(['gitk'] +
                                    branches +
                                    list(sha1s.values()),
                                   timeout=None)
                elif response.lower() == 'y':
                    print('Pushing')
                    ctx.runprocess(['git', 'push', remote] + push_args,
                                   timeout=60, verbosity=2)
                    break

            if response.lower() == 'y':  # was pushed
                ctx.push_info['pushed'] = True
            else:
                ctx.push_info['pushed'] = False

    finally:
        print('Cleaning up')
        ctx.runprocess(['git', 'am', '--abort'], check_returncode=None)
        ctx.runprocess(['git', 'reset', '--hard'], check_returncode=None)
        ctx.runprocess(['git', 'checkout', old_branch], check_returncode=None)
        ctx.runprocess(['git', 'clean', '-fxd'], check_returncode=None)

    if ctx.push_info['pushed']:
        for ticket in tickets:
            _update_issue(ctx, ticket)
            _close_issue(ctx, ticket)



def gh_repo(token, repo_full_name):
    """

    token - GitHub Personal access token
    repo_full_name GitHub repository identifier, owner/repo
    """
    (owner, repo,) = repo_full_name.split('/')

    gh = github3.login(token=token)
    return gh.repository(owner, repo)


def get_gh_repo(ctx):
    try:
        token = ctx.config['gh-token']
        repo_full_name = ctx.config['gh-repo']
    except KeyError:
        ctx.die("'gh-token' and/or 'gh-repo' is not set in config file")

    try:
        return gh_repo(token, repo_full_name)
    except Exception as e:
        ctx.die("Failed to access GitHub repository. Check 'gh-token' and "
                "'gh-repo' variables in config file.")


def labels_names(labels):
    return [l.name for l in labels]


def labels_colorize(labels):
    return [xtermcolor.colorize(l.name, rgb=int(l.color, 16)) for l in labels]


def patch_filename(msg, num):
    allowed = string.ascii_letters + string.digits + '-_'
    title = msg.split('\n\n', 1)[0]
    title = title.replace(' ', '-')
    title = ''.join(c for c in title if c in allowed)
    return '{:04}-{}.patch'.format(num, title)


def delete_patches(d, ignore_errors=True):
    for patch in os.listdir(d):
        try:
            os.remove(os.path.join(d,patch))
        except Exception:
            if not ignore_errors:
                raise


def sorted_commits(commits):
    # no merges! no magic!
    for c in commits:
        assert len(c.parents) == 1

    commit_ids = [c.sha for c in commits]

    # find first
    for c in commits:
        if c.parents[0]['sha'] not in commit_ids:
            result = [c]
            break
    else:
        raise RuntimeError("No first commit? There should be always one.")

    parent_id = c.sha
    while len(result) < len(commits):
        for c in commits:
            if c.parents[0]['sha'] == parent_id:
                result.append(c)
                parent_id = c.sha
                break  # end for loop
        else:
            raise RuntimeError(
                "Commit {} should have child but none found".format(parent_id)
            )

    return result


def backport(ctx, backport_branches, repo, pr):
    try:
        ctx.config['remote']
        ctx.config['gh-fork-remote']
        ctx.config['gh-token']
        ctx.config['clean-repo-path']
    except KeyError as exc:
        ctx.die("%s is not set in configuration file. Backport must be "
                "handled manually." % exc)

    patches = list(ctx.get_patches())
    os.chdir(cleanpath(ctx.config['clean-repo-path']))
    try:
        github_login = github3.login(token=ctx.config['gh-token']).me().login
    except KeyError as e:
        print(ctx.term.red('Github failure response: {}'.format(e)))
        return

    rev_parse = ctx.runprocess(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])
    old_branch = rev_parse.stdout.strip()

    for bb in backport_branches:
        try:
            res = ctx.runprocess(
                ['git', 'checkout', '%s/%s' % (ctx.config['remote'], bb)],
                check_returncode=None
            )
            if res.returncode:
                print(ctx.term.red(
                    "Failed to checkout %s/%s. Manual backport is needed. %s"
                    % (ctx.config['remote'], bb, res.stderr)
                ))
                continue

            try:
                sha = apply_patches(ctx, patches, bb, die_on_fail=False)
            except RuntimeError as e:
                print(ctx.term.red('Failure to apply patches: {}'.format(e)))
                print(ctx.term.red(
                    "Failed to apply patches onto %s/%s. Manual backport is "
                    "needed." % (ctx.config['remote'], bb)
                ))
                if ctx.verbosity:
                    print(ctx.term.red(res.stderr))
                continue
            print(
                "Applied patches on %s/%s" % (ctx.config['remote'], bb)
            )

            backport_name = 'backport_pr%d_%s' % (pr.number, bb)
            res = ctx.runprocess(
                ['git', 'push', ctx.config['gh-fork-remote'],
                 "%s:refs/heads/%s" % (sha, backport_name)],
                check_returncode=None, timeout=60,
            )
            if res.returncode:
                print(ctx.term.red(
                    "Failed to push %s to %s/%s"
                    % (sha, ctx.config['gh-fork-remote'], backport_name)
                ))
                continue

            print(
                "Pushed %s to %s/%s"
                % (sha, ctx.config['gh-fork-remote'], backport_name)
            )
            backport_pr = repo.create_pull(
                title="[Backport][%s] %s" % (bb, pr.title),
                base=bb,
                head="%s:%s" % (github_login, backport_name),
                body="This PR was opened automatically because PR #%d was "
                     "pushed to %s and backport to %s is required."
                     % (pr.number, pr.base.ref, bb),
            )
            backport_issue = backport_pr.issue()
            backport_issue.add_labels('ack')
            backport_issue.create_comment(
                'PR was ACKed automatically because this is backport of PR '
                '#%d. Wait for CI to finish before pushing. In case of '
                'questions or problems contact @%s who is author of the '
                'original PR.' % (pr.number, pr.user.login)
            )
            print(ctx.term.green(
                "Created and auto-ACKed PR %d against branch %s: %s"
                % (backport_pr.number, bb, backport_pr.html_url)
            ))
        finally:
            print('Cleaning up')
            ctx.runprocess(['git', 'am', '--abort'], check_returncode=None)
            ctx.runprocess(['git', 'reset', '--hard'], check_returncode=None)
            ctx.runprocess(['git', 'checkout', old_branch],
                           check_returncode=None)
            ctx.runprocess(['git', 'clean', '-fxd'], check_returncode=None)



@Context.command('backport')
def backport_command(ctx):
    if list(ctx.get_patches()):
        ctx.die('No patches are allowed when pushing pull request')

    repo = get_gh_repo(ctx)
    target_dir = os.path.expanduser(ctx.config['patchdir'])
    pr = repo.pull_request(ctx.options['PR_ID'])

    pr_is = repo.issue(pr.number)
    labels = labels_names(pr_is.labels())

    if 'ack' not in labels:
        ctx.die('Pull request is not ACKed')

    if 'rejected' in labels:
        ctx.die('Pull request is rejected')

    if 'pushed' not in labels and not pr.mergeable:
        ctx.die('Pull request is not mergeable.')

    most_recent_results = {}
    for s in repo.commit(pr.head.sha).statuses():
        if s.context not in most_recent_results:
            most_recent_results[s.context] = s.state

    states = most_recent_results.values()
    if 'error' in states or 'failure' in states:
        ctx.die('Pull request failed CI test(s)')
    elif 'pending' in states:
        ctx.die('CI have not completed testing the pull request yet')
    # remove ^^^

    try:
        commit_num = 0
        for commit in sorted_commits(list(pr.commits())):
            patch_name = patch_filename(commit.message, commit_num)
            patch_path = os.path.join(target_dir, patch_name)
            with open(patch_path, 'wb') as patch_file:
                patch_file.write(commit.patch())
            commit_num += 1
    except Exception as e:
        delete_patches(target_dir)
        ctx.die('Failed to get patch(es): {}'.format(e))

    backport(ctx, ctx.options.get('--branch', []), repo, pr)
    delete_patches(target_dir)



@Context.command('pr-push')
def pr_push_command(ctx):
    if list(ctx.get_patches()):
        ctx.die('No patches are allowed when pushing pull request')

    target_dir = os.path.expanduser(ctx.config['patchdir'])
    repo = get_gh_repo(ctx)
    pr = repo.pull_request(ctx.options['PR_ID'])

    pr_is = repo.issue(pr.number)
    labels = labels_names(pr_is.labels())

    if pr_is.is_closed():
        ctx.die('Pull request is already closed')

    if 'ack' not in labels:
        ctx.die('Pull request is not ACKed')

    if 'rejected' in labels:
        ctx.die('Pull request is rejected')

    if 'pushed' in labels:
        ctx.die('Pull request was already pushed')

    if not pr.mergeable:
        ctx.die('Pull request is not mergeable.')

    # TODO: use with CombinedStatus once released
    # status = repo.commit(pr.head.sha).status()
    # if status.state in ('error', 'failure',):
    #     ctx.die('Pull request failed CI test(s)')
    # elif status.state == 'pending':
    #     ctx.die('CI have not completed testing the pull request yet')
    # remove vvv
    most_recent_results = {}
    for s in repo.commit(pr.head.sha).statuses():
        if s.context not in most_recent_results:
            most_recent_results[s.context] = s.state

    states = most_recent_results.values()
    if 'error' in states or 'failure' in states:
        ctx.die('Pull request failed CI test(s)')
    elif 'pending' in states:
        ctx.die('CI have not completed testing the pull request yet')
    # remove ^^^

    try:
        commit_num = 0
        for commit in sorted_commits(list(pr.commits())):
            patch_name = patch_filename(commit.message, commit_num)
            patch_path = os.path.join(target_dir, patch_name)
            with open(patch_path, 'wb') as patch_file:
                patch_file.write(commit.patch())
            commit_num += 1
    except Exception as e:
        delete_patches(target_dir)
        ctx.die('Failed to get patch(es): {}'.format(e))

    ctx.options['--branch'] = [pr.base.ref]
    try:
        # use regular `ipatool push`
        push_command(ctx)
    except libpagure.exceptions.APIError as e:
        ctx.die('Pagure API Error: {}'.format(e))
    except Exception:
        pass
    else:
        if ((not ctx.options.get('--dry-run', False)) and
                ctx.push_info.get('pushed', False)):
            print("Adding label 'pushed'")
            pr_is.add_labels('pushed')
            pr_is.refresh(conditional=True)

            pr_is.create_comment(ctx.push_info['pagure_comment'])
            pr_is.refresh(conditional=True)

            print("Closing pull request {}".format(pr.number))
            pr_is.close()

            backport_branches = set(ctx.options.get('--backport', []))
            if ctx.options['--autobackport']:
                pat = re.compile(r'^ipa-\d+-\d+$')
                backport_branches.update(l for l in labels if pat.match(l))
            if backport_branches:
                backport(ctx, backport_branches, repo, pr)
    finally:
        delete_patches(target_dir)


@Context.command('pr-ack')
def pr_ack_command(ctx):
    repo = get_gh_repo(ctx)
    pr_is = repo.issue(ctx.options['PR_ID'])
    labels = labels_names(pr_is.labels())

    if pr_is.is_closed():
        ctx.die('Pull request was already closed')

    if 'rejected' in labels:
        ctx.die('Pull request was rejected')

    print("Adding 'ack' label to PR {}".format(pr_is.number))
    pr_is.add_labels('ack')
    pr_is.refresh(conditional=True)

    pr_is.create_comment(ctx.options['--comment'])


@Context.command('pr-reject')
def pr_reject_command(ctx):
    repo = get_gh_repo(ctx)
    pr_is = repo.issue(ctx.options['PR_ID'])
    labels = labels_names(pr_is.labels())

    if 'rejected' in labels:
        print("Pull request is already rejected")
    else:
        print("Adding 'rejected' label to PR {}".format(pr_is.number))
        pr_is.add_labels('rejected')
        pr_is.refresh(conditional=True)

    if 'ack' in labels:
        print("Removing 'ack' label from PR {}".format(pr_is.number))
        pr_is.remove_label('ack')
        pr_is.refresh(conditional=True)

    pr_is.create_comment(ctx.options['--comment'])
    pr_is.refresh(conditional=True)

    if not pr_is.is_closed():
        print("Closing pull request")
        pr_is.close()


@Context.command('pr-list')
def pr_list_command(ctx):
    prline_template = '{num: 5}\t{title:.50}\t{labels}\t{url}\t{statuses}'

    states = ctx.options['--state']
    labels = ctx.options['--label']
    repo = get_gh_repo(ctx)

    def parse_signed(lst):
        pos = set()
        neg = set()

        for item in lst:
            if item.startswith('-'):
                neg.add(item[1:])
            elif item.startswith('+'):
                pos.add(item[1:])
            else:
                pos.add(item)
        return (pos, neg,)

    states_pos, states_neg = parse_signed(states)
    labels_pos, labels_neg = parse_signed(labels)

    # if there is intersection between _pos and _neg there will be no
    # results (PR can hardly be closed and not closed at the same time)
    # we can safely stop here
    # TODO: raise an error?
    if states_pos & states_neg:
        return

    if labels_pos & labels_neg:
        return

    if 'all' in states_neg:
        return

    if not states_neg:
        if not states_pos:
            states_pos.add('open')
            search_state = 'open'
        elif len(states_pos) == 1:
            search_state = list(states_pos)[0]
        else:
            search_state = 'all'
    else:
        search_state = 'all'

    # 'all' is not real state
    states_pos.discard('all')

    for pr in repo.pull_requests(state=search_state):
        pr_is = repo.issue(pr.number)
        if states_pos and pr.state not in states_pos:
            continue
        if pr.state in states_neg:
            continue
        labels = pr_is.labels()
        if labels:
            lnames = set(labels_names(labels))
            if labels_pos and not lnames & labels_pos:
                continue
            if lnames & labels_neg:
                continue
        commit_statuses = []
        for c in pr.commits():
            for statuses in c.statuses():
                commit_statuses.append(statuses.state)
        status_result = [list(i) for j, i in groupby(commit_statuses, lambda a: a)]
        statuses = {}
        for status in status_result:
            statuses[status[0]] = len(status)
        print(prline_template.format(
            num=pr.number,
            title=pr.title,
            labels=' '.join(labels_colorize(labels)),
            url=pr.html_url,
            statuses=statuses,
        ))

    # Human error detection section
    print(xtermcolor.colorize("Checking for common mistakes...", rgb=0xff3311))
    pull_requests = repo.pull_requests(
        state='all', sort='updated', direction='desc', number=100
    )
    for pr in pull_requests:
        pr_is = repo.issue(pr.number)
        labels = pr_is.labels()
        lnames = labels_names(labels)
        if (pr.is_merged() and 'pushed' not in lnames):
            print(xtermcolor.colorize(
                "Pull request was merged but not labeled 'pushed'!",
                rgb=0xff0000,
            ))
            print(prline_template.format(
                num=pr.number,
                title=pr.title,
                labels=' '.join(labels_colorize(labels)),
                url=pr.html_url,
            ))
        if ('pushed' in lnames and 'ack' not in lnames):
            print(xtermcolor.colorize(
                "Pull request was pushed without 'ack'!",
                rgb=0xff0000,
            ))
            print(prline_template.format(
                num=pr.number,
                title=pr.title,
                labels=' '.join(labels_colorize(labels)),
                url=pr.html_url,
            ))



@Context.command('start-review')
def start_review_command(ctx):
    ticket_numbers = set(ctx.options['--ticket'])
    if ctx.options['PATCH']:
        patches = ctx.get_patches()
    if not ticket_numbers:
        print(ctx.term.yellow('Using patches from %s' % ctx.config['patchdir']))
        patches = ctx.get_patches()
    else:
        patches = ()
    for patch in patches:
        ticket_numbers.update(patch.ticket_numbers)
    ticket_numbers = sorted(ticket_numbers)
    if ctx.verbosity:
        print('Tickets selected: %s' % ticket_numbers)
    tickets = [Ticket(ctx.pagure, int(n)) for n in ticket_numbers]
    if not tickets:
        ctx.die('No tickets selected')
    if not ctx.pagure:
        ctx.die('Cannot work with --no-pagure')

    existing_reviewers = []
    for ticket in tickets:
        print(ctx.term.blue('Ticket #%s' % ticket.number))
        print('- summary:', ticket.summary)
        reviewer = ticket.reviewer
        print('- reviewer:', ctx.term.yellow(str(reviewer or '')) or 'none')
        if reviewer:
            existing_reviewers.append(reviewer)
    if existing_reviewers:
        if ctx.options['--force']:
            print(ctx.term.yellow('Existing reviewer(s) found'))
        else:
            ctx.die("Existing reviewer(s) found; "
                    "won't overwrite without --force")

    if ctx.options['--dry-run']:
        ctx.die('Exiting, --dry-run specified')
    else:
        while True:
            response = input('Start review on these tickets? [y/n] ')
            if response.lower() == 'n':
                return
            elif response.lower() == 'y':
                break

    for ticket in tickets:
        print('Starting review: #%s' % ticket.number)
        print(ctx.term.red('Does not work temporary, update tickets manually'))

    if ctx.options['--am']:
        print('Applying patches to worktree...')
        am_patches(ctx, patches)


def am_patches(ctx, patches):
    for patch in patches:
        print('Applying patch:', patch.filename)
        ctx.runprocess(ctx.config['am-command'],
                       stdin_string=''.join(patch.lines),
                       timeout=60, verbosity=2)


@Context.command('am')
def am_command(ctx):
    patches = ctx.get_patches()
    am_patches(ctx, patches)


if __name__ == '__main__':
    Context(docopt.docopt(__doc__)).run()