From be51b0d8faca19d2b86027f4ce9c51ccaa3864e2 Mon Sep 17 00:00:00 2001 From: Paul Hebble Date: Mon, 11 Jan 2021 16:34:40 -0600 Subject: [PATCH] Pull request merge script --- bin/changes-since-last-release.pl | 47 ---------- bin/ckan-build-info.py | 76 --------------- bin/ckan-build.py | 20 ---- bin/ckan-merge-pr.py | 86 +++++++++++++++++ bin/ckan-release-maker.py | 64 ------------- bin/ckan-release-promoter.py | 100 -------------------- bin/ckan-validate.py | 0 bin/ckan_github_utils.py | 147 ------------------------------ bin/close-old-support-tickets.pl | 86 ----------------- bin/requirements.txt | 3 + 10 files changed, 89 insertions(+), 540 deletions(-) delete mode 100755 bin/changes-since-last-release.pl delete mode 100644 bin/ckan-build-info.py delete mode 100644 bin/ckan-build.py create mode 100755 bin/ckan-merge-pr.py delete mode 100644 bin/ckan-release-maker.py delete mode 100644 bin/ckan-release-promoter.py mode change 100644 => 100755 bin/ckan-validate.py delete mode 100644 bin/ckan_github_utils.py delete mode 100755 bin/close-old-support-tickets.pl diff --git a/bin/changes-since-last-release.pl b/bin/changes-since-last-release.pl deleted file mode 100755 index 27043335bf..0000000000 --- a/bin/changes-since-last-release.pl +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/perl -w -use 5.010; -use strict; -use warnings; -use autodie qw(:all); -use FindBin qw($Bin); -use Getopt::Std; - -# Shows everything which changed since the last release. -# This should be easy, but the current build processes don't -# actually tag the repos. So we have to look in the `build-tag` -# file. Still a better love story than Twilight. - -my %opts = ( - 'f' => 0, # Fetch from upstream - 'u' => 'origin', # Upstream name -); - -# Why doesn't C# have something this easy for option handling? - -getopts('fu:', \%opts); - -open(my $fh, '<', "$Bin/../build-tag"); - -my %last_release_commit; - -my $tags = <$fh>; - -foreach my $segment (split(/\|/,$tags)) { - $segment =~ /^(?[^+]+)\+(?[0-9a-f]+)$/ - or die "Cannot parse $segment"; - - $last_release_commit{$+{repo}} = $+{commit}; -} - -# Walk through each repo and display changes. - -foreach my $repo (keys %last_release_commit) { - chdir("$Bin/../$repo"); - - say "\n== $repo ==\n"; - - system("git fetch origin") if $opts{f}; - - # By piping to cat, we avoid git invoking the pager. - system(qq{git log --no-merges --pretty="format:%h %s (%aN)" --abbrev-commit $last_release_commit{$repo}..$opts{u}/master}); -} diff --git a/bin/ckan-build-info.py b/bin/ckan-build-info.py deleted file mode 100644 index dfcd8ae558..0000000000 --- a/bin/ckan-build-info.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python - -import os, sys - -BUILD_INFO_MESSAGE = """ ->BUILD_TAG %s -************************************************************* -************************************************************* -************************************************************* - -This CKAN build references the following repositories and commit hashes: -%s - -You can fetch the source used for this build by running: -%s - -************************************************************* -************************************************************* -************************************************************* -""" - -def main(): - if len(sys.argv) < 2: - print 'Usage:' - print sys.argv[0] + ' ' - - repositories = sys.argv[1:] - - os.system('touch hashes') - os.system('touch urls') - - cwd = os.getcwd() - - for repo in repositories: - repo_name = repo[:] - if repo[-1] == '.': - repo_name = repo[:-1] - os.chdir(os.path.join(cwd, repo_name)) - os.system('git rev-parse HEAD >> ../hashes') - os.system('git config --get remote.origin.url >> ../urls') - - os.chdir(cwd) - - hashes = {} - urls = {} - - with open('hashes', 'r') as hashes_file: - with open('urls', 'r') as urls_file: - line_index = 0 - url_lines = urls_file.readlines() - - for line in hashes_file.readlines(): - hashes[repositories[line_index]] = line.strip() - urls[repositories[line_index]] = url_lines[line_index].strip() - line_index += 1 - - repo_msg = '' - fetch_msg = '' - build_tag = '' - - for repo, commit_hash in hashes.iteritems(): - repo_name = repo[:] - if repo[-1] == '.': - repo_name = repo[:-1] - - build_tag += '%s+%s|' % (repo_name, commit_hash) - repo_msg += '* %s - %s\n' % (repo_name, commit_hash) - rev_parse = '' - if repo[-1] == '.': - rev_parse = 'git fetch --tags --progress %s +refs/pull/*:refs/remotes/origin/pr/*; ' % urls[repo] - fetch_msg += 'git clone %s; cd %s; %sgit checkout -f %s; cd ..;\n' % (urls[repo], repo_name, rev_parse, commit_hash) - - print BUILD_INFO_MESSAGE % (build_tag, repo_msg, fetch_msg) - -if __name__ == "__main__": - main() diff --git a/bin/ckan-build.py b/bin/ckan-build.py deleted file mode 100644 index d6862bce42..0000000000 --- a/bin/ckan-build.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python - -from ckan_github_utils import * - -import os, sys -import argparse - -def main(): - parser = argparse.ArgumentParser(description='Builds CKAN from a list of commit hashes') - - parser.add_argument('--ckan-core-hash', dest='core_hash', action='store', help='The commit hash for CKAN-core', required=False) - parser.add_argument('--ckan-gui-hash', dest='gui_hash', action='store', help='The commit hash for CKAN-GUI', required=False) - parser.add_argument('--ckan-cmdline-hash', dest='cmdline_hash', action='store', help='The commit hash for CKAN-cmdline', required=False) - parser.add_argument('--ckan-release-version', dest='release_version', action='store', help='The version with which to stamp ckan.exe', required=False) - args = parser.parse_args() - - build_ckan(args.core_hash, args.gui_hash, args.cmdline_hash, args.release_version) - -if __name__ == "__main__": - main() diff --git a/bin/ckan-merge-pr.py b/bin/ckan-merge-pr.py new file mode 100755 index 0000000000..56c55b8c2e --- /dev/null +++ b/bin/ckan-merge-pr.py @@ -0,0 +1,86 @@ +#!/usr/bin/python3 + +# https://github.com/KSP-CKAN/CKAN/wiki/Releasing-a-new-CKAN-client#when-merging-pull-requests + +import sys +from pathlib import Path +from subprocess import run +from exitstatus import ExitStatus +import click +from click import command, option, argument +from git import Repo +from github import Github + +@command() +@option('--repo-path', type=click.Path(exists=True, file_okay=False), + default='.', help='Path to CKAN working copy') +@option('--token', required=False, envvar='GITHUB_TOKEN') +@argument('pr_num', type=click.INT) +def merge_pr(repo_path: str, token: str, pr_num: int) -> None: + r = Repo(repo_path) + pr = Github(token).get_repo('KSP-CKAN/CKAN').get_pull(pr_num) + # Make sure master is checked out + if r.head.ref.name != 'master': + print('Not on master branch!') + # Bail and let user get his or her repo in order + sys.exit(ExitStatus.failure) + # Make sure master is up to date + master_remote = r.remotes[r.head.ref.tracking_branch().remote_name] + print(f'Fetching {master_remote.name}...') + master_remote.fetch() + remote_master = r.head.ref.tracking_branch() + if r.head.commit.hexsha != remote_master.commit.hexsha: + print(f'master branch is not up to date!') + sys.exit(ExitStatus.failure) + # Make sure PR's branch exists + local_branch = pr.head.ref + branch = r.heads[local_branch] + # Fetch remote to make sure we aren't missing changes + tracking = branch.tracking_branch() + if not tracking: + print(f'Upstream {pr.head.label} missing!') + sys.exit(ExitStatus.failure) + remote = r.remotes[tracking.remote_name] + if remote.name != master_remote.name: + print(f'Fetching {remote.name}...') + remote.fetch() + # Make sure branch is identical to its upstream + if branch.commit.hexsha != pr.head.sha: + print(f'Branch {local_branch} not up to date!') + # Bail and let the user figure it out, no way we can safely cover all possibilities here + sys.exit(ExitStatus.failure) + # Get reviewers from PR + reviewers = [rvw.user.login for rvw in pr.get_reviews() if rvw.state == 'APPROVED'] + if not reviewers: + print(f'PR #{pr_num} is not approved!') + sys.exit(ExitStatus.failure) + # Get title from PR + pr_title = pr.title + # Get author from PR + author = pr.user.login + # Merge the branch with no-commit and no-ff + base = r.merge_base(branch, r.head) + r.index.merge_tree(branch, base=base) + # Update the working copy + r.index.checkout(force=True) + # Print line to add to CHANGELOG.md at top of file, user needs to move it to the right spot + changelog_path = Path(repo_path) / 'CHANGELOG.md' + with open(changelog_path, 'r+') as changelog: + lines = [f'- [UNKNOWN] {pr_title} (#{pr_num} by: {author}; reviewed: {", ".join(reviewers)})\n', + *changelog.readlines()] + changelog.seek(0) + changelog.writelines(lines) + # Edit CHANGELOG.md + editor=r.config_reader().get('core', 'editor') + run([editor, changelog_path]) + # Stage change log + r.index.add([changelog_path.as_posix()]) + # Commit + r.index.commit(f'Merge #{pr_num} {pr_title}', + parent_commits=(r.head.commit, branch.commit)) + + # Don't push, let the user inspect and decide + sys.exit(ExitStatus.success) + +if __name__ == '__main__': + merge_pr() diff --git a/bin/ckan-release-maker.py b/bin/ckan-release-maker.py deleted file mode 100644 index 67de30f84c..0000000000 --- a/bin/ckan-release-maker.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python - -""" -This script can create GitHub tags/ releases and push build artifacts to them. -""" - -# ---* DO NOT EDIT BELOW THIS LINE *--- - -import os, sys -import argparse - -from ckan_github_utils import * - -def main(): - parser = argparse.ArgumentParser(description='Create GitHub releases and upload build artifacts') - - parser.add_argument('--user', dest='user', action='store', help='Sets the GitHub user for the API', required=True) - parser.add_argument('--token', dest='token', action='store', help='Sets the GitHub access token for the API', required=True) - parser.add_argument('--repository', dest='repository', action='store', help='Sets the GitHub repository in which to make the release. Syntax: :owner/:repo', required=True) - parser.add_argument('--tag', dest='tag', action='store', help='Sets the name of the tag that will be created for the release', required=True) - parser.add_argument('--name', dest='name', action='store', help='Sets the name of the release that will be created', required=True) - parser.add_argument('--body', dest='body', action='store', help='Sets the body text of the release', required=True) - parser.add_argument('--draft', dest='draft', action='store_true', help='Sets the release as draft', required=False) - parser.add_argument('--prerelease', dest='prerelease', action='store_true', help='Sets the release as a pre-release', required=False) - parser.add_argument('--push-build-tag-file', dest='build_tag_file', action='store_true', help='Pushes a special build-tag file to the repository', required=False) - parser.add_argument('artifacts', metavar='file', type=str, nargs='+', help='build artifact') - args = parser.parse_args() - - if len(sys.argv) == 1: - parser.print_help() - sys.exit(0) - - if args.build_tag_file: - response = push_github_file(args.user, args.token, args.repository, 'build-tag', str(datetime.datetime.now())) - if response.status_code < 400: - print 'Build-tag file pushed to repository!' - else: - print 'There was an issue pushing the build-tag file! - %s' % response.text - sys.exit(1) - - response = make_github_release(args.user, args.token, args.repository, args.tag, args.name, args.body, args.draft, args.prerelease) - response_json = json.loads(response.text) - - if response.status_code == 201: - print 'Release created successfully!' - else: - print 'There was an issue creating the release - %s' % response.text - sys.exit(1) - - upload_url = response_json['upload_url'] - - for artifact in args.artifacts: - response = make_github_release_artifact(args.user, args.token, upload_url, artifact) - if response.status_code == 201: - print 'Asset successfully uploaded' - else: - print 'There was an issue uploading your asset! - %s' % response.text - sys.exit(1) - - print 'Done!' - sys.exit(0) - -if __name__ == "__main__": - main() diff --git a/bin/ckan-release-promoter.py b/bin/ckan-release-promoter.py deleted file mode 100644 index 37a68d38c7..0000000000 --- a/bin/ckan-release-promoter.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python - -from ckan_github_utils import * - -import os, sys -import argparse -import urllib2 - -def main(): - parser = argparse.ArgumentParser(description='Promote nightly releases to production') - - parser.add_argument('--user', dest='user', action='store', help='Sets the GitHub user for the API', required=True) - parser.add_argument('--token', dest='token', action='store', help='Sets the GitHub access token for the API', required=True) - parser.add_argument('--repository', dest='repository', action='store', help='Sets the GitHub repository in which to make the release. Syntax: :owner/:repo', required=True) - parser.add_argument('--jenkins-build', dest='jenkins_build', action='store', help='The URL to the jenkins build that will be promoted', required=True) - parser.add_argument('--release-version', dest='release_version', action='store', help='Sets the CKAN release version', required=True) - parser.add_argument('--name', dest='name', action='store', help='Sets the name of the release that will be created', required=True) - parser.add_argument('--draft', dest='draft', action='store_true', help='Sets the release as draft', required=False) - parser.add_argument('--prerelease', dest='prerelease', action='store_true', help='Sets the release as a pre-release', required=False) - args = parser.parse_args() - - build_log = urllib2.urlopen(args.jenkins_build + '/consoleText').read() - build_tag = None - - for line in build_log.split('\n'): - line = line.strip() - if line.startswith('>BUILD_TAG'): - build_tag = line.split(' ')[1] - break - - if build_tag == None: - print 'Error: >BUILD_TAG not found in build log, bailing out..' - sys.exit(1) - - print 'Found build tag: ' + build_tag - - core_hash = '' - gui_hash = '' - cmdline_hash = '' - - for item in build_tag.split('|'): - if len(item) == 0: - continue - - repo, commit_hash = item.split('+') - if repo == 'CKAN-core': - core_hash = commit_hash - elif repo == 'CKAN-GUI': - gui_hash = commit_hash - elif repo == 'CKAN-cmdline': - cmdline_hash = commit_hash - - print 'Reproducing build..' - - build_ckan(core_hash, gui_hash, cmdline_hash, args.release_version + '-0-g0000000') - os.system('mono ckan.exe --version') - - release_diff = '' - - - response = push_github_file(args.user, args.token, args.repository, 'build-tag', build_tag) - if response.status_code < 400: - print 'Build-tag file pushed to repository!' - else: - print 'There was an issue pushing the build-tag file! - %s' % response.text - sys.exit(1) - - body_text = """Promoted from [%s](%s) -* CKAN-core - %s -* CKAN-GUI - %s -* CKAN-cmdline - %s - ---- -Changes since last version: -%s -""" % (args.jenkins_build, args.jenkins_build, core_hash, gui_hash, cmdline_hash, release_diff) - - response = make_github_release(args.user, args.token, args.repository, args.release_version, args.name, body_text, args.draft, args.prerelease) - response_json = json.loads(response.text) - - if response.status_code == 201: - print 'Release created successfully!' - else: - print 'There was an issue creating the release - %s' % response.text - sys.exit(1) - - upload_url = response_json['upload_url'] - - response = make_github_release_artifact(args.user, args.token, upload_url, 'ckan.exe') - if response.status_code == 201: - print 'Asset successfully uploaded' - else: - print 'There was an issue uploading your asset! - %s' % response.text - sys.exit(1) - - print 'Done!' - sys.exit(0) - -if __name__ == "__main__": - main() diff --git a/bin/ckan-validate.py b/bin/ckan-validate.py old mode 100644 new mode 100755 diff --git a/bin/ckan_github_utils.py b/bin/ckan_github_utils.py deleted file mode 100644 index ed075b254a..0000000000 --- a/bin/ckan_github_utils.py +++ /dev/null @@ -1,147 +0,0 @@ -GITHUB_API = 'https://api.github.com' -CKAN_CORE_VERSION_STRING = 'private readonly static string BUILD_VERSION = null;' -CKAN_CORE_VERSION_STRING_TARGET = 'private readonly static string BUILD_VERSION = "%s";' - -# ---* DO NOT EDIT BELOW THIS LINE *--- - -import os, sys - -import urllib -import requests -from urlparse import urljoin - -import datetime -import base64 - -import json -import shutil - -def run_git_clone(repo, commit_hash): - if os.path.isdir(repo): - shutil.rmtree(repo) - - cwd = os.getcwd() - if os.system('git clone https://github.com/KSP-CKAN/%s.git' % repo) != 0: - sys.exit(1) - - os.chdir(os.path.join(cwd, repo)) - - if commit_hash != 'master': - if os.system('git checkout -f %s' % commit_hash) != 0: - sys.exit(1) - - os.chdir(cwd) - -def build_repo(repo): - cwd = os.getcwd() - os.chdir(os.path.join(cwd, repo)) - - if os.system('sh build.sh') != 0: - sys.exit(1) - - os.chdir(cwd) - -def stamp_ckan_version(version): - cwd = os.getcwd() - os.chdir(os.path.join(cwd, 'CKAN-core')) - - meta_contents = None - - with open('Meta.cs', 'r') as meta_file: - meta_contents = meta_file.read() - - if meta_contents == None: - print 'Error reading Meta.cs' - sys.exit(1) - - meta_contents = meta_contents.replace(CKAN_CORE_VERSION_STRING, CKAN_CORE_VERSION_STRING_TARGET % version) - - with open("Meta.cs", "w") as meta_file: - meta_file.write(meta_contents) - - os.chdir(cwd) - -def build_ckan(core_hash, gui_hash, cmdline_hash, release_version): - if core_hash == None: - core_hash = 'master' - if gui_hash == None: - gui_hash = 'master' - if cmdline_hash == None: - cmdline_hash = 'master' - - print 'Building CKAN from the following commit hashes:' - print 'CKAN-core: %s' % core_hash - print 'CKAN-GUI: %s' % gui_hash - print 'CKAN-cmdline: %s' % cmdline_hash - - run_git_clone('CKAN-core', core_hash) - run_git_clone('CKAN-GUI', gui_hash) - run_git_clone('CKAN-cmdline', cmdline_hash) - - if release_version != None: - stamp_ckan_version(release_version) - - build_repo('CKAN-core') - build_repo('CKAN-GUI') - build_repo('CKAN-cmdline') - - print 'Done!' - -def make_github_post_request(url_part, username, password, payload): - url = urljoin(GITHUB_API, url_part) - print '::make_github_post_request - %s' % url - return requests.post(url, auth = (username, password), data = json.dumps(payload), verify=False) - -def make_github_get_request(url_path, username, password, payload): - url = urljoin(GITHUB_API, url_path) - print '::make_github_get_request - %s' % url - return requests.get(url, auth = (username, password), data = json.dumps(payload), verify=False) - -def make_github_put_request(url_path, username, password, payload): - url = urljoin(GITHUB_API, url_path) - print '::make_github_put_request - %s' % url - return requests.put(url, auth = (username, password), data = json.dumps(payload), verify=False) - -def make_github_post_request_raw(url_part, username, password, payload, content_type): - url = urljoin(GITHUB_API, url_part) - print '::make_github_post_request_raw - %s' % url - - headers = { 'Content-Type': content_type } - return requests.post(url, auth = (username, password), data = payload, verify=False, headers=headers) - -def make_github_release(username, password, repo, tag_name, name, body, draft, prerelease): - payload = {} - payload['tag_name'] = tag_name - payload['name'] = name - payload['body'] = body - payload['draft'] = draft - payload['prerelease'] = prerelease - return make_github_post_request('/repos/%s/releases' % repo, username, password, payload) - -def make_github_release_artifact(username, password, upload_url, filepath, content_type = 'application/zip'): - filename = os.path.basename(filepath) - query = { 'name': filename } - url = '%s?%s' % (upload_url[:-7], urllib.urlencode(query)) - payload = file(filepath, 'r').read() - return make_github_post_request_raw(url, username, password, payload, content_type) - -def get_github_file(username, password, repo, path): - return make_github_get_request('/repos/%s/contents/%s' % (repo, path), username, password, {}) - -def push_github_file_sha(username, password, repo, path, sha, content, branch='master'): - payload = {} - payload['path'] = path - payload['message'] = 'Updating build-tag' - payload['content'] = base64.b64encode(content) - payload['sha'] = sha - payload['branch'] = branch - return make_github_put_request('/repos/%s/contents/%s' % (repo, path), username, password, payload) - -def push_github_file(username, password, repo, path, content, branch='master'): - response = get_github_file(username, password, repo, path) - if response.status_code >= 400: - print 'There was an issue fetching "%s"! Status: %s - %s' % (path, str(response.status_code), response.text) - sys.exit(1) - - response_json = json.loads(response.text) - return push_github_file_sha(username, password, repo, response_json['path'], response_json['sha'], content) diff --git a/bin/close-old-support-tickets.pl b/bin/close-old-support-tickets.pl deleted file mode 100755 index 4f5de3b80f..0000000000 --- a/bin/close-old-support-tickets.pl +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/perl -w -use 5.010; -use strict; -use warnings; -use autodie; - -use Net::GitHub; -use Date::Parse qw(str2time); - -# Old support ticket closer. (GH #942) -# -# Finds tickets tagged with support, which haven't been active in 7 -# days, which are unassigned, and which has comments, and closed them. -# Requires a github token. Ideally run from a bot account. -# -# License: Same as CKAN itself. - -my $token = $ARGV[0] or die "Usage: $0 [token]\n"; - -my $github = Net::GitHub->new( - RaiseError => 1, - access_token => $token, -); - -# 7 days ago -my $date_cutoff = time() - 7 * 86400; - -foreach my $repo (("CKAN", "NetKAN")) { - $github->set_default_user_repo("KSP-CKAN", $repo); - - my $issues = $github->issue; - - # Get all our candidate issues - - my @candidates = $issues->repos_issues({ - state => 'open', - labels => 'support', - assignee => "none", - }); - - # Walk through each one, and see if we can close it - - foreach my $candidate (@candidates) { - my $id = $candidate->{number}; - my $title = "$candidate->{title} (#$id)"; - my $author = $candidate->{user}{login}; - - my $num_comments = +$candidate->{comments}; - if ($num_comments == 0) { - say "Skipped (no comments) : $title"; - next; - } - - # Skip if last comment is by OP - my @comments = $issues->comments($id); - my $last_comment = $comments[$num_comments - 1]; - if ($last_comment->{user}{login} eq $author) { - say "Skipped (author comment) : $title"; - next; - } - - my $last_update = str2time($candidate->{updated_at}); - - if ($last_update > $date_cutoff) { - say "Skipped (recent update) : $title"; - next; - } - - # Yay! Something we can close! - say "Closing $title"; - - close_ticket($issues, $id); - } - -} -say "Done!"; - -sub close_ticket { - my ($issues, $id) = @_; - - $issues->create_comment($id, { - body => "Hey there! I'm a fun-loving automated bot who's responsible for making sure old support tickets get closed out. As we haven't seen any activity on this ticket for a while, we're hoping the problem has been resolved and I'm closing out the ticket automatically. If I'm doing this in error, please add a comment to this ticket to let us know, and we'll re-open it!" - }); - - $issues->update_issue( $id, { state => "closed" }); -} diff --git a/bin/requirements.txt b/bin/requirements.txt index cbfb8e1fb6..24017f9522 100644 --- a/bin/requirements.txt +++ b/bin/requirements.txt @@ -1,2 +1,5 @@ requests jsonschema +click +git +github