[DA-172] Automatically create a JIRA issue to track staging releases …

…(with release notes) (#206)

ci/release_notes.py

@@ -5,30 +5,48 @@
 current version, then gets commit messages from that tag to HEAD, and formats the messages (with
 JIRA's style of markup) to make release notes. HEAD is assumed (but not required) to also be tagged
 for the release that's going out / the release for which we're generating notes.
+
+This requires the
+    JIRA_API_USER_PASSWORD and
+    JIRA_API_USER_NAME
+environment variables to be set. If it is also set, the comma-separated list of JIRA user names in
+    JIRA_WATCHER_NAMES
+will be set as watchers on newly created release trackers.
 """
 
 import httplib
 import json
 import logging
+import os
 import re
 import subprocess
 import sys
 import urllib2
 
+import jira
+
 _REPO_BASE_URL = 'https://github.com/vanderbilt/pmi-data'
 
-# Git tags of this format denote releases.
-_RELEASE_TAG_RE = re.compile(r'v[0-9]+(?:-[0-9]+)*-rc[0-9]+')
+# Git tags of this format denote releases. Final letter suffixes denote cherry-picks.
+# This should match circle.yml.
+_RELEASE_TAG_RE = re.compile(r'v[0-9]+(?:-[0-9]+)*-rc[0-9]+[a-z]*$')
+_CHERRY_PICK_RE = re.compile(r'(.*-rc[0-9]+)([a-z]*)')
 
 # Formatting for release notes in JIRA comments.
 # Note that JIRA auto-linkifies JIRA IDs, so avoid using commit message text in a link.
 _LOG_LINE_FORMAT = "--format=*   [%aN %ad|" + _REPO_BASE_URL + "/commit/%h] %s"
 # Overall release notes template.
 _RELEASE_NOTES_T = """h1. Release Notes for %(current)s
-h2. since %(prev)s
+h2. deployed to %(project)s, listing changes since %(prev)s
 %(history)s
 """
 
+_JIRA_INSTANCE_URL = 'https://precisionmedicineinitiative.atlassian.net/'
+_JIRA_PROJECT_ID = 'DA'
+_JIRA_NAME_VARNAME = 'JIRA_API_USER_NAME'
+_JIRA_PASSWORD_VARNAME = 'JIRA_API_USER_PASSWORD'
+_JIRA_WATCHERS_VARNAME = 'JIRA_WATCHER_NAMES'
+
 
 def _linkify_pull_request_ids(text):
   """Converts all substrings like "(#123)" to links to pull requests."""
@@ -59,7 +77,7 @@ def _get_deployed_version(project_id):
   return app_version.split('.')[0]
 
 
-def _get_release_notes_since_tag(deployed_tag):
+def _get_release_notes_since_tag(deployed_tag, project_id, current_tag):
   """Formats release notes for JIRA from commit messages, from the given tag to HEAD."""
   process = subprocess.Popen(
       ['git', 'log', deployed_tag + '..', _LOG_LINE_FORMAT],
@@ -69,7 +87,8 @@ def _get_release_notes_since_tag(deployed_tag):
   stdout, _ = process.communicate()
   commit_messages = stdout.decode('utf8')  # Keep further text processing from choking on non-ASCII.
   return _RELEASE_NOTES_T % {
-    'current': _find_current_commit_tag(),
+    'current': current_tag,
+    'project': project_id,
     'prev': deployed_tag,
     'history': _linkify_pull_request_ids(commit_messages),
   }
@@ -82,22 +101,90 @@ def _find_current_commit_tag():
     raise RuntimeError('Getting current tag.')
   stdout, _ = process.communicate()
   tag = stdout.strip()
-  if not _RELEASE_TAG_RE.match(tag):
-    logging.warning('Current tag %r does not look like a release tag.', tag)
   return tag
 
 
+def _connect_to_jira():
+  """Opens a JIRA API connection based on username/pw from env vars."""
+  for varname in (_JIRA_PASSWORD_VARNAME, _JIRA_NAME_VARNAME):
+    if varname not in os.environ:
+      raise RuntimeError('No environment variable value for %r.' % varname)
+  return jira.JIRA(
+      _JIRA_INSTANCE_URL,
+      basic_auth=(os.getenv(_JIRA_NAME_VARNAME), os.getenv(_JIRA_PASSWORD_VARNAME)))
+
+
+def _strip_cherry_pick(version_id):
+  """Returns a tuple of (version ID without cherry-pick suffix, boolean is_cherry_pick)."""
+  match = _CHERRY_PICK_RE.search(version_id)
+  if match is None:
+    # Not a recognized format, don't try to parse it.
+    return version_id, False
+  else:
+    return match.group(1), bool(match.group(2))
+
+
+def _get_watchers():
+  watchers = set()
+  for name in [n.strip() for n in os.getenv(_JIRA_WATCHERS_VARNAME, '').split(',')]:
+    if name:
+      watchers.add(name)
+  return watchers
+
+
+def _update_or_create_release_tracker(jira_connection, project_id, full_version_id, release_notes):
+  """Adds release notes to a new or existing JIRA issue."""
+  version_id, is_cherry_pick = _strip_cherry_pick(full_version_id)
+  summary = 'Release tracker for %s on %s' % (version_id, project_id)
+  issues = jira_connection.search_issues(
+      'project = "%s" AND summary ~ "%s" ORDER BY created DESC' % (_JIRA_PROJECT_ID, summary))
+  if issues:
+    if len(issues) > 1:
+      logging.warning(
+          'Found multiple release tracker matches, using newest. %s',
+          ', '.join('[%s] %s' % (issue.key, issue.fields().summary) for issue in issues))
+    issue = issues[0]
+    jira_connection.add_comment(issue, release_notes)
+    what_happened = 'Updated'
+  else:
+    if is_cherry_pick:
+      logging.warning(
+          'Expected %r to exist since %s looks like a cherry-pick. Creating a new issue instead.',
+          summary, full_version_id)
+    issue = jira_connection.create_issue(
+        project=_JIRA_PROJECT_ID,
+        summary=summary,
+        description=release_notes,
+        issuetype={'name': 'Task'})
+    for watcher_username in _get_watchers():
+      try:
+        jira_connection.add_watcher(issue, watcher_username)
+      except jira.exceptions.JIRAError, e:
+        logging.warning('Skipping invalid watcher %r (got %s).', watcher_username, e.status_code)
+    what_happened = 'Created'
+  logging.info('%s [%s] with release notes for %s.', what_happened, issue.key, full_version_id)
+
+
 def main():
-  """Looks up version tags, gets commit logs, and prints release notes."""
+  """Looks up version tags, gets commit logs, and publishes release in JIRA."""
+  logging.getLogger().setLevel(logging.INFO)
   if len(sys.argv) != 2:
     logging.critical('Usage: %s appengine_project_id', sys.argv[0])
     sys.exit(1)
-  appengine_id = sys.argv[1]
-  deployed_version = _get_deployed_version(appengine_id)
+  project_id = sys.argv[1]
+
+  deployed_version = _get_deployed_version(project_id)
   if not _RELEASE_TAG_RE.match(deployed_version):
     logging.warning(
-        'Tag %r from %r does not look like a release tag.', deployed_version, appengine_id)
-  print _get_release_notes_since_tag(deployed_version)
+        'Tag %r from %r does not look like a release tag.', deployed_version, project_id)
+  current_version = _find_current_commit_tag()
+  if not _RELEASE_TAG_RE.match(current_version):
+    logging.warning('Current tag %r does not look like a release tag.', current_version)
+
+  jira_connection = _connect_to_jira()
+  release_notes = _get_release_notes_since_tag(deployed_version, project_id, current_version)
+  logging.info(release_notes)
+  _update_or_create_release_tracker(jira_connection, project_id, current_version, release_notes)
 
 
 if __name__ == '__main__':

circle.yml

@@ -5,6 +5,7 @@ dependencies:
     # when upgrading or installing new components.
     - sudo chmod o+w /opt
     - pip install --upgrade httplib2
+    - pip install --upgrade jira
     - pip install --upgrade oauth2client
     - pip install --upgrade pylint
 
@@ -28,12 +29,13 @@ deployment:
   # the form v0-1-1-rc0. We use tags as AppEngine version names, so they must
   # follow AE rules (no dots).
   staging:
-    tag: /v[0-9]+(-[0-9]+)*-rc[0-9]+/
+    # This should match _RELEASE_TAG_RE in release_notes.py.
+    tag: /v[0-9]+(-[0-9]+)*-rc[0-9]+[a-z]*/
     commands:
       - ./ci/activate_creds.sh ~/gcloud-credentials.key
+      - ./ci/release_notes.py all-of-us-rdr-staging
       - ./ci/deploy.sh all-of-us-rdr-staging config/config_staging.json ~/gcloud-credentials.key
       - ./rest-api/test/test_server.sh -i https://all-of-us-rdr-staging.appspot.com
-      - ./ci/release_notes.py all-of-us-rdr-staging
 
 # Stable and production deployments are not done automatically but can be
 # manually executed with the test.sh and deploy.sh pair, e.g.:

rest-api/README.md

@@ -164,16 +164,21 @@ requests), as second verification of the service account's auth.
 
 ### Deploying to staging
 
-* Go to https://github.com/vanderbilt/pmi-data/releases/new
-* Enter a tag name of the form vX-Y-rcZZ -- e.g. v0-1-rc14
-* Unless this is intended to be pushed to prod eventually, check the "This is a pre-release" box.
-* Submit.
+*   Go to https://github.com/vanderbilt/pmi-data/releases/new
+*   Enter a tag name of the form vX-Y-rcZZ -- e.g. v0-1-rc14. For cherry picks,
+    add an additional letter (rc14a).
+*   Unless this is intended to be pushed to prod eventually, check the
+    "This is a pre-release" box.
+*   Submit.
 
 CircleCI should automatically push to staging, based on logic found in
 https://github.com/vanderbilt/pmi-data/blob/master/circle.yml
 
-If you are adding new indexes, the tests may fail when they aren't ready yet; use Rebuild in
-CircleCI to retry.
+If you are adding new indexes, the tests may fail when they aren't ready yet;
+use Rebuild in CircleCI to retry.
+
+CircleCI will also open a new JIRA issue to track the release (or update an
+existing issue for cherry picks).
 
 ### Tools