release creation script(s)

dev-tools/scripts/buildAndPushRelease.py

@@ -0,0 +1,358 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import datetime
+import re
+import time
+import shutil
+import os
+import sys
+import subprocess
+import textwrap
+import urllib.request, urllib.error, urllib.parse
+import xml.etree.ElementTree as ET
+
+LOG = '/tmp/release.log'
+
+def log(msg):
+  f = open(LOG, mode='ab')
+  f.write(msg.encode('utf-8'))
+  f.close()
+
+def run(command):
+  log('\n\n%s: RUN: %s\n' % (datetime.datetime.now(), command))
+  if os.system('%s >> %s 2>&1' % (command, LOG)):
+    msg = '    FAILED: %s [see log %s]' % (command, LOG)
+    print(msg)
+    raise RuntimeError(msg)
+
+def runAndSendGPGPassword(command, password):
+  p = subprocess.Popen(command, shell=True, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
+  f = open(LOG, 'ab')
+  while True:
+    p.stdout.flush()
+    line = p.stdout.readline()
+    if len(line) == 0:
+      break
+    f.write(line)
+    if line.find(b'Enter GPG keystore password:') != -1:
+      time.sleep(1.0)
+      p.stdin.write((password + '\n').encode('UTF-8'))
+      p.stdin.write('\n'.encode('UTF-8'))
+
+  result = p.poll()
+  if result != 0:
+    msg = '    FAILED: %s [see log %s]' % (command, LOG)
+    print(msg)
+    raise RuntimeError(msg)
+
+def load(urlString):
+  try:
+    content = urllib.request.urlopen(urlString).read().decode('utf-8')
+  except Exception as e:
+    print('Retrying download of url %s after exception: %s' % (urlString, e))
+    content = urllib.request.urlopen(urlString).read().decode('utf-8')
+  return content
+
+def getGitRev():
+  status = os.popen('git status').read().strip()
+  if 'nothing to commit, working directory clean' not in status and 'nothing to commit, working tree clean' not in status:
+    raise RuntimeError('git clone is dirty:\n\n%s' % status)
+  branch = os.popen('git rev-parse --abbrev-ref HEAD').read().strip()
+  command = 'git log origin/%s..' % branch
+  unpushedCommits = os.popen(command).read().strip()
+  if len(unpushedCommits) > 0:
+    raise RuntimeError('There are unpushed commits - "%s" output is:\n\n%s' % (command, unpushedCommits))
+
+  print('  git clone is clean')
+  return os.popen('git rev-parse HEAD').read().strip()
+
+def prepare(root, version, gpgKeyID, gpgPassword):
+  print()
+  print('Prepare release...')
+  if os.path.exists(LOG):
+    os.remove(LOG)
+
+  os.chdir(root)
+  print('  git pull...')
+  run('git pull')
+
+  rev = getGitRev()
+  print('  git rev: %s' % rev)
+  log('\nGIT rev: %s\n' % rev)
+
+  print('  Check DOAP files')
+  checkDOAPfiles(version)
+
+  print('  ant clean test validate documentation-lint')
+  run('ant clean test validate documentation-lint')
+
+  open('rev.txt', mode='wb').write(rev.encode('UTF-8'))
+
+  print('  lucene prepare-release')
+  os.chdir('lucene')
+  cmd = 'ant -Dversion=%s' % version
+  if gpgKeyID is not None:
+    cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID
+  else:
+    cmd += ' prepare-release-no-sign'
+
+  if gpgPassword is not None:
+    runAndSendGPGPassword(cmd, gpgPassword)
+  else:
+    run(cmd)
+
+  print('  solr prepare-release')
+  os.chdir('../solr')
+  cmd = 'ant -Dversion=%s' % version
+  if gpgKeyID is not None:
+    cmd += ' -Dgpg.key=%s prepare-release' % gpgKeyID
+  else:
+    cmd += ' prepare-release-no-sign'
+
+  if gpgPassword is not None:
+    runAndSendGPGPassword(cmd, gpgPassword)
+  else:
+    run(cmd)
+
+  print('  done!')
+  print()
+  return rev
+
+reVersion1 = re.compile(r'\>(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?/\<', re.IGNORECASE)
+reVersion2 = re.compile(r'-(\d+)\.(\d+)\.(\d+)(-alpha|-beta)?\.zip<', re.IGNORECASE)
+reDoapRevision = re.compile(r'(\d+)\.(\d+)(?:\.(\d+))?(-alpha|-beta)?', re.IGNORECASE)
+def checkDOAPfiles(version):
+  # In Lucene and Solr DOAP files, verify presence of all releases less than the one being produced.
+  errorMessages = []
+  for product in 'lucene', 'solr':
+    url = 'https://archive.apache.org/dist/lucene/%s' % ('java' if product == 'lucene' else product)
+    distpage = load(url)
+    releases = set()
+    for regex in reVersion1, reVersion2:
+      for tup in regex.findall(distpage):
+        if tup[0] in ('1', '2'):                    # Ignore 1.X and 2.X releases
+          continue
+        releases.add(normalizeVersion(tup))
+    doapNS = '{http://usefulinc.com/ns/doap#}'
+    xpathRevision = '{0}Project/{0}release/{0}Version/{0}revision'.format(doapNS)
+    doapFile = "dev-tools/doap/%s.rdf" % product
+    treeRoot = ET.parse(doapFile).getroot()
+    doapRevisions = set()
+    for revision in treeRoot.findall(xpathRevision):
+      match = reDoapRevision.match(revision.text)
+      if (match is not None):
+        if (match.group(1) not in ('0', '1', '2')): # Ignore 0.X, 1.X and 2.X revisions
+          doapRevisions.add(normalizeVersion(match.groups()))
+      else:
+        errorMessages.append('ERROR: Failed to parse revision: %s in %s' % (revision.text, doapFile))
+    missingDoapRevisions = set()
+    for release in releases:
+      if release not in doapRevisions and release < version: # Ignore releases greater than the one being produced
+        missingDoapRevisions.add(release)
+    if len(missingDoapRevisions) > 0:
+      errorMessages.append('ERROR: Missing revision(s) in %s: %s' % (doapFile, ', '.join(sorted(missingDoapRevisions))))
+  if (len(errorMessages) > 0):
+    raise RuntimeError('\n%s\n(Hint: copy/paste from the stable branch version of the file(s).)'
+                       % '\n'.join(errorMessages))
+
+def normalizeVersion(tup):
+  suffix = ''
+  if tup[-1] is not None and tup[-1].lower() == '-alpha':
+    tup = tup[:(len(tup) - 1)]
+    suffix = '-ALPHA'
+  elif tup[-1] is not None and tup[-1].lower() == '-beta':
+    tup = tup[:(len(tup) - 1)]
+    suffix = '-BETA'
+  while tup[-1] in ('', None):
+    tup = tup[:(len(tup) - 1)]
+  while len(tup) < 3:
+    tup = tup + ('0',)
+  return '.'.join(tup) + suffix
+
+def pushLocal(version, root, rev, rcNum, localDir):
+  print('Push local [%s]...' % localDir)
+  os.makedirs(localDir)
+
+  dir = 'lucene-solr-%s-RC%d-rev%s' % (version, rcNum, rev)
+  os.makedirs('%s/%s/lucene' % (localDir, dir))
+  os.makedirs('%s/%s/solr' % (localDir, dir))
+  print('  Lucene')
+  os.chdir('%s/lucene/dist' % root)
+  print('    zip...')
+  if os.path.exists('lucene.tar.bz2'):
+    os.remove('lucene.tar.bz2')
+  run('tar cjf lucene.tar.bz2 *')
+
+  os.chdir('%s/%s/lucene' % (localDir, dir))
+  print('    unzip...')
+  run('tar xjf "%s/lucene/dist/lucene.tar.bz2"' % root)
+  os.remove('%s/lucene/dist/lucene.tar.bz2' % root)
+
+  print('  Solr')
+  os.chdir('%s/solr/package' % root)
+  print('    zip...')
+  if os.path.exists('solr.tar.bz2'):
+    os.remove('solr.tar.bz2')
+  run('tar cjf solr.tar.bz2 *')
+  print('    unzip...')
+  os.chdir('%s/%s/solr' % (localDir, dir))
+  run('tar xjf "%s/solr/package/solr.tar.bz2"' % root)
+  os.remove('%s/solr/package/solr.tar.bz2' % root)
+
+  print('  KEYS')
+  run('wget http://home.apache.org/keys/group/lucene.asc')
+  os.rename('lucene.asc', 'KEYS')
+  run('chmod a+r-w KEYS')
+  run('cp KEYS ../lucene')
+
+  print('  chmod...')
+  os.chdir('..')
+  run('chmod -R a+rX-w .')
+
+  print('  done!')
+  return 'file://%s/%s' % (os.path.abspath(localDir), dir)
+
+def read_version(path):
+  version_props_file = os.path.join(path, 'lucene', 'version.properties')
+  return re.search(r'version\.base=(.*)', open(version_props_file).read()).group(1)
+
+def parse_config(ponymail):
+  epilogue = textwrap.dedent('''
+    Example usage for a Release Manager:
+    python3 -u dev-tools/scripts/buildAndPushRelease.py --push-local /tmp/releases/6.0.1 --sign 6E68DA61 --rc-num 1
+  ''')
+  if ponymail:
+    epilogue = textwrap.dedent('''
+      Example usage for a Release Manager:
+      python3 -u dev-tools/scripts/buildAndPushRelease.py 0.10
+    ''')
+  description = 'Utility to build, push, and test a release.'
+  parser = argparse.ArgumentParser(description=description, epilog=epilogue,
+                                   formatter_class=argparse.RawDescriptionHelpFormatter)
+
+  if ponymail:
+    parser.add_argument('version', type=str)
+    return parser.parse_args()
+
+  parser.add_argument('--no-prepare', dest='prepare', default=True, action='store_false',
+                      help='Use the already built release in the provided checkout')
+  parser.add_argument('--push-local', metavar='PATH',
+                      help='Push the release to the local path')
+  parser.add_argument('--sign', metavar='KEYID',
+                      help='Sign the release with the given gpg key')
+  parser.add_argument('--rc-num', metavar='NUM', type=int, default=1,
+                      help='Release Candidate number.  Default: 1')
+  parser.add_argument('--root', metavar='PATH', default='.',
+                      help='Root of Git working tree for lucene-solr.  Default: "." (the current directory)')
+  config = parser.parse_args()
+
+  if not config.prepare and config.sign:
+    parser.error('Cannot sign already built release')
+  if config.push_local is not None and os.path.exists(config.push_local):
+    parser.error('Cannot push to local path that already exists')
+  if config.rc_num <= 0:
+    parser.error('Release Candidate number must be a positive integer')
+  if not os.path.isdir(config.root):
+    parser.error('Root path "%s" is not a directory' % config.root)
+  cwd = os.getcwd()
+  os.chdir(config.root)
+  config.root = os.getcwd() # Absolutize root dir
+  if os.system('git rev-parse') or 3 != len([d for d in ('dev-tools','lucene','solr') if os.path.isdir(d)]):
+    parser.error('Root path "%s" is not a valid lucene-solr checkout' % config.root)
+  os.chdir(cwd)
+
+  config.version = read_version(config.root)
+  print('Building version: %s' % config.version)
+
+  if config.sign:
+    sys.stdout.flush()
+    import getpass
+    config.key_id = config.sign
+    config.key_password = getpass.getpass('Enter GPG keystore password: ')
+  else:
+    config.gpg_password = None
+
+  return config
+
+def check_cmdline_tools():  # Fail fast if there are cmdline tool problems
+  if os.system('git --version >/dev/null 2>/dev/null'):
+    raise RuntimeError('"git --version" returned a non-zero exit code.')
+  antVersion = os.popen('ant -version').read().strip()
+  if not antVersion.startswith('Apache Ant(TM) version 1.8') and not antVersion.startswith('Apache Ant(TM) version 1.9'):
+    raise RuntimeError('ant version is not 1.8.X: "%s"' % antVersion)
+
+def main():
+  check_cmdline_tools()
+
+  ponymail = True
+  c = parse_config(ponymail)
+
+  if ponymail:
+
+    project = 'ponymail'
+    base_branch_name = 'master'
+    release_branch_name = c.version
+    git_archive_format = 'tar.gz'
+    tarball_name = '~/'+project+'-'+release_branch_name+'.'+git_archive_format
+    tarball_name_sha256 = tarball_name + '.sha256'
+    tarball_name_sha256_asc = tarball_name_sha256 + '.asc'
+    tarball_name_asc = tarball_name + '.asc'
+
+    print('Create a new branch off '+base_branch_name+' called '+release_branch_name+' (ideally, '+base_branch_name+' is always releasable).')
+    print('  git checkout -b '+release_branch_name+' '+base_branch_name)
+
+    print('Tarball the '+release_branch_name+' branch, sans the .git directory:')
+    print('  git archive --format='+git_archive_format+' -o '+tarball_name+' HEAD')
+
+    print('Create checksums of the archive (make sure your PGP key is in our KEYS file!):')
+
+    print('  Make a checksum for the archive itself:')
+    print('    sha256sum '+tarball_name+' > '+tarball_name_sha256)
+
+    print('  Sign the archive:')
+    print('    gpg --output '+tarball_name_asc+' --sign '+tarball_name)
+    print('  OR')
+    print('    gpg --output '+tarball_name_sha256_asc+' --sign '+tarball_name_sha256)
+
+    print('Push the artefacts to https://dist.apache.org/repos/dist/dev/incubator/ponymail/ via subversion')
+    print('  svn ???')
+    return
+
+  if c.prepare:
+    rev = prepare(c.root, c.version, c.key_id, c.key_password)
+  else:
+    os.chdir(root)
+    rev = open('rev.txt', encoding='UTF-8').read()
+
+  if c.push_local:
+    url = pushLocal(c.version, c.root, rev, c.rc_num, c.push_local)
+  else:
+    url = None
+
+  if url is not None:
+    print('  URL: %s' % url)
+    print('Next run the smoker tester:')
+    p = re.compile(".*/")
+    m = p.match(sys.argv[0])
+    print('%s -u %ssmokeTestRelease.py %s' % (sys.executable, m.group(), url))
+
+if __name__ == '__main__':
+  try:
+    main()
+  except KeyboardInterrupt:
+    print('Keyboard interrupt...exiting')
+