From 53b010c17db35255b1a8288a0eeab2105334484e Mon Sep 17 00:00:00 2001 From: skyjake Date: Wed, 14 Dec 2011 17:22:08 +0200 Subject: [PATCH] Builder: Refactored changelog generator --- distrib/autobuild.py | 197 ++++++++++-------------------------- distrib/builder/__init__.py | 1 + distrib/builder/changes.py | 138 +++++++++++++++++++++++++ distrib/builder/config.py | 1 - distrib/builder/event.py | 71 +++++++------ distrib/builder/utils.py | 25 ++++- 6 files changed, 258 insertions(+), 175 deletions(-) create mode 100644 distrib/builder/changes.py diff --git a/distrib/autobuild.py b/distrib/autobuild.py index 5d9865e7f1..a0b2dcbee3 100755 --- a/distrib/autobuild.py +++ b/distrib/autobuild.py @@ -15,121 +15,7 @@ from builder.git import * from builder.utils import * - -def update_changes(fromTag=None, toTag=None, debChanges=False): - """Generates the list of commits for the latest build.""" - if debChanges: - # Make sure we have the latest changes. - git_pull() - - # Use the apt repo for determining fromTag. - os.system('dpkg --print-architecture > debarch.tmp') - arch = file('debarch.tmp', 'rt').read().strip() - os.remove('debarch.tmp') - debs = aptrepo_by_time(arch) - - biggest = 0 - for deb in debs: - number = int(deb[deb.find('-build')+6 : deb.find('_'+arch)]) - biggest = max(biggest, number) - - fromTag = 'build' + str(biggest) - toTag = 'master' # Everything up to now. - else: - # Determine automatically? - if fromTag is None or toTag is None: - builds = builder.events_by_time() - if len(builds) < 2: return - fromTag = builds[1][1] - toTag = builds[0][1] - - # Generate a changelog. - if not debChanges: - buildDir = os.path.join(builder.config.EVENT_DIR, toTag) - fn = os.path.join(buildDir, 'changes.html') - changes = file(fn, 'wt') - print >> changes, '
    ' - else: - buildDir = builder.config.EVENT_DIR - - tmpName = os.path.abspath(os.path.join(buildDir, 'ctmp')) - format = '{{{{li}}}}{{{{b}}}}[[subjectline]]%s[[/subjectline]]{{{{/b}}}}' + \ - '{{{{br/}}}}by {{{{i}}}}%an{{{{/i}}}} on ' + \ - '%ai ' + \ - '{{{{a href=\\"http://deng.git.sourceforge.net/git/gitweb.cgi?' + \ - 'p=deng/deng;a=commit;h=%H\\"}}}}(show in repository){{{{/a}}}}' + \ - '{{{{blockquote}}}}%b{{{{/blockquote}}}}' - os.system("git log %s..%s --format=\"%s\" >> %s" % (fromTag, toTag, format, tmpName)) - - logText = unicode(file(tmpName, 'rt').read(), 'utf-8') - logText = logText.replace(u'ä', u'ä') - logText = logText.encode('utf-8') - logText = logText.replace('<', '<') - logText = logText.replace('>', '>') - logText = logText.replace('{{{{', '<') - logText = logText.replace('}}}}', '>') - - # Check that the subject lines are not too long. - MAX_SUBJECT = 100 - pos = 0 - changeEntries = [] - while True: - pos = logText.find('[[subjectline]]', pos) - if pos < 0: break - end = logText.find('[[/subjectline]]', pos) - - subject = logText[pos+15:end] - extra = '' - if len(collated(subject)) > MAX_SUBJECT: - extra = '...' + subject[MAX_SUBJECT:] + ' ' - subject = subject[:MAX_SUBJECT] + '...' - else: - # If there is a single dot at the end of the subject, remove it. - if subject[-1] == '.' and subject[-2] != '.': - subject = subject[:-1] - - if subject not in changeEntries: - changeEntries.append(subject) - - # Do the replace. - logText = logText[:pos] + subject + logText[end+16:] - - if len(extra): - # Place the extra bit in the blockquote. - bq = logText.find('
    ', pos) - logText = logText[:bq+12] + extra + logText[bq+12:] - - if not debChanges: - logText = logText.replace('\n\n', '

    ').replace('\n', ' ').replace('

    ', '') - print >> changes, logText - - if not debChanges: - print >> changes, '
' - changes.close() - else: - # Append the changes to the debian package changelog. - os.chdir(os.path.join(builder.config.DISTRIB_DIR, 'linux')) - - # First we need to update the version. - build_version.find_version() - debVersion = build_version.DOOMSDAY_VERSION_FULL_PLAIN + '-' + todays_build_tag() - - # Always make one entry. - print 'Marking new version...' - msg = 'New release: %s build %i.' % (build_version.DOOMSDAY_RELEASE_TYPE, - Event().number()) - os.system("dch --check-dirname-level 0 -v %s \"%s\"" % (debVersion, msg)) - - for ch in changeEntries: - # Quote it for the command line. - qch = ch.replace('"', '\\"').replace('!', '\\!') - print ' *', qch - os.system("dch --check-dirname-level 0 -a \"%s\"" % qch) - - os.remove(tmpName) - - def create_build_event(): """Creates and tags a new build for with today's number.""" print 'Creating a new build event.' @@ -153,7 +39,7 @@ def create_build_event(): if prevBuild: update_changes(prevBuild, todaysBuild) - + def todays_platform_release(): """Build today's release for the current platform.""" @@ -192,6 +78,49 @@ def todays_platform_release(): git_checkout('master') +def update_changes(fromTag=None, toTag=None, debChanges=False): + """Generates the list of commits for the latest build.""" + + if debChanges: + # Make sure we have the latest changes. + git_pull() + fromTag = aptrepo_find_latest_tag() + toTag = 'master' # Everything up to now. + else: + # Use the two most recent builds by default. + if fromTag is None or toTag is None: + builds = builder.events_by_time() + if len(builds) < 2: return + fromTag = builds[1][1] + toTag = builds[0][1] + + changes = builder.Changes(fromTag, toTag) + + if debChanges: + changes.generate('deb') + else: + changes.generate('html') + + +def update_debian_changelog(): + """Updates the Debian changelog at (distrib)/debian/changelog.""" + # Update debian changelog. + update_changes(None, None, True) + + +def rebuild_apt_repository(): + """Rebuilds the Apt repository by running apt-ftparchive.""" + aptDir = builder.config.APT_REPO_DIR + print 'Rebuilding the apt repository in %s...' % aptDir + + os.system("apt-ftparchive generate ~/Dropbox/APT/ftparchive.conf") + os.system("apt-ftparchive -c ~/Dropbox/APT/ftparchive-release.conf release %s/dists/unstable > %s/dists/unstable/Release" % (aptDir, aptDir)) + os.chdir("%s/dists/unstable" % aptDir) + os.remove("Release.gpg") + os.system("gpg --output Release.gpg -ba Release") + os.system("~/Dropbox/Scripts/mirror-tree.py %s %s" % (aptDir, os.path.join(builder.config.EVENT_DIR, 'apt'))) + + def write_index_html(tag): ev = Event(tag) f = file(ev.filePath('index.html'), 'wt') @@ -206,6 +135,7 @@ def write_index_html(tag): def update_feed(): """Generate events.rss into the event directory.""" + feedName = os.path.join(builder.config.EVENT_DIR, "events.rss") print "Updating feed in %s..." % feedName @@ -241,19 +171,6 @@ def update_feed(): # Close. print >> out, '' print >> out, '' - - -def rebuild_apt_repository(): - """Rebuilds the Apt repository by running apt-ftparchive.""" - aptDir = builder.config.APT_REPO_DIR - print 'Rebuilding the apt repository in %s...' % aptDir - - os.system("apt-ftparchive generate ~/Dropbox/APT/ftparchive.conf") - os.system("apt-ftparchive -c ~/Dropbox/APT/ftparchive-release.conf release %s/dists/unstable > %s/dists/unstable/Release" % (aptDir, aptDir)) - os.chdir("%s/dists/unstable" % aptDir) - os.remove("Release.gpg") - os.system("gpg --output Release.gpg -ba Release") - os.system("~/Dropbox/Scripts/mirror-tree.py %s %s" % (aptDir, os.path.join(builder.config.EVENT_DIR, 'apt'))) def purge_apt_repository(atLeastSeconds): @@ -296,30 +213,28 @@ def dir_cleanup(): print 'Cleanup done.' -def update_debian_changelog(): - """Updates the Debian changelog at (distrib)/debian/changelog.""" - # Update debian changelog. - update_changes(None, None, True) - - def show_help(): """Prints a description of each command.""" - sortedCommands = commands.keys() - sortedCommands.sort() - for cmd in sortedCommands: + for cmd in sorted_commands(): if commands[cmd].__doc__: print "%-17s " % (cmd + ":") + commands[cmd].__doc__ else: print cmd +def sorted_commands(): + sc = commands.keys() + sc.sort() + return sc + + commands = { 'create': create_build_event, 'platform_release': todays_platform_release, - 'feed': update_feed, - 'apt': rebuild_apt_repository, 'changes': update_changes, 'debchanges': update_debian_changelog, + 'apt': rebuild_apt_repository, + 'feed': update_feed, 'purge': purge_obsolete, 'cleanup': dir_cleanup, 'help': show_help @@ -328,9 +243,7 @@ def show_help(): if __name__ == '__main__': if len(sys.argv) < 2: print 'The arguments must be: (command) [args]' - sortedCommands = commands.keys() - sortedCommands.sort() - print 'Commands:', string.join(sortedCommands) + print 'Commands:', string.join(sorted_commands()) print 'Arguments:' print '--distrib Doomsday distrib directory' print '--events Event directory (builds are stored here in subdirs)' diff --git a/distrib/builder/__init__.py b/distrib/builder/__init__.py index 87c9752ab0..fc7a932168 100644 --- a/distrib/builder/__init__.py +++ b/distrib/builder/__init__.py @@ -1,5 +1,6 @@ # This is a Python module containing the builder sources. import config from event import * +from changes import * #import git #import utils diff --git a/distrib/builder/changes.py b/distrib/builder/changes.py new file mode 100644 index 0000000000..9d255ae723 --- /dev/null +++ b/distrib/builder/changes.py @@ -0,0 +1,138 @@ +import os +import utils +import build_version + +class Entry: + def __init__(): + self.subject = '' + self.extra = '' + self.author = '' + self.date = '' + self.link = '' + self.message = '' + + def setSubject(subject): + extra = '' + # Check that the subject lines are not too long. + if len(utils.collated(subject)) > MAX_SUBJECT: + extra = '...' + subject[MAX_SUBJECT:] + ' ' + subject = subject[:MAX_SUBJECT] + '...' + else: + # If there is a single dot at the end of the subject, remove it. + if subject[-1] == '.' and subject[-2] != '.': + subject = subject[:-1] + self.subject = subject + self.extra = extra + + def setMessage(message): + self.message = message.strip() + if extra: + self.message = extra + ' ' + self.message + self.message = self.message.replace('\n\n', '

').replace('\n', ' ').strip() + + +class Changes: + def __init__(fromTag, toTag): + self.fromTag = fromTag + self.toTag = toTag + self.parse() + + def parse(): + tmpName = '__ctmp' + + format = '[[Subject]]%s[[/Subject]]' + \ + '[[Author]]%an[[/Author]]' + \ + '[[Date]]%ai[[/Date]]' + \ + '[[Link]]http://deng.git.sourceforge.net/git/gitweb.cgi?' + \ + 'p=deng/deng;a=commit;h=%H[[/Link]]' + \ + '[[Message]]%b[[/Message]]' + os.system("git log %s..%s --format=\"%s\" >> %s" % (fromTag, toTag, format, tmpName)) + + logText = unicode(file(tmpName, 'rt').read(), 'utf-8') + logText = logText.replace(u'ä', u'ä') + logText = logText.replace(u'ö', u'ö') + logText = logText.replace(u'Ä', u'Ä') + logText = logText.replace(u'Ö', u'Ö') + logText = logText.encode('utf-8') + logText = logText.replace('<', '<') + logText = logText.replace('>', '>') + + os.remove(tmpName) + + MAX_SUBJECT = 100 + pos = 0 + self.entries = [] + self.debChangeEntries = [] + while True: + entry = Entry() + + pos = logText.find('[[Subject]]', pos) + if pos < 0: break # No more. + end = logText.find('[[/Subject]]', pos) + + entry.setSubject(logText[pos+11:end]) + + # Debian changelog just gets the subjects. + if entry.subject not in self.debChangeEntries: + self.debChangeEntries.append(entry.subject) + + # Author. + pos = logText.find('[[Author]]', pos) + end = logText.find('[[/Author]]', pos) + entry.author = logText[pos+10:end] + + # Date. + pos = logText.find('[[Date]]', pos) + end = logText.find('[[/Date]]', pos) + entry.date = logText[pos+8:end] + + # Link. + pos = logText.find('[[Link]]', pos) + end = logText.find('[[/Link]]', pos) + entry.link = logText[pos+8:end] + + # Message. + pos = logText.find('[[Message]]', pos) + end = logText.find('[[/Message]]', pos) + entry.setMessage(logText[pos+11:end]) + + self.entries.append(entry) + + + def generate(format): + fromTag = self.fromTag + toTag = self.toTag + + if format == 'html': + out = file(Event(toTag).filePath('changes.html'), 'wt') + print >> out, '
    ' + + # Write a list entry for each commit. + for entry in self.entries: + print >> out, '
  1. %s
    ' % entry.subject + print >> out, 'by %s on %s' % (entry.author, entry.date) + print >> out, '(show in repository)' % entry.link + print >> out, '
    %s
    ' % entry.message + + print >> out, '
' + out.close() + + elif format == 'deb': + # Append the changes to the debian package changelog. + os.chdir(os.path.join(builder.config.DISTRIB_DIR, 'linux')) + + # First we need to update the version. + build_version.find_version() + debVersion = build_version.DOOMSDAY_VERSION_FULL_PLAIN + '-' + Event().tag() + + # Always make one entry. + print 'Marking new version...' + msg = 'New release: %s build %i.' % (build_version.DOOMSDAY_RELEASE_TYPE, + Event().number()) + os.system("dch --check-dirname-level 0 -v %s \"%s\"" % (debVersion, msg)) + + for entry in self.debChangeEntries: + # Quote it for the command line. + qch = entry.replace('"', '\\"').replace('!', '\\!') + print ' *', qch + os.system("dch --check-dirname-level 0 -a \"%s\"" % qch) diff --git a/distrib/builder/config.py b/distrib/builder/config.py index 6b8909404c..c245191efe 100644 --- a/distrib/builder/config.py +++ b/distrib/builder/config.py @@ -1,7 +1,6 @@ import os import sys - def get_arg(label): """Find the value for the command line option @a label.""" if label in sys.argv: diff --git a/distrib/builder/event.py b/distrib/builder/event.py index 7af4be834a..bf30517bdd 100644 --- a/distrib/builder/event.py +++ b/distrib/builder/event.py @@ -20,9 +20,27 @@ def __init__(build=None): raise Exception("Event build name must begin with 'build'") self.name = build self.number = int(build[5:]) + # Where the build is located. self.buildDir = os.path.join(config.EVENT_DIR, self.name) + self.packages = ['doomsday', 'fmod'] + + # Platforms: Name File ext sys_id() + self.oses = [('Windows (x86)', '.exe', 'win32-32bit'), + ('Mac OS X 10.4+ (i386/ppc)', '.dmg', 'darwin-32bit'), + ('Ubuntu (x86)', 'i386.deb', 'linux2-32bit'), + ('Ubuntu (x86_64)', 'amd64.deb', 'linux2-64bit')] + + # Prepare compiler logs present in the build dir. + self.compress_logs() + + def package_from_filename(name): + if 'fmod' in name: + return 'fmod' + else: + return 'doomsday' + def tag(): return self.name @@ -86,38 +104,38 @@ def text_summary(): return msg + def compress_logs(): + if not os.path.exists(self.buildDir): return + + """Combines the stdout and stderr logs for a package and compresses + them with gzip (requires gzip on the system path).""" + for package in self.packages: + for osName, osExt, osIdent in self.oses: + names = glob.glob(self.filePath('%s-*-%s.txt' % (package, osIdent))) + if not names: continue + # Join the logs into a single file. + combinedName = self.filePath('buildlog-%s-%s.txt' % (package, osIdent)) + combined = file(combinedName, 'wt') + for n in names: + combined.write(file(n).read() + "\n\n") + # Remove the original log. + os.remove(n) + combined.close() + os.system('gzip -f9 %s' % combinedName) + def html_description(encoded=True): """Composes an HTML build report. Compresses any .txt logs present in the build directory into a combined .txt.gz (one per package).""" - + name = self.name buildDir = self.buildDir + oses = self.oses msg = '

' + self.text_summary() + '

' # What do we have here? files = self.list_package_files() - oses = [('Windows (x86)', '.exe', 'win32-32bit'), - ('Mac OS X 10.4+ (i386/ppc)', '.dmg', 'darwin-32bit'), - ('Ubuntu (x86)', 'i386.deb', 'linux2-32bit'), - ('Ubuntu (x86_64)', 'amd64.deb', 'linux2-64bit')] - - # Prepare compiler logs. - for package in ['doomsday', 'fmod']: - for osName, osExt, osIdent in oses: - names = glob.glob(os.path.join(buildDir, '%s-*-%s.txt' % (package, osIdent))) - if not names: continue - # Join the logs into a single file. - combinedName = os.path.join(buildDir, 'buildlog-%s-%s.txt' % (package, osIdent)) - combined = file(combinedName, 'wt') - for n in names: - combined.write(file(n).read() + "\n\n") - # Remove the original log. - os.remove(n) - combined.close() - os.system('gzip -f9 %s' % combinedName) - # Print out the matrix. msg += '

' msg += '' @@ -145,14 +163,9 @@ def html_description(encoded=True): msg += '%s' % (config.BUILD_URI, name, binary, binary) - if 'fmod' in binary: - packageName = 'fmod' - else: - packageName = 'doomsday' - # Status of the log. - logName = 'buildlog-%s-%s.txt.gz' % (packageName, osIdent) - logFileName = os.path.join(buildDir, logName) + logName = 'buildlog-%s-%s.txt.gz' % (self.package_from_filename(binary), osIdent) + logFileName = self.filePath(logName) if not os.path.exists(logFileName): msg += '' continue @@ -176,7 +189,7 @@ def html_description(encoded=True): msg += '
OSBinaryLogsEr/Wrn

' # Changes. - chgFn = os.path.join(buildDir, 'changes.html') + chgFn = self.filePath('changes.html') if os.path.exists(chgFn): if utils.count_word('
  • ', file(chgFn).read()): msg += '

    Commits

    ' + file(chgFn, 'rt').read() diff --git a/distrib/builder/utils.py b/distrib/builder/utils.py index d10b2ae647..8f87f97367 100644 --- a/distrib/builder/utils.py +++ b/distrib/builder/utils.py @@ -28,13 +28,32 @@ def todays_build_tag(): return 'build' + build_number.todays_build() -def aptrepo_by_time(arch): +def deb_arch(): + os.system('dpkg --print-architecture > __debarch.tmp') + arch = file('__debarch.tmp', 'rt').read().strip() + os.remove('__debarch.tmp') + return arch + + +def aptrepo_by_time(): files = [] - for fn in os.listdir(os.path.join(config.APT_REPO_DIR, 'dists/unstable/main/binary-' + arch)): + for fn in os.listdir(os.path.join(config.APT_REPO_DIR, + 'dists/unstable/main/binary-' + deb_arch())): if fn[-4:] == '.deb': files.append(fn) return files - + + +def aptrepo_find_latest_tag(): + debs = aptrepo_by_time() + if not debs: return 'master' + arch = deb_arch() + biggest = 0 + for deb in debs: + number = int(deb[deb.find('-build')+6 : deb.find('_'+arch)]) + biggest = max(biggest, number) + return 'build' + str(biggest) + def count_log_word(fn, word): txt = unicode(gzip.open(fn).read(), 'latin1').lower()