From 53b010c17db35255b1a8288a0eeab2105334484e Mon Sep 17 00:00:00 2001
From: skyjake ' + self.text_summary() + ''
- else:
- buildDir = builder.config.EVENT_DIR
-
- tmpName = os.path.abspath(os.path.join(buildDir, 'ctmp'))
- format = '{{{{li}}}}{{{{b}}}}[[subjectline]]%s[[/subjectline]]{{{{/b}}}}' + \
- '{{{{br/}}}}by {{{{i}}}}%an{{{{/i}}}} on ' + \
- '%ai ' + \
- '{{{{a href=\\"http://deng.git.sourceforge.net/git/gitweb.cgi?' + \
- 'p=deng/deng;a=commit;h=%H\\"}}}}(show in repository){{{{/a}}}}' + \
- '{{{{blockquote}}}}%b{{{{/blockquote}}}}'
- os.system("git log %s..%s --format=\"%s\" >> %s" % (fromTag, toTag, format, tmpName))
-
- logText = unicode(file(tmpName, 'rt').read(), 'utf-8')
- logText = logText.replace(u'ä', u'ä')
- logText = logText.encode('utf-8')
- logText = logText.replace('<', '<')
- logText = logText.replace('>', '>')
- logText = logText.replace('{{{{', '<')
- logText = logText.replace('}}}}', '>')
-
- # Check that the subject lines are not too long.
- MAX_SUBJECT = 100
- pos = 0
- changeEntries = []
- while True:
- pos = logText.find('[[subjectline]]', pos)
- if pos < 0: break
- end = logText.find('[[/subjectline]]', pos)
-
- subject = logText[pos+15:end]
- extra = ''
- if len(collated(subject)) > MAX_SUBJECT:
- extra = '...' + subject[MAX_SUBJECT:] + ' '
- subject = subject[:MAX_SUBJECT] + '...'
- else:
- # If there is a single dot at the end of the subject, remove it.
- if subject[-1] == '.' and subject[-2] != '.':
- subject = subject[:-1]
-
- if subject not in changeEntries:
- changeEntries.append(subject)
-
- # Do the replace.
- logText = logText[:pos] + subject + logText[end+16:]
-
- if len(extra):
- # Place the extra bit in the blockquote.
- bq = logText.find('
'
- changes.close()
- else:
- # Append the changes to the debian package changelog.
- os.chdir(os.path.join(builder.config.DISTRIB_DIR, 'linux'))
-
- # First we need to update the version.
- build_version.find_version()
- debVersion = build_version.DOOMSDAY_VERSION_FULL_PLAIN + '-' + todays_build_tag()
-
- # Always make one entry.
- print 'Marking new version...'
- msg = 'New release: %s build %i.' % (build_version.DOOMSDAY_RELEASE_TYPE,
- Event().number())
- os.system("dch --check-dirname-level 0 -v %s \"%s\"" % (debVersion, msg))
-
- for ch in changeEntries:
- # Quote it for the command line.
- qch = ch.replace('"', '\\"').replace('!', '\\!')
- print ' *', qch
- os.system("dch --check-dirname-level 0 -a \"%s\"" % qch)
-
- os.remove(tmpName)
-
-
def create_build_event():
"""Creates and tags a new build for with today's number."""
print 'Creating a new build event.'
@@ -153,7 +39,7 @@ def create_build_event():
if prevBuild:
update_changes(prevBuild, todaysBuild)
-
+
def todays_platform_release():
"""Build today's release for the current platform."""
@@ -192,6 +78,49 @@ def todays_platform_release():
git_checkout('master')
+def update_changes(fromTag=None, toTag=None, debChanges=False):
+ """Generates the list of commits for the latest build."""
+
+ if debChanges:
+ # Make sure we have the latest changes.
+ git_pull()
+ fromTag = aptrepo_find_latest_tag()
+ toTag = 'master' # Everything up to now.
+ else:
+ # Use the two most recent builds by default.
+ if fromTag is None or toTag is None:
+ builds = builder.events_by_time()
+ if len(builds) < 2: return
+ fromTag = builds[1][1]
+ toTag = builds[0][1]
+
+ changes = builder.Changes(fromTag, toTag)
+
+ if debChanges:
+ changes.generate('deb')
+ else:
+ changes.generate('html')
+
+
+def update_debian_changelog():
+ """Updates the Debian changelog at (distrib)/debian/changelog."""
+ # Update debian changelog.
+ update_changes(None, None, True)
+
+
+def rebuild_apt_repository():
+ """Rebuilds the Apt repository by running apt-ftparchive."""
+ aptDir = builder.config.APT_REPO_DIR
+ print 'Rebuilding the apt repository in %s...' % aptDir
+
+ os.system("apt-ftparchive generate ~/Dropbox/APT/ftparchive.conf")
+ os.system("apt-ftparchive -c ~/Dropbox/APT/ftparchive-release.conf release %s/dists/unstable > %s/dists/unstable/Release" % (aptDir, aptDir))
+ os.chdir("%s/dists/unstable" % aptDir)
+ os.remove("Release.gpg")
+ os.system("gpg --output Release.gpg -ba Release")
+ os.system("~/Dropbox/Scripts/mirror-tree.py %s %s" % (aptDir, os.path.join(builder.config.EVENT_DIR, 'apt')))
+
+
def write_index_html(tag):
ev = Event(tag)
f = file(ev.filePath('index.html'), 'wt')
@@ -206,6 +135,7 @@ def write_index_html(tag):
def update_feed():
"""Generate events.rss into the event directory."""
+
feedName = os.path.join(builder.config.EVENT_DIR, "events.rss")
print "Updating feed in %s..." % feedName
@@ -241,19 +171,6 @@ def update_feed():
# Close.
print >> out, ''
print >> out, ''
-
-
-def rebuild_apt_repository():
- """Rebuilds the Apt repository by running apt-ftparchive."""
- aptDir = builder.config.APT_REPO_DIR
- print 'Rebuilding the apt repository in %s...' % aptDir
-
- os.system("apt-ftparchive generate ~/Dropbox/APT/ftparchive.conf")
- os.system("apt-ftparchive -c ~/Dropbox/APT/ftparchive-release.conf release %s/dists/unstable > %s/dists/unstable/Release" % (aptDir, aptDir))
- os.chdir("%s/dists/unstable" % aptDir)
- os.remove("Release.gpg")
- os.system("gpg --output Release.gpg -ba Release")
- os.system("~/Dropbox/Scripts/mirror-tree.py %s %s" % (aptDir, os.path.join(builder.config.EVENT_DIR, 'apt')))
def purge_apt_repository(atLeastSeconds):
@@ -296,30 +213,28 @@ def dir_cleanup():
print 'Cleanup done.'
-def update_debian_changelog():
- """Updates the Debian changelog at (distrib)/debian/changelog."""
- # Update debian changelog.
- update_changes(None, None, True)
-
-
def show_help():
"""Prints a description of each command."""
- sortedCommands = commands.keys()
- sortedCommands.sort()
- for cmd in sortedCommands:
+ for cmd in sorted_commands():
if commands[cmd].__doc__:
print "%-17s " % (cmd + ":") + commands[cmd].__doc__
else:
print cmd
+def sorted_commands():
+ sc = commands.keys()
+ sc.sort()
+ return sc
+
+
commands = {
'create': create_build_event,
'platform_release': todays_platform_release,
- 'feed': update_feed,
- 'apt': rebuild_apt_repository,
'changes': update_changes,
'debchanges': update_debian_changelog,
+ 'apt': rebuild_apt_repository,
+ 'feed': update_feed,
'purge': purge_obsolete,
'cleanup': dir_cleanup,
'help': show_help
@@ -328,9 +243,7 @@ def show_help():
if __name__ == '__main__':
if len(sys.argv) < 2:
print 'The arguments must be: (command) [args]'
- sortedCommands = commands.keys()
- sortedCommands.sort()
- print 'Commands:', string.join(sortedCommands)
+ print 'Commands:', string.join(sorted_commands())
print 'Arguments:'
print '--distrib Doomsday distrib directory'
print '--events Event directory (builds are stored here in subdirs)'
diff --git a/distrib/builder/__init__.py b/distrib/builder/__init__.py
index 87c9752ab0..fc7a932168 100644
--- a/distrib/builder/__init__.py
+++ b/distrib/builder/__init__.py
@@ -1,5 +1,6 @@
# This is a Python module containing the builder sources.
import config
from event import *
+from changes import *
#import git
#import utils
diff --git a/distrib/builder/changes.py b/distrib/builder/changes.py
new file mode 100644
index 0000000000..9d255ae723
--- /dev/null
+++ b/distrib/builder/changes.py
@@ -0,0 +1,138 @@
+import os
+import utils
+import build_version
+
+class Entry:
+ def __init__():
+ self.subject = ''
+ self.extra = ''
+ self.author = ''
+ self.date = ''
+ self.link = ''
+ self.message = ''
+
+ def setSubject(subject):
+ extra = ''
+ # Check that the subject lines are not too long.
+ if len(utils.collated(subject)) > MAX_SUBJECT:
+ extra = '...' + subject[MAX_SUBJECT:] + ' '
+ subject = subject[:MAX_SUBJECT] + '...'
+ else:
+ # If there is a single dot at the end of the subject, remove it.
+ if subject[-1] == '.' and subject[-2] != '.':
+ subject = subject[:-1]
+ self.subject = subject
+ self.extra = extra
+
+ def setMessage(message):
+ self.message = message.strip()
+ if extra:
+ self.message = extra + ' ' + self.message
+ self.message = self.message.replace('\n\n', '', pos)
- logText = logText[:bq+12] + extra + logText[bq+12:]
-
- if not debChanges:
- logText = logText.replace('\n\n', '
').replace('\n', ' ').replace('
', '')
- print >> changes, logText
-
- if not debChanges:
- print >> changes, '
').replace('\n', ' ').strip()
+
+
+class Changes:
+ def __init__(fromTag, toTag):
+ self.fromTag = fromTag
+ self.toTag = toTag
+ self.parse()
+
+ def parse():
+ tmpName = '__ctmp'
+
+ format = '[[Subject]]%s[[/Subject]]' + \
+ '[[Author]]%an[[/Author]]' + \
+ '[[Date]]%ai[[/Date]]' + \
+ '[[Link]]http://deng.git.sourceforge.net/git/gitweb.cgi?' + \
+ 'p=deng/deng;a=commit;h=%H[[/Link]]' + \
+ '[[Message]]%b[[/Message]]'
+ os.system("git log %s..%s --format=\"%s\" >> %s" % (fromTag, toTag, format, tmpName))
+
+ logText = unicode(file(tmpName, 'rt').read(), 'utf-8')
+ logText = logText.replace(u'ä', u'ä')
+ logText = logText.replace(u'ö', u'ö')
+ logText = logText.replace(u'Ä', u'Ä')
+ logText = logText.replace(u'Ö', u'Ö')
+ logText = logText.encode('utf-8')
+ logText = logText.replace('<', '<')
+ logText = logText.replace('>', '>')
+
+ os.remove(tmpName)
+
+ MAX_SUBJECT = 100
+ pos = 0
+ self.entries = []
+ self.debChangeEntries = []
+ while True:
+ entry = Entry()
+
+ pos = logText.find('[[Subject]]', pos)
+ if pos < 0: break # No more.
+ end = logText.find('[[/Subject]]', pos)
+
+ entry.setSubject(logText[pos+11:end])
+
+ # Debian changelog just gets the subjects.
+ if entry.subject not in self.debChangeEntries:
+ self.debChangeEntries.append(entry.subject)
+
+ # Author.
+ pos = logText.find('[[Author]]', pos)
+ end = logText.find('[[/Author]]', pos)
+ entry.author = logText[pos+10:end]
+
+ # Date.
+ pos = logText.find('[[Date]]', pos)
+ end = logText.find('[[/Date]]', pos)
+ entry.date = logText[pos+8:end]
+
+ # Link.
+ pos = logText.find('[[Link]]', pos)
+ end = logText.find('[[/Link]]', pos)
+ entry.link = logText[pos+8:end]
+
+ # Message.
+ pos = logText.find('[[Message]]', pos)
+ end = logText.find('[[/Message]]', pos)
+ entry.setMessage(logText[pos+11:end])
+
+ self.entries.append(entry)
+
+
+ def generate(format):
+ fromTag = self.fromTag
+ toTag = self.toTag
+
+ if format == 'html':
+ out = file(Event(toTag).filePath('changes.html'), 'wt')
+ print >> out, ''
+
+ # Write a list entry for each commit.
+ for entry in self.entries:
+ print >> out, '
'
+ out.close()
+
+ elif format == 'deb':
+ # Append the changes to the debian package changelog.
+ os.chdir(os.path.join(builder.config.DISTRIB_DIR, 'linux'))
+
+ # First we need to update the version.
+ build_version.find_version()
+ debVersion = build_version.DOOMSDAY_VERSION_FULL_PLAIN + '-' + Event().tag()
+
+ # Always make one entry.
+ print 'Marking new version...'
+ msg = 'New release: %s build %i.' % (build_version.DOOMSDAY_RELEASE_TYPE,
+ Event().number())
+ os.system("dch --check-dirname-level 0 -v %s \"%s\"" % (debVersion, msg))
+
+ for entry in self.debChangeEntries:
+ # Quote it for the command line.
+ qch = entry.replace('"', '\\"').replace('!', '\\!')
+ print ' *', qch
+ os.system("dch --check-dirname-level 0 -a \"%s\"" % qch)
diff --git a/distrib/builder/config.py b/distrib/builder/config.py
index 6b8909404c..c245191efe 100644
--- a/distrib/builder/config.py
+++ b/distrib/builder/config.py
@@ -1,7 +1,6 @@
import os
import sys
-
def get_arg(label):
"""Find the value for the command line option @a label."""
if label in sys.argv:
diff --git a/distrib/builder/event.py b/distrib/builder/event.py
index 7af4be834a..bf30517bdd 100644
--- a/distrib/builder/event.py
+++ b/distrib/builder/event.py
@@ -20,9 +20,27 @@ def __init__(build=None):
raise Exception("Event build name must begin with 'build'")
self.name = build
self.number = int(build[5:])
+
# Where the build is located.
self.buildDir = os.path.join(config.EVENT_DIR, self.name)
+ self.packages = ['doomsday', 'fmod']
+
+ # Platforms: Name File ext sys_id()
+ self.oses = [('Windows (x86)', '.exe', 'win32-32bit'),
+ ('Mac OS X 10.4+ (i386/ppc)', '.dmg', 'darwin-32bit'),
+ ('Ubuntu (x86)', 'i386.deb', 'linux2-32bit'),
+ ('Ubuntu (x86_64)', 'amd64.deb', 'linux2-64bit')]
+
+ # Prepare compiler logs present in the build dir.
+ self.compress_logs()
+
+ def package_from_filename(name):
+ if 'fmod' in name:
+ return 'fmod'
+ else:
+ return 'doomsday'
+
def tag():
return self.name
@@ -86,38 +104,38 @@ def text_summary():
return msg
+ def compress_logs():
+ if not os.path.exists(self.buildDir): return
+
+ """Combines the stdout and stderr logs for a package and compresses
+ them with gzip (requires gzip on the system path)."""
+ for package in self.packages:
+ for osName, osExt, osIdent in self.oses:
+ names = glob.glob(self.filePath('%s-*-%s.txt' % (package, osIdent)))
+ if not names: continue
+ # Join the logs into a single file.
+ combinedName = self.filePath('buildlog-%s-%s.txt' % (package, osIdent))
+ combined = file(combinedName, 'wt')
+ for n in names:
+ combined.write(file(n).read() + "\n\n")
+ # Remove the original log.
+ os.remove(n)
+ combined.close()
+ os.system('gzip -f9 %s' % combinedName)
+
def html_description(encoded=True):
"""Composes an HTML build report. Compresses any .txt logs present in
the build directory into a combined .txt.gz (one per package)."""
-
+
name = self.name
buildDir = self.buildDir
+ oses = self.oses
msg = '
' % entry.subject
+ print >> out, 'by %s on %s' % (entry.author, entry.date)
+ print >> out, '(show in repository)' % entry.link
+ print >> out, '%s
' % entry.message
+
+ print >> out, ''
msg += '
'
@@ -145,14 +163,9 @@ def html_description(encoded=True):
msg += '%s' % (config.BUILD_URI,
name, binary, binary)
- if 'fmod' in binary:
- packageName = 'fmod'
- else:
- packageName = 'doomsday'
-
# Status of the log.
- logName = 'buildlog-%s-%s.txt.gz' % (packageName, osIdent)
- logFileName = os.path.join(buildDir, logName)
+ logName = 'buildlog-%s-%s.txt.gz' % (self.package_from_filename(binary), osIdent)
+ logFileName = self.filePath(logName)
if not os.path.exists(logFileName):
msg += ''
continue
@@ -176,7 +189,7 @@ def html_description(encoded=True):
msg += 'OS Binary Logs Er/Wrn
Commits
' + file(chgFn, 'rt').read() diff --git a/distrib/builder/utils.py b/distrib/builder/utils.py index d10b2ae647..8f87f97367 100644 --- a/distrib/builder/utils.py +++ b/distrib/builder/utils.py @@ -28,13 +28,32 @@ def todays_build_tag(): return 'build' + build_number.todays_build() -def aptrepo_by_time(arch): +def deb_arch(): + os.system('dpkg --print-architecture > __debarch.tmp') + arch = file('__debarch.tmp', 'rt').read().strip() + os.remove('__debarch.tmp') + return arch + + +def aptrepo_by_time(): files = [] - for fn in os.listdir(os.path.join(config.APT_REPO_DIR, 'dists/unstable/main/binary-' + arch)): + for fn in os.listdir(os.path.join(config.APT_REPO_DIR, + 'dists/unstable/main/binary-' + deb_arch())): if fn[-4:] == '.deb': files.append(fn) return files - + + +def aptrepo_find_latest_tag(): + debs = aptrepo_by_time() + if not debs: return 'master' + arch = deb_arch() + biggest = 0 + for deb in debs: + number = int(deb[deb.find('-build')+6 : deb.find('_'+arch)]) + biggest = max(biggest, number) + return 'build' + str(biggest) + def count_log_word(fn, word): txt = unicode(gzip.open(fn).read(), 'latin1').lower()