diff --git a/master/buildbot/db/logs.py b/master/buildbot/db/logs.py index 85e51e87c27..620dccbf6cc 100644 --- a/master/buildbot/db/logs.py +++ b/master/buildbot/db/logs.py @@ -83,7 +83,7 @@ def getLogBySlug(self, stepid, slug): tbl = self.db.model.logs return self._getLog((tbl.c.slug == slug) & (tbl.c.stepid == stepid)) - def getLogs(self, stepid): + def getLogs(self, stepid=None): def thd(conn): tbl = self.db.model.logs q = tbl.select() @@ -241,19 +241,18 @@ def compressLog(self, logid): def thd(conn): # get the set of chunks tbl = self.db.model.logchunks - q = sa.select([tbl.c.first_line, tbl.c.last_line, sa.func.length(tbl.c.content), tbl.c.compressed]) + q = sa.select([tbl.c.first_line, tbl.c.last_line, sa.func.length(tbl.c.content), + tbl.c.compressed]) q = q.where(tbl.c.logid == logid) q = q.order_by(tbl.c.first_line) rows = conn.execute(q) uncompressed_length = 0 numchunks = 0 totlength = 0 - totlines = 0 for row in rows: if row.compressed == 0: uncompressed_length += row.length_1 totlength += row.length_1 - totlines = row.last_line - row.first_line numchunks += 1 # do nothing if its not worth. @@ -265,7 +264,7 @@ def thd(conn): rows = conn.execute(q) wholelog = "" for row in rows: - wholelog += self.COMPRESSION_BYID[row.compressed](row.content).decode('utf-8') + "\n" + wholelog += self.COMPRESSION_BYID[row.compressed]["read"](row.content).decode('utf-8') + "\n" d = tbl.delete() d = d.where(tbl.c.logid == logid) diff --git a/master/buildbot/scripts/base.py b/master/buildbot/scripts/base.py index e87e1a90f8e..417011912f7 100644 --- a/master/buildbot/scripts/base.py +++ b/master/buildbot/scripts/base.py @@ -17,11 +17,79 @@ import copy import os import stat +import sys +import traceback from twisted.python import runtime + +from buildbot import config as config_module +from contextlib import contextmanager +from twisted.internet import defer from twisted.python import usage +@contextmanager +def captureErrors(errors, msg): + try: + yield + except errors as e: + print(msg) + print(e) + defer.returnValue(1) + + +def checkBasedir(config): + if not config['quiet']: + print("checking basedir") + + if not isBuildmasterDir(config['basedir']): + return False + + if runtime.platformType != 'win32': # no pids on win32 + if not config['quiet']: + print("checking for running master") + pidfile = os.path.join(config['basedir'], 'twistd.pid') + if os.path.exists(pidfile): + print("'%s' exists - is this master still running?" % (pidfile,)) + return False + + tac = getConfigFromTac(config['basedir']) + if tac: + if isinstance(tac.get('rotateLength', 0), str): + print("ERROR: rotateLength is a string, it should be a number") + print("ERROR: Please, edit your buildbot.tac file and run again") + print("ERROR: See http://trac.buildbot.net/ticket/2588 for more details") + return False + if isinstance(tac.get('maxRotatedFiles', 0), str): + print("ERROR: maxRotatedFiles is a string, it should be a number") + print("ERROR: Please, edit your buildbot.tac file and run again") + print("ERROR: See http://trac.buildbot.net/ticket/2588 for more details") + return False + + return True + + +def loadConfig(config, configFileName='master.cfg'): + if not config['quiet']: + print("checking %s" % configFileName) + + try: + master_cfg = config_module.MasterConfig.loadConfig( + config['basedir'], configFileName) + except config_module.ConfigErrors as e: + print("Errors loading configuration:") + + for msg in e.errors: + print(" " + msg) + return + except Exception: + print("Errors loading configuration:") + traceback.print_exc(file=sys.stdout) + return + + return master_cfg + + def isBuildmasterDir(dir): def print_error(error_message): print("%s\ninvalid buildmaster directory '%s'" % (error_message, dir)) diff --git a/master/buildbot/scripts/cleanupdb.py b/master/buildbot/scripts/cleanupdb.py index 79dfe4d9af6..19ea6201465 100644 --- a/master/buildbot/scripts/cleanupdb.py +++ b/master/buildbot/scripts/cleanupdb.py @@ -13,14 +13,13 @@ # # Copyright Buildbot Team Members +from __future__ import print_function from __future__ import with_statement import os import sys -import time -from .upgrade_master import checkBasedir -from .upgrade_master import loadConfig +from buildbot import config as config_module from buildbot import monkeypatches from buildbot.db import connector from buildbot.master import BuildMaster @@ -32,11 +31,11 @@ @defer.inlineCallbacks def doCleanupDatabase(config, master_cfg): if not config['quiet']: - print "cleaning database (%s)" % (master_cfg.db['db_url']) + print("cleaning database (%s)" % (master_cfg.db['db_url'])) master = BuildMaster(config['basedir']) master.config = master_cfg - print master.config.logCompressionMethod + print(master.config.logCompressionMethod) db = connector.DBConnector(master, basedir=config['basedir']) yield db.setup(check_version=False, verbose=not config['quiet']) @@ -49,14 +48,16 @@ def doCleanupDatabase(config, master_cfg): i += 1 if not config['quiet'] and percent != i * 100 / len(res): percent = i * 100 / len(res) - print " {}% {} saved".format(percent, saved) + print(" {0}% {1} saved".format(percent, saved)) saved = 0 sys.stdout.flush() if master_cfg.db['db_url'].startswith("sqlite"): if not config['quiet']: - print "executing sqlite vacuum function..." + print("executing sqlite vacuum function...") + # sqlite vacuum function rebuild the whole database to claim + # free disk space back def thd(engine): r = engine.execute("vacuum;") r.close() @@ -64,25 +65,32 @@ def thd(engine): @in_reactor -@defer.inlineCallbacks -def cleanupDatabase(config, _noMonkey=False): - if not _noMonkey: # pragma: no cover +def cleanupDatabase(config, _noMonkey=False): # pragma: no cover + # we separate the actual implementation to protect unit tests + # from @in_reactor which stops the reactor + if not _noMonkey: monkeypatches.patch_all() + return _cleanupDatabase(config, _noMonkey=False) + + +@defer.inlineCallbacks +def _cleanupDatabase(config, _noMonkey=False): - if not checkBasedir(config): + if not base.checkBasedir(config): defer.returnValue(1) return + config['basedir'] = os.path.abspath(config['basedir']) os.chdir(config['basedir']) - try: + with base.captureErrors((SyntaxError, ImportError), + "Unable to load 'buildbot.tac' from '%s':" % (config['basedir'],)): configFile = base.getConfigFileFromTac(config['basedir']) - except (SyntaxError, ImportError), e: - print "Unable to load 'buildbot.tac' from '%s':" % config['basedir'] - print e - defer.returnValue(1) - return - master_cfg = loadConfig(config, configFile) + + with base.captureErrors(config_module.ConfigErrors, + "Unable to load '%s' from '%s':" % (configFile, config['basedir'])): + master_cfg = base.loadConfig(config, configFile) + if not master_cfg: defer.returnValue(1) return @@ -90,6 +98,6 @@ def cleanupDatabase(config, _noMonkey=False): yield doCleanupDatabase(config, master_cfg) if not config['quiet']: - print "cleanup complete" + print("cleanup complete") defer.returnValue(0) diff --git a/master/buildbot/scripts/runner.py b/master/buildbot/scripts/runner.py index 85c6c5eb121..59c2c6dca0d 100644 --- a/master/buildbot/scripts/runner.py +++ b/master/buildbot/scripts/runner.py @@ -656,6 +656,8 @@ class CleanupDBOptions(base.BasedirMixin, base.SubcommandOptions): subcommandFunction = "buildbot.scripts.cleanupdb.cleanupDatabase" optFlags = [ ["quiet", "q", "Do not emit the commands being run"], + # when this command has several maintainance jobs, we should make + # them optional here. For now there is only one. ] optParameters = [ ] @@ -665,7 +667,12 @@ def getSynopsis(self): longdesc = """ This command takes an existing buildmaster working directory and - do some optimization on the database + do some optimization on the database. + + This command is frontend for various database maintainance jobs: + + - optimiselogs: This optimization groups logs into bigger chunks + to apply higher level of compression. This command uses the database specified in the master configuration file. If you wish to use a database other than diff --git a/master/buildbot/scripts/upgrade_master.py b/master/buildbot/scripts/upgrade_master.py index db6a9b4a2e0..1e3829cbddb 100644 --- a/master/buildbot/scripts/upgrade_master.py +++ b/master/buildbot/scripts/upgrade_master.py @@ -15,72 +15,16 @@ from __future__ import print_function import os -import sys -import traceback -from buildbot import config as config_module from buildbot import monkeypatches from buildbot.db import connector from buildbot.master import BuildMaster from buildbot.scripts import base from buildbot.util import in_reactor from twisted.internet import defer -from twisted.python import runtime from twisted.python import util -def checkBasedir(config): - if not config['quiet']: - print("checking basedir") - - if not base.isBuildmasterDir(config['basedir']): - return False - - if runtime.platformType != 'win32': # no pids on win32 - if not config['quiet']: - print("checking for running master") - pidfile = os.path.join(config['basedir'], 'twistd.pid') - if os.path.exists(pidfile): - print("'%s' exists - is this master still running?" % (pidfile,)) - return False - - tac = base.getConfigFromTac(config['basedir']) - if tac: - if isinstance(tac.get('rotateLength', 0), str): - print("ERROR: rotateLength is a string, it should be a number") - print("ERROR: Please, edit your buildbot.tac file and run again") - print("ERROR: See http://trac.buildbot.net/ticket/2588 for more details") - return False - if isinstance(tac.get('maxRotatedFiles', 0), str): - print("ERROR: maxRotatedFiles is a string, it should be a number") - print("ERROR: Please, edit your buildbot.tac file and run again") - print("ERROR: See http://trac.buildbot.net/ticket/2588 for more details") - return False - - return True - - -def loadConfig(config, configFileName='master.cfg'): - if not config['quiet']: - print("checking %s" % configFileName) - - try: - master_cfg = config_module.MasterConfig.loadConfig( - config['basedir'], configFileName) - except config_module.ConfigErrors as e: - print("Errors loading configuration:") - - for msg in e.errors: - print(" " + msg) - return - except Exception: - print("Errors loading configuration:") - traceback.print_exc(file=sys.stdout) - return - - return master_cfg - - def installFile(config, target, source, overwrite=False): with open(source, "rt") as f: new_contents = f.read() @@ -139,7 +83,7 @@ def upgradeFiles(config): try: print("Notice: Moving %s to %s." % (index_html, root_html)) print(" You can (and probably want to) remove it if " - "you haven't modified this file.") + "you haven't modified this file.") os.renames(index_html, root_html) except Exception as e: print("Error moving %s to %s: %s" % (index_html, root_html, @@ -166,7 +110,7 @@ def upgradeMaster(config, _noMonkey=False): if not _noMonkey: # pragma: no cover monkeypatches.patch_all() - if not checkBasedir(config): + if not base.checkBasedir(config): defer.returnValue(1) return @@ -180,7 +124,7 @@ def upgradeMaster(config, _noMonkey=False): defer.returnValue(1) return - master_cfg = loadConfig(config, configFile) + master_cfg = base.loadConfig(config, configFile) if not master_cfg: defer.returnValue(1) return diff --git a/master/buildbot/test/fake/fakedb.py b/master/buildbot/test/fake/fakedb.py index 05696447cae..17afc9ea644 100644 --- a/master/buildbot/test/fake/fakedb.py +++ b/master/buildbot/test/fake/fakedb.py @@ -2042,7 +2042,7 @@ def getLogBySlug(self, stepid, slug): return defer.succeed(None) return defer.succeed(self._row2dict(row)) - def getLogs(self, stepid): + def getLogs(self, stepid=None): return defer.succeed([ self._row2dict(row) for row in self.logs.itervalues() diff --git a/master/buildbot/test/unit/test_db_logs.py b/master/buildbot/test/unit/test_db_logs.py index 1f3f572e4a1..f68b348d887 100644 --- a/master/buildbot/test/unit/test_db_logs.py +++ b/master/buildbot/test/unit/test_db_logs.py @@ -103,7 +103,7 @@ def getLogBySlug(self, stepid, slug): def test_signature_getLogs(self): @self.assertArgSpecMatches(self.db.logs.getLogs) - def getLogs(self, stepid): + def getLogs(self, stepid=None): pass def test_signature_getLogLines(self): diff --git a/master/buildbot/test/unit/test_scripts_base.py b/master/buildbot/test/unit/test_scripts_base.py index 057c732eb33..fb99f6d8a4a 100644 --- a/master/buildbot/test/unit/test_scripts_base.py +++ b/master/buildbot/test/unit/test_scripts_base.py @@ -17,9 +17,11 @@ import string import textwrap +from buildbot import config as config_module from buildbot.scripts import base from buildbot.test.util import dirs from buildbot.test.util import misc +from buildbot.test.util.decorators import skipUnlessPlatformIs from twisted.python import runtime from twisted.python import usage from twisted.trial import unittest @@ -267,3 +269,92 @@ def test_loadOptionsFile_toomany(self): # NOTE: testing the ownership check requires patching os.stat, which causes # other problems since it is so heavily used. + + +def mkconfig(**kwargs): + config = dict(quiet=False, replace=False, basedir='test') + config.update(kwargs) + return config + + +class TestLoadConfig(dirs.DirsMixin, misc.StdoutAssertionsMixin, + unittest.TestCase): + + def setUp(self): + self.setUpDirs('test') + self.setUpStdoutAssertions() + + def tearDown(self): + self.tearDownDirs() + + def activeBasedir(self, extra_lines=()): + with open(os.path.join('test', 'buildbot.tac'), 'wt') as f: + f.write("from twisted.application import service\n") + f.write("service.Application('buildmaster')\n") + f.write("\n".join(extra_lines)) + + def test_checkBasedir(self): + self.activeBasedir() + rv = base.checkBasedir(mkconfig()) + self.assertTrue(rv) + self.assertInStdout('checking basedir') + + def test_checkBasedir_quiet(self): + self.activeBasedir() + rv = base.checkBasedir(mkconfig(quiet=True)) + self.assertTrue(rv) + self.assertWasQuiet() + + def test_checkBasedir_no_dir(self): + rv = base.checkBasedir(mkconfig(basedir='doesntexist')) + self.assertFalse(rv) + self.assertInStdout('invalid buildmaster directory') + + @skipUnlessPlatformIs('posix') + def test_checkBasedir_active_pidfile(self): + self.activeBasedir() + open(os.path.join('test', 'twistd.pid'), 'w').close() + rv = base.checkBasedir(mkconfig()) + self.assertFalse(rv) + self.assertInStdout('still running') + + def test_checkBasedir_invalid_rotateLength(self): + self.activeBasedir(extra_lines=['rotateLength="32"']) + rv = base.checkBasedir(mkconfig()) + self.assertFalse(rv) + self.assertInStdout('ERROR') + self.assertInStdout('rotateLength') + + def test_checkBasedir_invalid_maxRotatedFiles(self): + self.activeBasedir(extra_lines=['maxRotatedFiles="64"']) + rv = base.checkBasedir(mkconfig()) + self.assertFalse(rv) + self.assertInStdout('ERROR') + self.assertInStdout('maxRotatedFiles') + + def test_loadConfig(self): + @classmethod + def loadConfig(cls, basedir, filename): + return config_module.MasterConfig() + self.patch(config_module.MasterConfig, 'loadConfig', loadConfig) + cfg = base.loadConfig(mkconfig()) + self.assertIsInstance(cfg, config_module.MasterConfig) + self.assertInStdout('checking') + + def test_loadConfig_ConfigErrors(self): + @classmethod + def loadConfig(cls, basedir, filename): + raise config_module.ConfigErrors(['oh noes']) + self.patch(config_module.MasterConfig, 'loadConfig', loadConfig) + cfg = base.loadConfig(mkconfig()) + self.assertIdentical(cfg, None) + self.assertInStdout('oh noes') + + def test_loadConfig_exception(self): + @classmethod + def loadConfig(cls, basedir, filename): + raise RuntimeError() + self.patch(config_module.MasterConfig, 'loadConfig', loadConfig) + cfg = base.loadConfig(mkconfig()) + self.assertIdentical(cfg, None) + self.assertInStdout('RuntimeError') diff --git a/master/buildbot/test/unit/test_scripts_cleanupdb.py b/master/buildbot/test/unit/test_scripts_cleanupdb.py new file mode 100644 index 00000000000..3fa866c0226 --- /dev/null +++ b/master/buildbot/test/unit/test_scripts_cleanupdb.py @@ -0,0 +1,150 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members +import os +import sqlalchemy as sa +import textwrap + +from twisted.internet import defer +from twisted.trial import unittest + +import test_db_logs + +from buildbot.db.connector import DBConnector +from buildbot.scripts import cleanupdb +from buildbot.test.fake import fakemaster +from buildbot.test.util import db +from buildbot.test.util import dirs +from buildbot.test.util import misc +try: + import lz4 + [lz4] + hasLz4 = True +except ImportError: + hasLz4 = False + + +def mkconfig(**kwargs): + config = dict(quiet=False, basedir=os.path.abspath('basedir')) + config.update(kwargs) + return config + + +class TestCleanupDb(misc.StdoutAssertionsMixin, dirs.DirsMixin, + db.RealDatabaseMixin, unittest.TestCase): + + def setUp(self): + self.origcwd = os.getcwd() + self.setUpDirs('basedir') + with open(os.path.join('basedir', 'buildbot.tac'), 'wt') as f: + f.write(textwrap.dedent(""" + from twisted.application import service + application = service.Application('buildmaster') + """)) + self.setUpStdoutAssertions() + + def tearDown(self): + os.chdir(self.origcwd) + self.tearDownDirs() + + def createMasterCfg(self, extraconfig=""): + os.chdir(self.origcwd) + with open(os.path.join('basedir', 'master.cfg'), 'wt') as f: + f.write(textwrap.dedent(""" + from buildbot.plugins import * + c = BuildmasterConfig = dict() + c['db_url'] = "{dburl}" + c['multiMaster'] = True # dont complain for no builders + {extraconfig} + """.format(dburl=os.environ.get("BUILDBOT_TEST_DB_URL"), + extraconfig=extraconfig))) + + @defer.inlineCallbacks + def test_cleanup_not_basedir(self): + res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='doesntexist')) + self.assertEqual(res, 1) + self.assertInStdout('invalid buildmaster directory') + + @defer.inlineCallbacks + def test_cleanup_bad_config(self): + res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='basedir')) + self.assertEqual(res, 1) + self.assertInStdout("master.cfg' does not exist") + + @defer.inlineCallbacks + def test_cleanup_bad_config2(self): + self.createMasterCfg(extraconfig="++++ # syntaxerror") + res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='basedir')) + self.assertEqual(res, 1) + self.assertInStdout("error while parsing config") + # config logs an error via log.err, we must eat it or trial will complain + self.flushLoggedErrors() + + @defer.inlineCallbacks + def test_cleanup(self): + + # test may use mysql or pg if configured in env + if "BUILDBOT_TEST_DB_URL" not in os.environ: + os.environ["BUILDBOT_TEST_DB_URL"] = "sqlite:///" + os.path.join(self.origcwd, + "basedir", "state.sqlite") + # we reuse RealDatabaseMixin to setup the db + yield self.setUpRealDatabase(table_names=['logs', 'logchunks', 'steps', 'builds', 'builders', + 'masters', 'buildrequests', 'buildsets', + 'buildslaves']) + master = fakemaster.make_master() + master.config.db['db_url'] = self.db_url + self.db = DBConnector(master, self.basedir) + self.db.pool = self.db_pool + + # we reuse the fake db background data from db.logs unit tests + yield self.insertTestData(test_db_logs.Tests.backgroundData) + + # insert a log with lots of redundancy + LOGDATA = "xx\n" * 2000 + logid = yield self.db.logs.addLog(102, "x", "x", "s") + yield self.db.logs.appendLog(logid, LOGDATA) + + # test all methods + lengths = {} + for mode in self.db.logs.COMPRESSION_MODE.keys(): + if mode == "lz4" and not hasLz4: + # ok.. lz4 is not installed, dont fail + lengths["lz4"] = 40 + continue + # create a master.cfg with different compression method + self.createMasterCfg("c['logCompressionMethod'] = '%s'" % (mode,)) + res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='basedir')) + self.assertEqual(res, 0) + + # make sure the compression don't change the data we can retrieve via api + res = yield self.db.logs.getLogLines(logid, 0, 2000) + self.assertEqual(res, LOGDATA) + + # retrieve the actual data size in db using raw sqlalchemy + def thd(conn): + tbl = self.db.model.logchunks + q = sa.select([tbl.c.content]) + q = q.where(tbl.c.logid == logid) + return sum([len(row.content) for row in conn.execute(q)]) + lengths[mode] = yield self.db.pool.do(thd) + + self.assertDictAlmostEqual(lengths, {'raw': 5999, 'bz2': 44, 'lz4': 40, 'gz': 31}) + + def assertDictAlmostEqual(self, d1, d2): + # The test shows each methods return different size + # but we still make a fuzzy comparaison to resist if underlying libraries + # improve efficiency + self.assertEqual(len(d1), len(d2)) + for k in d2.keys(): + self.assertApproximates(d1[k], d2[k], 10) diff --git a/master/buildbot/test/unit/test_scripts_upgrade_master.py b/master/buildbot/test/unit/test_scripts_upgrade_master.py index c78c6231b37..8d500234122 100644 --- a/master/buildbot/test/unit/test_scripts_upgrade_master.py +++ b/master/buildbot/test/unit/test_scripts_upgrade_master.py @@ -19,11 +19,11 @@ from buildbot.db import connector from buildbot.db import masters from buildbot.db import model +from buildbot.scripts import base from buildbot.scripts import upgrade_master from buildbot.test.util import dirs from buildbot.test.util import misc from buildbot.test.util import www -from buildbot.test.util.decorators import skipUnlessPlatformIs from twisted.internet import defer from twisted.trial import unittest @@ -51,12 +51,12 @@ def patchFunctions(self, basedirOk=True, configOk=True): def checkBasedir(config): self.calls.append('checkBasedir') return basedirOk - self.patch(upgrade_master, 'checkBasedir', checkBasedir) + self.patch(base, 'checkBasedir', checkBasedir) def loadConfig(config, configFileName='master.cfg'): self.calls.append('loadConfig') return config_module.MasterConfig() if configOk else False - self.patch(upgrade_master, 'loadConfig', loadConfig) + self.patch(base, 'loadConfig', loadConfig) def upgradeFiles(config): self.calls.append('upgradeFiles') @@ -119,12 +119,6 @@ def setUp(self): def tearDown(self): self.tearDownDirs() - def activeBasedir(self, extra_lines=()): - with open(os.path.join('test', 'buildbot.tac'), 'wt') as f: - f.write("from twisted.application import service\n") - f.write("service.Application('buildmaster')\n") - f.write("\n".join(extra_lines)) - def writeFile(self, path, contents): with open(path, 'wt') as f: f.write(contents) @@ -135,72 +129,6 @@ def readFile(self, path): # tests - def test_checkBasedir(self): - self.activeBasedir() - rv = upgrade_master.checkBasedir(mkconfig()) - self.assertTrue(rv) - self.assertInStdout('checking basedir') - - def test_checkBasedir_quiet(self): - self.activeBasedir() - rv = upgrade_master.checkBasedir(mkconfig(quiet=True)) - self.assertTrue(rv) - self.assertWasQuiet() - - def test_checkBasedir_no_dir(self): - rv = upgrade_master.checkBasedir(mkconfig(basedir='doesntexist')) - self.assertFalse(rv) - self.assertInStdout('invalid buildmaster directory') - - @skipUnlessPlatformIs('posix') - def test_checkBasedir_active_pidfile(self): - self.activeBasedir() - open(os.path.join('test', 'twistd.pid'), 'w').close() - rv = upgrade_master.checkBasedir(mkconfig()) - self.assertFalse(rv) - self.assertInStdout('still running') - - def test_checkBasedir_invalid_rotateLength(self): - self.activeBasedir(extra_lines=['rotateLength="32"']) - rv = upgrade_master.checkBasedir(mkconfig()) - self.assertFalse(rv) - self.assertInStdout('ERROR') - self.assertInStdout('rotateLength') - - def test_checkBasedir_invalid_maxRotatedFiles(self): - self.activeBasedir(extra_lines=['maxRotatedFiles="64"']) - rv = upgrade_master.checkBasedir(mkconfig()) - self.assertFalse(rv) - self.assertInStdout('ERROR') - self.assertInStdout('maxRotatedFiles') - - def test_loadConfig(self): - @classmethod - def loadConfig(cls, basedir, filename): - return config_module.MasterConfig() - self.patch(config_module.MasterConfig, 'loadConfig', loadConfig) - cfg = upgrade_master.loadConfig(mkconfig()) - self.assertIsInstance(cfg, config_module.MasterConfig) - self.assertInStdout('checking') - - def test_loadConfig_ConfigErrors(self): - @classmethod - def loadConfig(cls, basedir, filename): - raise config_module.ConfigErrors(['oh noes']) - self.patch(config_module.MasterConfig, 'loadConfig', loadConfig) - cfg = upgrade_master.loadConfig(mkconfig()) - self.assertIdentical(cfg, None) - self.assertInStdout('oh noes') - - def test_loadConfig_exception(self): - @classmethod - def loadConfig(cls, basedir, filename): - raise RuntimeError() - self.patch(config_module.MasterConfig, 'loadConfig', loadConfig) - cfg = upgrade_master.loadConfig(mkconfig()) - self.assertIdentical(cfg, None) - self.assertInStdout('RuntimeError') - def test_installFile(self): self.writeFile('test/srcfile', 'source data') upgrade_master.installFile(mkconfig(), 'test/destfile', 'test/srcfile') diff --git a/master/docs/manual/cmdline.rst b/master/docs/manual/cmdline.rst index d1fa73011d1..6513022ef43 100644 --- a/master/docs/manual/cmdline.rst +++ b/master/docs/manual/cmdline.rst @@ -142,6 +142,21 @@ This checks if the buildmaster configuration is well-formed and contains no depr If no arguments are used or the base directory is passed as the argument the config file specified in :file:`buildbot.tac` is checked. If the argument is the path to a config file then it will be checked without using the :file:`buildbot.tac` file. + +.. bb:cmdline:: cleanupdb + +cleanupdb ++++++++++ + +.. code-block:: none + + buildbot cleanupdb {BASEDIR|CONFIG_FILE} [-q] + +This command is frontend for various database maintainance jobs: + +- optimiselogs: This optimization groups logs into bigger chunks + to apply higher level of compression. + Developer Tools ~~~~~~~~~~~~~~~