From 7b290830bfbd4e51c9b06c2e258290208d77fb2f Mon Sep 17 00:00:00 2001 From: Pierre Tardy Date: Tue, 17 Feb 2015 16:11:55 +0100 Subject: [PATCH] add dbconfig DbConfig is an utility for master.cfg to get easy key/value storage in the buildbot database DbConfig can get and store any jsonable object to the db for use by other masters or separate ui plugins to edit them. The design is voluntary simplistic, the focus is on the easy use. Example :: from buildbot.plugins import util, buildslave dbConfig = util.DbConfig(c['db']['db_url'], basedir) slaves = dbConfig.get("slaves") c['slaves'] = [ buildslave.BuildSlave(slave['name'], slave['passwd'], properties=slave.get('properties')), for slave in slaves ] --- master/buildbot/db/dbconfig.py | 68 +++++++++++++++++++ master/buildbot/test/unit/test_db_dbconfig.py | 44 ++++++++++++ master/buildbot/test/util/db.py | 2 +- master/docs/developer/database.rst | 2 + master/docs/manual/cfg-dbconfig.rst | 21 ++++++ master/docs/manual/configuration.rst | 1 + master/setup.py | 4 +- 7 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 master/buildbot/db/dbconfig.py create mode 100644 master/buildbot/test/unit/test_db_dbconfig.py create mode 100644 master/docs/manual/cfg-dbconfig.rst diff --git a/master/buildbot/db/dbconfig.py b/master/buildbot/db/dbconfig.py new file mode 100644 index 00000000000..d8a0d49234b --- /dev/null +++ b/master/buildbot/db/dbconfig.py @@ -0,0 +1,68 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +from buildbot.db import enginestrategy +from buildbot.db import model +from buildbot.db import state + + +class FakeDBConnector(object): + pass + + +class FakeCacheManager(object): + + def get_cache(self, cache_name, miss_fn): + return None + + +class FakeMaster(object): + pass + + +class FakePool(object): + pass + + +class DbConfig(object): + + def __init__(self, db_url, basedir, name="config"): + self.db_url = db_url + self.basedir = basedir + self.name = name + + def getDb(self): + db_engine = enginestrategy.create_engine(self.db_url, + basedir=self.basedir) + db = FakeDBConnector() + db.master = FakeMaster() + db.pool = FakePool() + db.pool.engine = db_engine + db.master.caches = FakeCacheManager() + db.model = model.Model(db) + db.state = state.StateConnectorComponent(db) + self.objectid = db.state.thdGetObjectId(db_engine, self.name, "DbConfig")['id'] + return db + + def get(self, name, default=state.StateConnectorComponent.Thunk): + db = self.getDb() + ret = db.state.thdGetState(db.pool.engine, self.objectid, name, default=default) + db.pool.engine.close() + return ret + + def set(self, name, value): + db = self.getDb() + db.state.thdSetState(db.pool.engine, self.objectid, name, value) + db.pool.engine.close() diff --git a/master/buildbot/test/unit/test_db_dbconfig.py b/master/buildbot/test/unit/test_db_dbconfig.py new file mode 100644 index 00000000000..f84ea0403b8 --- /dev/null +++ b/master/buildbot/test/unit/test_db_dbconfig.py @@ -0,0 +1,44 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +from buildbot.db import dbconfig +from buildbot.test.util import db +from twisted.internet import defer +from twisted.internet import threads +from twisted.trial import unittest + + +class TestDbConfig(db.RealDatabaseMixin, unittest.TestCase): + + @defer.inlineCallbacks + def setUp(self): + # as we will open the db twice, we can't use in memory sqlite + yield self.setUpRealDatabase(table_names=['objects', 'object_state'], sqlite_memory=False) + yield threads.deferToThread(self.createDbConfig) + + def createDbConfig(self): + self.dbConfig = dbconfig.DbConfig(self.db_url, self.basedir) + + def tearDown(self): + return self.tearDownRealDatabase() + + def test_basic(self): + def thd(): + slavesInDB = ['foo', 'bar'] + self.dbConfig.set(u"slaves", slavesInDB) + slaves = self.dbConfig.get(u"slaves") + self.assertEqual(slaves, slavesInDB) + + return threads.deferToThread(thd) diff --git a/master/buildbot/test/util/db.py b/master/buildbot/test/util/db.py index f6a4f8c05ec..05a178a5c7d 100644 --- a/master/buildbot/test/util/db.py +++ b/master/buildbot/test/util/db.py @@ -119,7 +119,7 @@ def setUpRealDatabase(self, table_names=[], basedir='basedir', os.makedirs(basedir) self.db_url = os.environ.get('BUILDBOT_TEST_DB_URL', default) - + self.basedir = basedir self.db_engine = enginestrategy.create_engine(self.db_url, basedir=basedir) # if the caller does not want a pool, we're done. diff --git a/master/docs/developer/database.rst b/master/docs/developer/database.rst index 919b85b53be..6347e67930b 100644 --- a/master/docs/developer/database.rst +++ b/master/docs/developer/database.rst @@ -1251,6 +1251,8 @@ state Set the state value for ``name`` for the object with id ``objectid``, overwriting any existing value. + Those 3 methods have their threaded equivalent, ``thdGetObjectId``, ``thdGetState``, ``thdSetState`` that are intended to run in synchronous code, (e.g master.cfg environnement) + users ~~~~~ diff --git a/master/docs/manual/cfg-dbconfig.rst b/master/docs/manual/cfg-dbconfig.rst new file mode 100644 index 00000000000..186283604ed --- /dev/null +++ b/master/docs/manual/cfg-dbconfig.rst @@ -0,0 +1,21 @@ +.. bb:cfg:: dbconfig + +DbConfig +-------- + +DbConfig is an utility for master.cfg to get easy key/value storage in the buildbot database + +DbConfig can get and store any jsonable object to the db for use by other masters or separate ui plugins to edit them. + +The design is voluntary simplistic, the focus is on the easy use. + +Example :: + + from buildbot.plugins import util, buildslave + dbConfig = util.DbConfig(c['db']['db_url'], basedir) + slaves = dbConfig.get("slaves") + c['slaves'] = [ + buildslave.BuildSlave(slave['name'], slave['passwd'], + properties=slave.get('properties')), + for slave in slaves + ] diff --git a/master/docs/manual/configuration.rst b/master/docs/manual/configuration.rst index 9b1a723905a..7bdb7f1571d 100644 --- a/master/docs/manual/configuration.rst +++ b/master/docs/manual/configuration.rst @@ -25,3 +25,4 @@ The next section, :doc:`customization`, describes this approach, with frequent r cfg-statustargets cfg-www cfg-services + cfg-dbconfig diff --git a/master/setup.py b/master/setup.py index 005d2ba4480..ace27325977 100755 --- a/master/setup.py +++ b/master/setup.py @@ -334,7 +334,9 @@ def define_plugin_entries(groups): 'UserPasswordAuth', 'HTPasswdAuth', 'RemoteUserAuth']), ('buildbot.www.ldapuserinfos', ['LdapUserInfo']), ('buildbot.www.oauth2', [ - 'GoogleAuth', 'GitHubAuth']) + 'GoogleAuth', 'GitHubAuth']), + ('buildbot.db.state', [ + 'DbConfig']) ]) ]) }