From c0a1b4828891c04282f6fe5c3fa21b6762484d96 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Sat, 30 Jan 2016 22:48:51 +0000 Subject: [PATCH] always specfiy mysql_default_charset=utf8 --- .../db/migrate/versions/001_initial.py | 263 +++--- .../migrate/versions/007_add_object_tables.py | 30 +- .../versions/011_add_buildrequest_claims.py | 25 +- .../migrate/versions/012_add_users_table.py | 37 +- .../versions/018_add_sourcestampset.py | 19 +- .../019_merge_schedulers_to_objects.py | 15 +- .../versions/024_add_buildslaves_table.py | 12 +- .../migrate/versions/025_add_master_table.py | 43 +- .../versions/026_add_schedulers_table.py | 41 +- .../db/migrate/versions/027_builders_table.py | 28 +- .../versions/028_sourcestamps_refactor.py | 91 ++- .../versions/029_replace_builds_table.py | 31 +- .../migrate/versions/030_statusdb_tables.py | 57 +- .../versions/031_add_changesources_table.py | 29 +- .../migrate/versions/032_slave_connections.py | 33 +- .../db/migrate/versions/034_log_slug.py | 6 +- .../versions/041_add_N_N_tagsbuilders.py | 34 +- .../042_add_build_properties_table.py | 18 +- master/buildbot/db/migrate_utils.py | 11 +- master/buildbot/db/model.py | 769 +++++++++--------- ...te_versions_011_add_buildrequest_claims.py | 74 +- ...versions_015_remove_bad_master_objectid.py | 29 +- ...rsions_016_restore_buildrequest_indices.py | 28 +- ...rate_versions_017_restore_other_indices.py | 83 +- ...migrate_versions_018_add_sourcestampset.py | 62 +- ...ersions_019_merge_schedulers_to_objects.py | 58 +- ...igrate_versions_020_remove_change_links.py | 21 +- ...ate_versions_021_fix_postgres_sequences.py | 11 +- ...st_db_migrate_versions_022_add_codebase.py | 47 +- ..._023_increase_comments_property_lengths.py | 70 +- ...b_migrate_versions_025_add_master_table.py | 32 +- ...grate_versions_026_add_schedulers_table.py | 32 +- ..._db_migrate_versions_027_builders_table.py | 10 +- ...rate_versions_028_sourcestamps_refactor.py | 119 +-- ...grate_versions_029_replace_builds_table.py | 39 +- ...db_migrate_versions_030_statusdb_tables.py | 8 +- ...te_versions_031_add_changesources_table.py | 10 +- ..._migrate_versions_032_slave_connections.py | 30 +- ...e_versions_033_buildrequests_waited_for.py | 28 +- .../test_db_migrate_versions_034_log_slug.py | 7 +- ...st_db_migrate_versions_036_build_parent.py | 73 +- ...te_versions_037_buildrequests_builderid.py | 15 +- ...st_db_migrate_versions_038_state_string.py | 49 +- ...te_versions_039_add_builder_description.py | 15 +- ...b_migrate_versions_040_add_builder_tags.py | 17 +- ...grate_versions_041_add_N_N_tagsbuilders.py | 19 +- ...versions_042_add_build_properties_table.py | 10 +- ...migrate_versions_043_add_changes_parent.py | 82 +- ...db_migrate_versions_044_add_step_hidden.py | 23 +- master/buildbot/test/util/migration.py | 12 +- master/buildbot/util/sautils.py | 8 + 51 files changed, 1469 insertions(+), 1244 deletions(-) diff --git a/master/buildbot/db/migrate/versions/001_initial.py b/master/buildbot/db/migrate/versions/001_initial.py index cf4d62fab59..d777e417e01 100644 --- a/master/buildbot/db/migrate/versions/001_initial.py +++ b/master/buildbot/db/migrate/versions/001_initial.py @@ -20,123 +20,154 @@ from buildbot.db.migrate_utils import test_unicode from buildbot.util import json from buildbot.util import pickle +from buildbot.util import sautils from twisted.persisted import styles metadata = sa.MetaData() -last_access = sa.Table('last_access', metadata, - sa.Column('who', sa.String(256), nullable=False), - sa.Column('writing', sa.Integer, nullable=False), - sa.Column('last_access', sa.Integer, nullable=False), - ) - -changes_nextid = sa.Table('changes_nextid', metadata, - sa.Column('next_changeid', sa.Integer), - ) - -changes = sa.Table('changes', metadata, - sa.Column('changeid', sa.Integer, autoincrement=False, primary_key=True), - sa.Column('author', sa.String(256), nullable=False), - sa.Column('comments', sa.String(1024), nullable=False), - sa.Column('is_dir', sa.SmallInteger, nullable=False), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('revlink', sa.String(256)), - sa.Column('when_timestamp', sa.Integer, nullable=False), - sa.Column('category', sa.String(256)), - ) - -change_links = sa.Table('change_links', metadata, - sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), - sa.Column('link', sa.String(1024), nullable=False), - ) - -change_files = sa.Table('change_files', metadata, - sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), - sa.Column('filename', sa.String(1024), nullable=False), - ) - -change_properties = sa.Table('change_properties', metadata, - sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), - sa.Column('property_name', sa.String(256), nullable=False), - sa.Column('property_value', sa.String(1024), nullable=False), - ) - -schedulers = sa.Table("schedulers", metadata, - sa.Column('schedulerid', sa.Integer, autoincrement=False, primary_key=True), - sa.Column('name', sa.String(128), nullable=False), - sa.Column('state', sa.String(1024), nullable=False), - ) - -scheduler_changes = sa.Table('scheduler_changes', metadata, - sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')), - sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), - sa.Column('important', sa.SmallInteger), - ) - -scheduler_upstream_buildsets = sa.Table('scheduler_upstream_buildsets', metadata, - sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id')), - sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')), - sa.Column('active', sa.SmallInteger), - ) - -sourcestamps = sa.Table('sourcestamps', metadata, - sa.Column('id', sa.Integer, autoincrement=False, primary_key=True), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), - ) - -patches = sa.Table('patches', metadata, - sa.Column('id', sa.Integer, autoincrement=False, primary_key=True), - sa.Column('patchlevel', sa.Integer, nullable=False), - sa.Column('patch_base64', sa.Text, nullable=False), - sa.Column('subdir', sa.Text), - ) - -sourcestamp_changes = sa.Table('sourcestamp_changes', metadata, - sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False), - sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False), - ) - -buildsets = sa.Table('buildsets', metadata, - sa.Column('id', sa.Integer, autoincrement=False, primary_key=True), - sa.Column('external_idstring', sa.String(256)), - sa.Column('reason', sa.String(256)), - sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False), - sa.Column('submitted_at', sa.Integer, nullable=False), - sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")), - sa.Column('complete_at', sa.Integer), - sa.Column('results', sa.SmallInteger), - ) - -buildset_properties = sa.Table('buildset_properties', metadata, - sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), nullable=False), - sa.Column('property_name', sa.String(256), nullable=False), - sa.Column('property_value', sa.String(1024), nullable=False), - ) - -buildrequests = sa.Table('buildrequests', metadata, - sa.Column('id', sa.Integer, autoincrement=False, primary_key=True), - sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), nullable=False), - sa.Column('buildername', sa.String(length=256), nullable=False), - sa.Column('priority', sa.Integer, nullable=False, server_default=sa.DefaultClause("0")), - sa.Column('claimed_at', sa.Integer, server_default=sa.DefaultClause("0")), - sa.Column('claimed_by_name', sa.String(length=256)), - sa.Column('claimed_by_incarnation', sa.String(length=256)), - sa.Column('complete', sa.Integer, server_default=sa.DefaultClause("0")), - sa.Column('results', sa.SmallInteger), - sa.Column('submitted_at', sa.Integer, nullable=False), - sa.Column('complete_at', sa.Integer), - ) - -builds = sa.Table('builds', metadata, - sa.Column('id', sa.Integer, autoincrement=False, primary_key=True), - sa.Column('number', sa.Integer, nullable=False), - sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), nullable=False), - sa.Column('start_time', sa.Integer, nullable=False), - sa.Column('finish_time', sa.Integer), - ) +last_access = sautils.Table( + 'last_access', metadata, + sa.Column('who', sa.String(256), nullable=False), + sa.Column('writing', sa.Integer, nullable=False), + sa.Column('last_access', sa.Integer, nullable=False), +) + +changes_nextid = sautils.Table( + 'changes_nextid', metadata, + sa.Column('next_changeid', sa.Integer), +) + +changes = sautils.Table( + 'changes', metadata, + sa.Column('changeid', sa.Integer, autoincrement=False, primary_key=True), + sa.Column('author', sa.String(256), nullable=False), + sa.Column('comments', sa.String(1024), nullable=False), + sa.Column('is_dir', sa.SmallInteger, nullable=False), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('revlink', sa.String(256)), + sa.Column('when_timestamp', sa.Integer, nullable=False), + sa.Column('category', sa.String(256)), +) + +change_links = sautils.Table( + 'change_links', metadata, + sa.Column('changeid', sa.Integer, sa.ForeignKey( + 'changes.changeid'), nullable=False), + sa.Column('link', sa.String(1024), nullable=False), +) + +change_files = sautils.Table( + 'change_files', metadata, + sa.Column('changeid', sa.Integer, sa.ForeignKey( + 'changes.changeid'), nullable=False), + sa.Column('filename', sa.String(1024), nullable=False), +) + +change_properties = sautils.Table( + 'change_properties', metadata, + sa.Column('changeid', sa.Integer, sa.ForeignKey( + 'changes.changeid'), nullable=False), + sa.Column('property_name', sa.String(256), nullable=False), + sa.Column('property_value', sa.String(1024), nullable=False), +) + +schedulers = sautils.Table( + "schedulers", metadata, + sa.Column('schedulerid', sa.Integer, + autoincrement=False, primary_key=True), + sa.Column('name', sa.String(128), nullable=False), + sa.Column('state', sa.String(1024), nullable=False), +) + +scheduler_changes = sautils.Table( + 'scheduler_changes', metadata, + sa.Column('schedulerid', sa.Integer, + sa.ForeignKey('schedulers.schedulerid')), + sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), + sa.Column('important', sa.SmallInteger), +) + +scheduler_upstream_buildsets = sautils.Table( + 'scheduler_upstream_buildsets', metadata, + sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id')), + sa.Column('schedulerid', sa.Integer, + sa.ForeignKey('schedulers.schedulerid')), + sa.Column('active', sa.SmallInteger), +) + +sourcestamps = sautils.Table( + 'sourcestamps', metadata, + sa.Column('id', sa.Integer, autoincrement=False, primary_key=True), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), +) + +patches = sautils.Table( + 'patches', metadata, + sa.Column('id', sa.Integer, autoincrement=False, primary_key=True), + sa.Column('patchlevel', sa.Integer, nullable=False), + sa.Column('patch_base64', sa.Text, nullable=False), + sa.Column('subdir', sa.Text), +) + +sourcestamp_changes = sautils.Table( + 'sourcestamp_changes', metadata, + sa.Column('sourcestampid', sa.Integer, sa.ForeignKey( + 'sourcestamps.id'), nullable=False), + sa.Column('changeid', sa.Integer, sa.ForeignKey( + 'changes.changeid'), nullable=False), +) + +buildsets = sautils.Table( + 'buildsets', metadata, + sa.Column('id', sa.Integer, autoincrement=False, primary_key=True), + sa.Column('external_idstring', sa.String(256)), + sa.Column('reason', sa.String(256)), + sa.Column('sourcestampid', sa.Integer, sa.ForeignKey( + 'sourcestamps.id'), nullable=False), + sa.Column('submitted_at', sa.Integer, nullable=False), + sa.Column('complete', sa.SmallInteger, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('complete_at', sa.Integer), + sa.Column('results', sa.SmallInteger), +) + +buildset_properties = sautils.Table( + 'buildset_properties', metadata, + sa.Column('buildsetid', sa.Integer, sa.ForeignKey( + 'buildsets.id'), nullable=False), + sa.Column('property_name', sa.String(256), nullable=False), + sa.Column('property_value', sa.String(1024), nullable=False), +) + +buildrequests = sautils.Table( + 'buildrequests', metadata, + sa.Column('id', sa.Integer, autoincrement=False, primary_key=True), + sa.Column('buildsetid', sa.Integer, sa.ForeignKey( + "buildsets.id"), nullable=False), + sa.Column('buildername', sa.String(length=256), nullable=False), + sa.Column('priority', sa.Integer, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('claimed_at', sa.Integer, server_default=sa.DefaultClause("0")), + sa.Column('claimed_by_name', sa.String(length=256)), + sa.Column('claimed_by_incarnation', sa.String(length=256)), + sa.Column('complete', sa.Integer, server_default=sa.DefaultClause("0")), + sa.Column('results', sa.SmallInteger), + sa.Column('submitted_at', sa.Integer, nullable=False), + sa.Column('complete_at', sa.Integer), +) + +builds = sautils.Table( + 'builds', metadata, + sa.Column('id', sa.Integer, autoincrement=False, primary_key=True), + sa.Column('number', sa.Integer, nullable=False), + sa.Column('brid', sa.Integer, sa.ForeignKey( + 'buildrequests.id'), nullable=False), + sa.Column('start_time', sa.Integer, nullable=False), + sa.Column('finish_time', sa.Integer), +) def import_changes(migrate_engine): @@ -204,7 +235,8 @@ def remove_none(x): values = dict([(k, remove_none(v)) for k, v in iteritems(values)]) except UnicodeDecodeError, e: - raise UnicodeError("Trying to import change data as UTF-8 failed. Please look at contrib/fix_changes_pickle_encoding.py: %s" % str(e)) + raise UnicodeError( + "Trying to import change data as UTF-8 failed. Please look at contrib/fix_changes_pickle_encoding.py: %s" % str(e)) migrate_engine.execute(changes.insert(), **values) @@ -243,7 +275,8 @@ def flatten(l): next_changeid=max_changeid + 1) # if not quiet: - # print "moving changes.pck to changes.pck.old; delete it or keep it as a backup" + # print "moving changes.pck to changes.pck.old; delete it or keep it as a + # backup" os.rename(changes_pickle, changes_pickle + ".old") diff --git a/master/buildbot/db/migrate/versions/007_add_object_tables.py b/master/buildbot/db/migrate/versions/007_add_object_tables.py index 28148171944..7a7517912aa 100644 --- a/master/buildbot/db/migrate/versions/007_add_object_tables.py +++ b/master/buildbot/db/migrate/versions/007_add_object_tables.py @@ -15,24 +15,28 @@ import sqlalchemy as sa +from buildbot.util import sautils + def upgrade(migrate_engine): metadata = sa.MetaData() metadata.bind = migrate_engine - objects = sa.Table("objects", metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column('name', sa.String(128), nullable=False), - sa.Column('class_name', sa.String(128), nullable=False), - sa.UniqueConstraint('name', 'class_name', name='object_identity'), - ) + objects = sautils.Table( + "objects", metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column('name', sa.String(128), nullable=False), + sa.Column('class_name', sa.String(128), nullable=False), + sa.UniqueConstraint('name', 'class_name', name='object_identity'), + ) objects.create() - object_state = sa.Table("object_state", metadata, - sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), - nullable=False), - sa.Column("name", sa.String(length=256), nullable=False), - sa.Column("value_json", sa.Text, nullable=False), - sa.UniqueConstraint('objectid', 'name', name='name_per_object'), - ) + object_state = sautils.Table( + "object_state", metadata, + sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), + nullable=False), + sa.Column("name", sa.String(length=256), nullable=False), + sa.Column("value_json", sa.Text, nullable=False), + sa.UniqueConstraint('objectid', 'name', name='name_per_object'), + ) object_state.create() diff --git a/master/buildbot/db/migrate/versions/011_add_buildrequest_claims.py b/master/buildbot/db/migrate/versions/011_add_buildrequest_claims.py index adff758d91c..390467b0bcb 100644 --- a/master/buildbot/db/migrate/versions/011_add_buildrequest_claims.py +++ b/master/buildbot/db/migrate/versions/011_add_buildrequest_claims.py @@ -41,11 +41,11 @@ def migrate_claims(migrate_engine, metadata, buildrequests, objects, str(sautils.InsertFromSelect(objects, new_objects))) # now make a buildrequest_claims row for each claimed build request - join = buildrequests.join(objects, - (buildrequests.c.claimed_by_name == objects.c.name) - # (have to use sa.text because str, below, doesn't work - # with placeholders) - & (objects.c.class_name == sa.text("'BuildMaster'"))) + join = buildrequests.join( + objects, (buildrequests.c.claimed_by_name == objects.c.name) & + # (have to use sa.text because str, below, doesn't work + # with placeholders) + (objects.c.class_name == sa.text("'BuildMaster'"))) claims = sa.select([ buildrequests.c.id.label('brid'), objects.c.id.label('objectid'), @@ -105,13 +105,14 @@ def upgrade(migrate_engine): ) # and a new buildrequest_claims table - buildrequest_claims = sa.Table('buildrequest_claims', metadata, - sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), - index=True, unique=True), - sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id'), - index=True, nullable=True), - sa.Column('claimed_at', sa.Integer, nullable=False), - ) + buildrequest_claims = sautils.Table( + 'buildrequest_claims', metadata, + sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), + index=True, unique=True), + sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id'), + index=True, nullable=True), + sa.Column('claimed_at', sa.Integer, nullable=False), + ) # create the new table buildrequest_claims.create() diff --git a/master/buildbot/db/migrate/versions/012_add_users_table.py b/master/buildbot/db/migrate/versions/012_add_users_table.py index 5e15cd2ebce..c0810ac5a25 100644 --- a/master/buildbot/db/migrate/versions/012_add_users_table.py +++ b/master/buildbot/db/migrate/versions/012_add_users_table.py @@ -15,6 +15,8 @@ import sqlalchemy as sa +from buildbot.util import sautils + def upgrade(migrate_engine): @@ -22,22 +24,24 @@ def upgrade(migrate_engine): metadata.bind = migrate_engine # what defines a user - users = sa.Table("users", metadata, - sa.Column("uid", sa.Integer, primary_key=True), - sa.Column("identifier", sa.String(256), nullable=False), - ) + users = sautils.Table( + "users", metadata, + sa.Column("uid", sa.Integer, primary_key=True), + sa.Column("identifier", sa.String(256), nullable=False), + ) users.create() idx = sa.Index('users_identifier', users.c.identifier) idx.create() # ways buildbot knows about users - users_info = sa.Table("users_info", metadata, - sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), - nullable=False), - sa.Column("attr_type", sa.String(128), nullable=False), - sa.Column("attr_data", sa.String(128), nullable=False) - ) + users_info = sautils.Table( + "users_info", metadata, + sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), + nullable=False), + sa.Column("attr_type", sa.String(128), nullable=False), + sa.Column("attr_data", sa.String(128), nullable=False), + ) users_info.create() idx = sa.Index('users_info_uid', users_info.c.uid) @@ -51,12 +55,13 @@ def upgrade(migrate_engine): # correlates change authors and user uids sa.Table('changes', metadata, autoload=True) - change_users = sa.Table("change_users", metadata, - sa.Column("changeid", sa.Integer, sa.ForeignKey('changes.changeid'), - nullable=False), - sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), - nullable=False) - ) + change_users = sautils.Table( + "change_users", metadata, + sa.Column("changeid", sa.Integer, sa.ForeignKey('changes.changeid'), + nullable=False), + sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), + nullable=False), + ) change_users.create() idx = sa.Index('change_users_changeid', change_users.c.changeid) diff --git a/master/buildbot/db/migrate/versions/018_add_sourcestampset.py b/master/buildbot/db/migrate/versions/018_add_sourcestampset.py index 7d457f50d46..871f6ed844f 100644 --- a/master/buildbot/db/migrate/versions/018_add_sourcestampset.py +++ b/master/buildbot/db/migrate/versions/018_add_sourcestampset.py @@ -29,9 +29,10 @@ def upgrade(migrate_engine): # Create the sourcestampset table # that defines a sourcestampset - sourcestampsets_table = sa.Table("sourcestampsets", metadata, - sa.Column("id", sa.Integer, primary_key=True), - ) + sourcestampsets_table = sautils.Table( + "sourcestampsets", metadata, + sa.Column("id", sa.Integer, primary_key=True), + ) sourcestampsets_table.create() # All current sourcestampid's are migrated to sourcestampsetid @@ -46,7 +47,8 @@ def upgrade(migrate_engine): metadata.remove(buildsets_table) buildsets_table = sa.Table('buildsets', metadata, autoload=True) - cons = constraint.ForeignKeyConstraint([buildsets_table.c.sourcestampsetid], [sourcestampsets_table.c.id]) + cons = constraint.ForeignKeyConstraint([buildsets_table.c.sourcestampsetid], [ + sourcestampsets_table.c.id]) cons.create() # Add sourcestampsetid including index to sourcestamps table @@ -54,13 +56,16 @@ def upgrade(migrate_engine): ss_sourcestampsetid.create(sourcestamps_table) # Update the setid to the same value as sourcestampid - migrate_engine.execute(str(sourcestamps_table.update().values(sourcestampsetid=sourcestamps_table.c.id))) + migrate_engine.execute(str(sourcestamps_table.update().values( + sourcestampsetid=sourcestamps_table.c.id))) ss_sourcestampsetid.alter(nullable=False) # Data is up to date, now force integrity - cons = constraint.ForeignKeyConstraint([sourcestamps_table.c.sourcestampsetid], [sourcestampsets_table.c.id]) + cons = constraint.ForeignKeyConstraint([sourcestamps_table.c.sourcestampsetid], [ + sourcestampsets_table.c.id]) cons.create() # Add index for performance reasons to find all sourcestamps in a set quickly - idx = sa.Index('sourcestamps_sourcestampsetid', sourcestamps_table.c.sourcestampsetid, unique=False) + idx = sa.Index('sourcestamps_sourcestampsetid', + sourcestamps_table.c.sourcestampsetid, unique=False) idx.create() diff --git a/master/buildbot/db/migrate/versions/019_merge_schedulers_to_objects.py b/master/buildbot/db/migrate/versions/019_merge_schedulers_to_objects.py index 062e906ad51..0e5452a92d8 100644 --- a/master/buildbot/db/migrate/versions/019_merge_schedulers_to_objects.py +++ b/master/buildbot/db/migrate/versions/019_merge_schedulers_to_objects.py @@ -15,6 +15,8 @@ import sqlalchemy as sa +from buildbot.util import sautils + def upgrade(migrate_engine): metadata = sa.MetaData() @@ -52,13 +54,12 @@ def upgrade(migrate_engine): # schedulers and scheduler_upstream_buildsets aren't coming back, but # scheduler_changes is -- along with its indexes - scheduler_changes_tbl = sa.Table('scheduler_changes', metadata, - sa.Column('objectid', sa.Integer, - sa.ForeignKey('objects.id')), - sa.Column('changeid', sa.Integer, - sa.ForeignKey('changes.changeid')), - sa.Column('important', sa.Integer), - ) + scheduler_changes_tbl = sautils.Table( + 'scheduler_changes', metadata, + sa.Column('objectid', sa.Integer, sa.ForeignKey('objects.id')), + sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), + sa.Column('important', sa.Integer), + ) scheduler_changes_tbl.create() idx = sa.Index('scheduler_changes_objectid', diff --git a/master/buildbot/db/migrate/versions/024_add_buildslaves_table.py b/master/buildbot/db/migrate/versions/024_add_buildslaves_table.py index 69bf047866a..35cebb81b95 100644 --- a/master/buildbot/db/migrate/versions/024_add_buildslaves_table.py +++ b/master/buildbot/db/migrate/versions/024_add_buildslaves_table.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.db.types.json import JsonObject +from buildbot.util import sautils def upgrade(migrate_engine): @@ -23,11 +24,12 @@ def upgrade(migrate_engine): metadata = sa.MetaData() metadata.bind = migrate_engine - buildslaves = sa.Table("buildslaves", metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column("name", sa.String(256), nullable=False), - sa.Column("info", JsonObject, nullable=False), - ) + buildslaves = sautils.Table( + "buildslaves", metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("name", sa.String(256), nullable=False), + sa.Column("info", JsonObject, nullable=False), + ) buildslaves.create() idx = sa.Index('buildslaves_name', buildslaves.c.name, unique=True) diff --git a/master/buildbot/db/migrate/versions/025_add_master_table.py b/master/buildbot/db/migrate/versions/025_add_master_table.py index 3d2448df7ab..89f44db2615 100644 --- a/master/buildbot/db/migrate/versions/025_add_master_table.py +++ b/master/buildbot/db/migrate/versions/025_add_master_table.py @@ -49,27 +49,28 @@ def upgrade(migrate_engine): sa.Column('class_name', sa.String(128), nullable=False), ) - masters = sa.Table("masters", metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('name', sa.String(128), nullable=False), - sa.Column('name_hash', sa.String(40), nullable=False), - sa.Column('active', sa.Integer, nullable=False), - sa.Column('last_active', sa.Integer, nullable=False), - ) - - buildrequest_claims_old = sa.Table("buildrequest_claims_old", metadata, - sa.Column('brid', sa.Integer, index=True), - sa.Column('objectid', sa.Integer, index=True, nullable=True), - sa.Column('claimed_at', sa.Integer, nullable=False), - ) - - buildrequest_claims = sa.Table('buildrequest_claims', metadata, - sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), - nullable=False), - sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), - index=True, nullable=True), - sa.Column('claimed_at', sa.Integer, nullable=False) - ) + masters = sautils.Table( + "masters", metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('name', sa.String(128), nullable=False), + sa.Column('name_hash', sa.String(40), nullable=False), + sa.Column('active', sa.Integer, nullable=False), + sa.Column('last_active', sa.Integer, nullable=False), + ) + + buildrequest_claims_old = sautils.Table( + "buildrequest_claims_old", metadata, + sa.Column('brid', sa.Integer, index=True), + sa.Column('objectid', sa.Integer, index=True, nullable=True), + sa.Column('claimed_at', sa.Integer, nullable=False), + ) + + buildrequest_claims = sautils.Table( + 'buildrequest_claims', metadata, + sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), nullable=False), + sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), index=True, nullable=True), + sa.Column('claimed_at', sa.Integer, nullable=False), + ) # create the new table masters.create() diff --git a/master/buildbot/db/migrate/versions/026_add_schedulers_table.py b/master/buildbot/db/migrate/versions/026_add_schedulers_table.py index a4412876fbc..610807f0b59 100644 --- a/master/buildbot/db/migrate/versions/026_add_schedulers_table.py +++ b/master/buildbot/db/migrate/versions/026_add_schedulers_table.py @@ -15,6 +15,8 @@ import sqlalchemy as sa +from buildbot.util import sautils + def upgrade(migrate_engine): @@ -55,25 +57,28 @@ def upgrade(migrate_engine): # .. ) - schedulers = sa.Table('schedulers', metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column('name', sa.Text, nullable=False), - sa.Column('name_hash', sa.String(40), nullable=False), - ) - - scheduler_masters = sa.Table('scheduler_masters', metadata, - sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id'), - nullable=False, primary_key=True), - sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), - nullable=False), - ) + schedulers = sautils.Table( + 'schedulers', metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column('name', sa.Text, nullable=False), + sa.Column('name_hash', sa.String(40), nullable=False), + ) - scheduler_changes = sa.Table('scheduler_changes', metadata, - sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id')), - sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), - # true (nonzero) if this change is important to this scheduler - sa.Column('important', sa.Integer), - ) + scheduler_masters = sautils.Table( + 'scheduler_masters', metadata, + sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id'), + nullable=False, primary_key=True), + sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), + nullable=False), + ) + + scheduler_changes = sautils.Table( + 'scheduler_changes', metadata, + sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id')), + sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), + # true (nonzero) if this change is important to this scheduler + sa.Column('important', sa.Integer), + ) # create the new tables schedulers.create() diff --git a/master/buildbot/db/migrate/versions/027_builders_table.py b/master/buildbot/db/migrate/versions/027_builders_table.py index b49737d4ac2..98c1e0dbf6d 100644 --- a/master/buildbot/db/migrate/versions/027_builders_table.py +++ b/master/buildbot/db/migrate/versions/027_builders_table.py @@ -15,6 +15,8 @@ import sqlalchemy as sa +from buildbot.util import sautils + def upgrade(migrate_engine): metadata = sa.MetaData() @@ -25,20 +27,22 @@ def upgrade(migrate_engine): # .. ) - builders = sa.Table('builders', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('name', sa.Text, nullable=False), - sa.Column('name_hash', sa.String(40), nullable=False), - ) + builders = sautils.Table( + 'builders', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('name', sa.Text, nullable=False), + sa.Column('name_hash', sa.String(40), nullable=False), + ) builders.create() - builder_masters = sa.Table('builder_masters', metadata, - sa.Column('id', sa.Integer, primary_key=True, nullable=False), - sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'), - nullable=False), - sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), - nullable=False), - ) + builder_masters = sautils.Table( + 'builder_masters', metadata, + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'), + nullable=False), + sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), + nullable=False), + ) builder_masters.create() idx = sa.Index('builder_name_hash', builders.c.name_hash, unique=True) diff --git a/master/buildbot/db/migrate/versions/028_sourcestamps_refactor.py b/master/buildbot/db/migrate/versions/028_sourcestamps_refactor.py index 35cf6fb8f79..c51190e8e5a 100644 --- a/master/buildbot/db/migrate/versions/028_sourcestamps_refactor.py +++ b/master/buildbot/db/migrate/versions/028_sourcestamps_refactor.py @@ -17,6 +17,8 @@ import sqlalchemy as sa import time +from buildbot.util import sautils + def rename_sourcestamps_to_old(migrate_engine): metadata = sa.MetaData() @@ -52,43 +54,45 @@ def add_new_schema_parts(migrate_engine): sa.Column('id', sa.Integer, primary_key=True), # ... ) - sourcestamps = sa.Table('sourcestamps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('ss_hash', sa.String(40), nullable=False), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), - sa.Column('repository', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('codebase', sa.String(256), nullable=False, - server_default=sa.DefaultClause("")), - sa.Column('project', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('created_at', sa.Integer, nullable=False), - ) + sourcestamps = sautils.Table( + 'sourcestamps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('ss_hash', sa.String(40), nullable=False), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), + sa.Column('repository', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), + sa.Column('project', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('created_at', sa.Integer, nullable=False), + ) sourcestamps.create() idx = sa.Index('sourcestamps_ss_hash_key', sourcestamps.c.ss_hash, unique=True) idx.create() - changes = sa.Table('changes', metadata, - sa.Column('changeid', sa.Integer, primary_key=True), - sa.Column('author', sa.String(256), nullable=False), - sa.Column('comments', sa.String(1024), nullable=False), - sa.Column('is_dir', sa.SmallInteger, nullable=False), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('revlink', sa.String(256)), - sa.Column('when_timestamp', sa.Integer, nullable=False), - sa.Column('category', sa.String(256)), - sa.Column('repository', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('codebase', sa.String(256), nullable=False, - server_default=sa.DefaultClause("")), - sa.Column('project', sa.String(length=512), nullable=False, - server_default=''), - ) + changes = sautils.Table( + 'changes', metadata, + sa.Column('changeid', sa.Integer, primary_key=True), + sa.Column('author', sa.String(256), nullable=False), + sa.Column('comments', sa.String(1024), nullable=False), + sa.Column('is_dir', sa.SmallInteger, nullable=False), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('revlink', sa.String(256)), + sa.Column('when_timestamp', sa.Integer, nullable=False), + sa.Column('category', sa.String(256)), + sa.Column('repository', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), + sa.Column('project', sa.String(length=512), nullable=False, + server_default=''), + ) sourcestampid = sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id')) sourcestampid.create(changes, populate_default=True) @@ -231,15 +235,16 @@ def findSourceStampId(branch=None, revision=None, repository=None, sa.ForeignKey('sourcestampsets.id')), ) - buildset_sourcestamps = sa.Table('buildset_sourcestamps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('buildsetid', sa.Integer, - sa.ForeignKey('buildsets.id'), - nullable=False), - sa.Column('sourcestampid', sa.Integer, - sa.ForeignKey('sourcestamps.id'), - nullable=False), - ) + buildset_sourcestamps = sautils.Table( + 'buildset_sourcestamps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('buildsetid', sa.Integer, + sa.ForeignKey('buildsets.id'), + nullable=False), + sa.Column('sourcestampid', sa.Integer, + sa.ForeignKey('sourcestamps.id'), + nullable=False), + ) buildset_sourcestamps.create() idx = sa.Index('buildset_sourcestamps_buildsetid', buildset_sourcestamps.c.buildsetid) @@ -252,9 +257,9 @@ def findSourceStampId(branch=None, revision=None, repository=None, # now translate the existing buildset -> sourcestampset relationships into # buildset -> buildset_sourcestamps - f = buildsets.join(sourcestamps_old, - onclause=sourcestamps_old.c.sourcestampsetid - == buildsets.c.sourcestampsetid) + f = buildsets.join( + sourcestamps_old, + onclause=sourcestamps_old.c.sourcestampsetid == buildsets.c.sourcestampsetid) ss = sourcestamps_old q = sa.select([buildsets.c.id, ss.c.branch, ss.c.revision, ss.c.project, ss.c.codebase, ss.c.repository, ss.c.patchid], diff --git a/master/buildbot/db/migrate/versions/029_replace_builds_table.py b/master/buildbot/db/migrate/versions/029_replace_builds_table.py index b9a56c5b285..043f03416d1 100644 --- a/master/buildbot/db/migrate/versions/029_replace_builds_table.py +++ b/master/buildbot/db/migrate/versions/029_replace_builds_table.py @@ -15,6 +15,8 @@ import sqlalchemy as sa +from buildbot.util import sautils + # The existing builds table doesn't contain much useful information, and it's # horrendously denormalized. So we kill it dead. @@ -44,20 +46,21 @@ def add_new_builds(migrate_engine): sa.Column('id', sa.Integer, primary_key=True), ) - builds = sa.Table('builds', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('number', sa.Integer, nullable=False), - sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id')), - sa.Column('buildrequestid', sa.Integer, - sa.ForeignKey('buildrequests.id'), nullable=False), - sa.Column('buildslaveid', sa.Integer), - sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), - nullable=False), - sa.Column('started_at', sa.Integer, nullable=False), - sa.Column('complete_at', sa.Integer), - sa.Column('state_strings_json', sa.Text, nullable=False), - sa.Column('results', sa.Integer), - ) + builds = sautils.Table( + 'builds', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('number', sa.Integer, nullable=False), + sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id')), + sa.Column('buildrequestid', sa.Integer, + sa.ForeignKey('buildrequests.id'), nullable=False), + sa.Column('buildslaveid', sa.Integer), + sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), + nullable=False), + sa.Column('started_at', sa.Integer, nullable=False), + sa.Column('complete_at', sa.Integer), + sa.Column('state_strings_json', sa.Text, nullable=False), + sa.Column('results', sa.Integer), + ) builds.create() idx = sa.Index('builds_number', builds.c.builderid, builds.c.number, unique=True) diff --git a/master/buildbot/db/migrate/versions/030_statusdb_tables.py b/master/buildbot/db/migrate/versions/030_statusdb_tables.py index 65627120290..1a4bb09a73e 100644 --- a/master/buildbot/db/migrate/versions/030_statusdb_tables.py +++ b/master/buildbot/db/migrate/versions/030_statusdb_tables.py @@ -15,6 +15,8 @@ import sqlalchemy as sa +from buildbot.util import sautils + def upgrade(migrate_engine): metadata = sa.MetaData() @@ -25,34 +27,37 @@ def upgrade(migrate_engine): sa.Column('id', sa.Integer, primary_key=True), ) - steps = sa.Table('steps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('number', sa.Integer, nullable=False), - sa.Column('name', sa.String(50), nullable=False), - sa.Column('buildid', sa.Integer, sa.ForeignKey('builds.id')), - sa.Column('started_at', sa.Integer), - sa.Column('complete_at', sa.Integer), - sa.Column('state_strings_json', sa.Text, nullable=False), - sa.Column('results', sa.Integer), - sa.Column('urls_json', sa.Text, nullable=False), - ) + steps = sautils.Table( + 'steps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('number', sa.Integer, nullable=False), + sa.Column('name', sa.String(50), nullable=False), + sa.Column('buildid', sa.Integer, sa.ForeignKey('builds.id')), + sa.Column('started_at', sa.Integer), + sa.Column('complete_at', sa.Integer), + sa.Column('state_strings_json', sa.Text, nullable=False), + sa.Column('results', sa.Integer), + sa.Column('urls_json', sa.Text, nullable=False), + ) - logs = sa.Table('logs', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('name', sa.String(50), nullable=False), - sa.Column('stepid', sa.Integer, sa.ForeignKey('steps.id')), - sa.Column('complete', sa.SmallInteger, nullable=False), - sa.Column('num_lines', sa.Integer, nullable=False), - sa.Column('type', sa.String(1), nullable=False), - ) + logs = sautils.Table( + 'logs', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('name', sa.String(50), nullable=False), + sa.Column('stepid', sa.Integer, sa.ForeignKey('steps.id')), + sa.Column('complete', sa.SmallInteger, nullable=False), + sa.Column('num_lines', sa.Integer, nullable=False), + sa.Column('type', sa.String(1), nullable=False), + ) - logchunks = sa.Table('logchunks', metadata, - sa.Column('logid', sa.Integer, sa.ForeignKey('logs.id')), - sa.Column('first_line', sa.Integer, nullable=False), - sa.Column('last_line', sa.Integer, nullable=False), - sa.Column('content', sa.LargeBinary(65536)), - sa.Column('compressed', sa.SmallInteger, nullable=False), - ) + logchunks = sautils.Table( + 'logchunks', metadata, + sa.Column('logid', sa.Integer, sa.ForeignKey('logs.id')), + sa.Column('first_line', sa.Integer, nullable=False), + sa.Column('last_line', sa.Integer, nullable=False), + sa.Column('content', sa.LargeBinary(65536)), + sa.Column('compressed', sa.SmallInteger, nullable=False), + ) steps.create() logs.create() diff --git a/master/buildbot/db/migrate/versions/031_add_changesources_table.py b/master/buildbot/db/migrate/versions/031_add_changesources_table.py index c161bf2d169..a88b8a244e7 100644 --- a/master/buildbot/db/migrate/versions/031_add_changesources_table.py +++ b/master/buildbot/db/migrate/versions/031_add_changesources_table.py @@ -15,6 +15,8 @@ import sqlalchemy as sa +from buildbot.util import sautils + def upgrade(migrate_engine): @@ -26,18 +28,21 @@ def upgrade(migrate_engine): # .. ) - changesources = sa.Table('changesources', metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column('name', sa.Text, nullable=False), - sa.Column('name_hash', sa.String(40), nullable=False), - ) - - changesource_masters = sa.Table('changesource_masters', metadata, - sa.Column('changesourceid', sa.Integer, sa.ForeignKey('changesources.id'), - nullable=False, primary_key=True), - sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), - nullable=False), - ) + changesources = sautils.Table( + 'changesources', metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column('name', sa.Text, nullable=False), + sa.Column('name_hash', sa.String(40), nullable=False), + ) + + changesource_masters = sautils.Table( + 'changesource_masters', metadata, + sa.Column('changesourceid', sa.Integer, + sa.ForeignKey('changesources.id'), nullable=False, + primary_key=True), + sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), + nullable=False), + ) # create the new tables changesources.create() diff --git a/master/buildbot/db/migrate/versions/032_slave_connections.py b/master/buildbot/db/migrate/versions/032_slave_connections.py index 426f3b9ae98..07a93422ea3 100644 --- a/master/buildbot/db/migrate/versions/032_slave_connections.py +++ b/master/buildbot/db/migrate/versions/032_slave_connections.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.db.types.json import JsonObject +from buildbot.util import sautils def upgrade(migrate_engine): @@ -39,21 +40,23 @@ def upgrade(migrate_engine): sa.Column("info", JsonObject, nullable=False), ) - configured_buildslaves = sa.Table('configured_buildslaves', metadata, - sa.Column('id', sa.Integer, primary_key=True, nullable=False), - sa.Column('buildermasterid', sa.Integer, - sa.ForeignKey('builder_masters.id'), nullable=False), - sa.Column('buildslaveid', sa.Integer, sa.ForeignKey('buildslaves.id'), - nullable=False), - ) - - connected_buildslaves = sa.Table('connected_buildslaves', metadata, - sa.Column('id', sa.Integer, primary_key=True, nullable=False), - sa.Column('masterid', sa.Integer, - sa.ForeignKey('masters.id'), nullable=False), - sa.Column('buildslaveid', sa.Integer, sa.ForeignKey('buildslaves.id'), - nullable=False), - ) + configured_buildslaves = sautils.Table( + 'configured_buildslaves', metadata, + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('buildermasterid', sa.Integer, + sa.ForeignKey('builder_masters.id'), nullable=False), + sa.Column('buildslaveid', sa.Integer, + sa.ForeignKey('buildslaves.id'), nullable=False), + ) + + connected_buildslaves = sautils.Table( + 'connected_buildslaves', metadata, + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('masterid', sa.Integer, + sa.ForeignKey('masters.id'), nullable=False), + sa.Column('buildslaveid', sa.Integer, + sa.ForeignKey('buildslaves.id'), nullable=False), + ) # update the column length in bulidslaves buildslaves.c.name.alter(sa.String(50), nullable=False) diff --git a/master/buildbot/db/migrate/versions/034_log_slug.py b/master/buildbot/db/migrate/versions/034_log_slug.py index 62c4b283040..3f37daea8ca 100644 --- a/master/buildbot/db/migrate/versions/034_log_slug.py +++ b/master/buildbot/db/migrate/versions/034_log_slug.py @@ -15,6 +15,8 @@ import sqlalchemy as sa +from buildbot.util import sautils + def upgrade(migrate_engine): metadata = sa.MetaData() @@ -29,7 +31,7 @@ def upgrade(migrate_engine): metadata.bind = migrate_engine sa.Table('steps', metadata, autoload=True) - logs = sa.Table( + logs = sautils.Table( 'logs', metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('name', sa.Text, nullable=False), @@ -42,7 +44,7 @@ def upgrade(migrate_engine): ) logs.create() - logchunks = sa.Table( + logchunks = sautils.Table( 'logchunks', metadata, sa.Column('logid', sa.Integer, sa.ForeignKey('logs.id')), # 0-based line number range in this chunk (inclusive); note that for diff --git a/master/buildbot/db/migrate/versions/041_add_N_N_tagsbuilders.py b/master/buildbot/db/migrate/versions/041_add_N_N_tagsbuilders.py index 8b2dab25305..73c40fcaec5 100644 --- a/master/buildbot/db/migrate/versions/041_add_N_N_tagsbuilders.py +++ b/master/buildbot/db/migrate/versions/041_add_N_N_tagsbuilders.py @@ -15,6 +15,8 @@ import sqlalchemy as sa +from buildbot.util import sautils + def upgrade(migrate_engine): metadata = sa.MetaData() @@ -24,24 +26,24 @@ def upgrade(migrate_engine): # drop the tags column builders.c.tags.drop() - tags = sa.Table('tags', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # tag's name - sa.Column('name', sa.Text, nullable=False), - # sha1 of name; used for a unique index - sa.Column('name_hash', sa.String(40), nullable=False), - ) + tags = sautils.Table( + 'tags', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # tag's name + sa.Column('name', sa.Text, nullable=False), + # sha1 of name; used for a unique index + sa.Column('name_hash', sa.String(40), nullable=False), + ) # a many-to-may relationship between builders and tags - builders_tags = sa.Table('builders_tags', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('builderid', sa.Integer, - sa.ForeignKey('builders.id'), - nullable=False), - sa.Column('tagid', sa.Integer, - sa.ForeignKey('tags.id'), - nullable=False), - ) + builders_tags = sautils.Table( + 'builders_tags', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'), + nullable=False), + sa.Column('tagid', sa.Integer, sa.ForeignKey('tags.id'), + nullable=False), + ) # create the new tables tags.create() diff --git a/master/buildbot/db/migrate/versions/042_add_build_properties_table.py b/master/buildbot/db/migrate/versions/042_add_build_properties_table.py index 91d72eeac47..c5857bae1b9 100644 --- a/master/buildbot/db/migrate/versions/042_add_build_properties_table.py +++ b/master/buildbot/db/migrate/versions/042_add_build_properties_table.py @@ -15,6 +15,8 @@ import sqlalchemy as sa +from buildbot.util import sautils + def upgrade(migrate_engine): metadata = sa.MetaData() @@ -26,14 +28,14 @@ def upgrade(migrate_engine): ) # This table contains input properties for builds - build_properties = sa.Table('build_properties', metadata, - sa.Column('buildid', sa.Integer, sa.ForeignKey('builds.id'), - nullable=False), - sa.Column('name', sa.String(256), nullable=False), - # JSON-encoded value - sa.Column('value', sa.Text, nullable=False), - sa.Column('source', sa.Text, nullable=False), - ) + build_properties = sautils.Table( + 'build_properties', metadata, + sa.Column('buildid', sa.Integer, sa.ForeignKey('builds.id'), nullable=False), + sa.Column('name', sa.String(256), nullable=False), + # JSON-encoded value + sa.Column('value', sa.Text, nullable=False), + sa.Column('source', sa.Text, nullable=False), + ) # create the new table build_properties.create() diff --git a/master/buildbot/db/migrate_utils.py b/master/buildbot/db/migrate_utils.py index dd7a321f56d..5a177020705 100644 --- a/master/buildbot/db/migrate_utils.py +++ b/master/buildbot/db/migrate_utils.py @@ -17,6 +17,8 @@ import os import sqlalchemy as sa +from buildbot.util import sautils + def test_unicode(migrate_engine): """Test that the database can handle inserting and selecting Unicode""" @@ -24,10 +26,11 @@ def test_unicode(migrate_engine): submeta = sa.MetaData() submeta.bind = migrate_engine - test_unicode = sa.Table('test_unicode', submeta, - sa.Column('u', sa.Unicode(length=100)), - sa.Column('b', sa.LargeBinary), - ) + test_unicode = sautils.Table( + 'test_unicode', submeta, + sa.Column('u', sa.Unicode(length=100)), + sa.Column('b', sa.LargeBinary), + ) test_unicode.create() # insert a unicode value in there diff --git a/master/buildbot/db/model.py b/master/buildbot/db/model.py index 99cda5a862f..4208d69b552 100644 --- a/master/buildbot/db/model.py +++ b/master/buildbot/db/model.py @@ -23,6 +23,7 @@ from buildbot.db.migrate_utils import should_import_changes from buildbot.db.migrate_utils import test_unicode from buildbot.db.types.json import JsonObject +from buildbot.util import sautils from twisted.python import log from twisted.python import util @@ -58,364 +59,386 @@ class Model(base.DBConnectorComponent): # A BuildRequest is a request for a particular build to be performed. Each # BuildRequest is a part of a Buildset. BuildRequests are claimed by # masters, to avoid multiple masters running the same build. - buildrequests = sa.Table('buildrequests', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), - nullable=False), - sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'), - nullable=False), - sa.Column('priority', sa.Integer, nullable=False, - server_default=sa.DefaultClause("0")), - - # if this is zero, then the build is still pending - sa.Column('complete', sa.Integer, - server_default=sa.DefaultClause("0")), - - # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, - # etc - see master/buildbot/status/builder.py - sa.Column('results', sa.SmallInteger), - - # time the buildrequest was created - sa.Column('submitted_at', sa.Integer, nullable=False), - - # time the buildrequest was completed, or NULL - sa.Column('complete_at', sa.Integer), - - # boolean indicating whether there is a step blocking, waiting for this request to complete - sa.Column('waited_for', sa.SmallInteger, - server_default=sa.DefaultClause("0")), - ) + buildrequests = sautils.Table( + 'buildrequests', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), + nullable=False), + sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'), + nullable=False), + sa.Column('priority', sa.Integer, nullable=False, + server_default=sa.DefaultClause("0")), + + # if this is zero, then the build is still pending + sa.Column('complete', sa.Integer, + server_default=sa.DefaultClause("0")), + + # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, + # etc - see master/buildbot/status/builder.py + sa.Column('results', sa.SmallInteger), + + # time the buildrequest was created + sa.Column('submitted_at', sa.Integer, nullable=False), + + # time the buildrequest was completed, or NULL + sa.Column('complete_at', sa.Integer), + + # boolean indicating whether there is a step blocking, waiting for this + # request to complete + sa.Column('waited_for', sa.SmallInteger, + server_default=sa.DefaultClause("0")), + ) # Each row in this table represents a claimed build request, where the # claim is made by the object referenced by objectid. - buildrequest_claims = sa.Table('buildrequest_claims', metadata, - sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), - nullable=False), - sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), - index=True, nullable=True), - sa.Column('claimed_at', sa.Integer, nullable=False), - ) + buildrequest_claims = sautils.Table( + 'buildrequest_claims', metadata, + sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), + nullable=False), + sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), + index=True, nullable=True), + sa.Column('claimed_at', sa.Integer, nullable=False), + ) # builds # This table contains the build properties - build_properties = sa.Table('build_properties', metadata, - sa.Column('buildid', sa.Integer, sa.ForeignKey('builds.id'), - nullable=False), - sa.Column('name', sa.String(256), nullable=False), - # JSON encoded value - sa.Column('value', sa.Text, nullable=False), - sa.Column('source', sa.String(256), nullable=False), - ) + build_properties = sautils.Table( + 'build_properties', metadata, + sa.Column('buildid', sa.Integer, sa.ForeignKey('builds.id'), + nullable=False), + sa.Column('name', sa.String(256), nullable=False), + # JSON encoded value + sa.Column('value', sa.Text, nullable=False), + sa.Column('source', sa.String(256), nullable=False), + ) # This table contains basic information about each build. - builds = sa.Table('builds', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('number', sa.Integer, nullable=False), - sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id')), - # note that there is 1:N relationship here. - # In case of slave loss, build has results RETRY - # and buildrequest is unclaimed - sa.Column('buildrequestid', sa.Integer, sa.ForeignKey('buildrequests.id'), - nullable=False), - # slave which performed this build - # TODO: ForeignKey to buildslaves table, named buildslaveid (#3088) - # TODO: keep nullable to support slave-free builds (#3088) - sa.Column('buildslaveid', sa.Integer), - # master which controlled this build - sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), - nullable=False), - # start/complete times - sa.Column('started_at', sa.Integer, nullable=False), - sa.Column('complete_at', sa.Integer), - sa.Column('state_string', sa.Text, nullable=False, server_default=''), - sa.Column('results', sa.Integer), - ) + builds = sautils.Table( + 'builds', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('number', sa.Integer, nullable=False), + sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id')), + # note that there is 1:N relationship here. + # In case of slave loss, build has results RETRY + # and buildrequest is unclaimed + sa.Column('buildrequestid', sa.Integer, + sa.ForeignKey('buildrequests.id'), nullable=False), + # slave which performed this build + # TODO: ForeignKey to buildslaves table, named buildslaveid (#3088) + # TODO: keep nullable to support slave-free builds (#3088) + sa.Column('buildslaveid', sa.Integer), + # master which controlled this build + sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), + nullable=False), + # start/complete times + sa.Column('started_at', sa.Integer, nullable=False), + sa.Column('complete_at', sa.Integer), + sa.Column('state_string', sa.Text, nullable=False, server_default=''), + sa.Column('results', sa.Integer), + ) # steps - steps = sa.Table('steps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('number', sa.Integer, nullable=False), - sa.Column('name', sa.String(50), nullable=False), - sa.Column('buildid', sa.Integer, sa.ForeignKey('builds.id')), - sa.Column('started_at', sa.Integer), - sa.Column('complete_at', sa.Integer), - sa.Column('state_string', sa.Text, nullable=False, server_default=''), - sa.Column('results', sa.Integer), - sa.Column('urls_json', sa.Text, nullable=False), - sa.Column('hidden', sa.SmallInteger, nullable=False, server_default='0'), - ) + steps = sautils.Table( + 'steps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('number', sa.Integer, nullable=False), + sa.Column('name', sa.String(50), nullable=False), + sa.Column('buildid', sa.Integer, sa.ForeignKey('builds.id')), + sa.Column('started_at', sa.Integer), + sa.Column('complete_at', sa.Integer), + sa.Column('state_string', sa.Text, nullable=False, server_default=''), + sa.Column('results', sa.Integer), + sa.Column('urls_json', sa.Text, nullable=False), + sa.Column('hidden', sa.SmallInteger, nullable=False, server_default='0'), + ) # logs - logs = sa.Table('logs', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('name', sa.Text, nullable=False), - sa.Column('slug', sa.String(50), nullable=False), - sa.Column('stepid', sa.Integer, sa.ForeignKey('steps.id')), - sa.Column('complete', sa.SmallInteger, nullable=False), - sa.Column('num_lines', sa.Integer, nullable=False), - # 's' = stdio, 't' = text, 'h' = html - sa.Column('type', sa.String(1), nullable=False), - ) - - logchunks = sa.Table('logchunks', metadata, - sa.Column('logid', sa.Integer, sa.ForeignKey('logs.id')), - # 0-based line number range in this chunk (inclusive); note that for - # HTML logs, this counts lines of HTML, not lines of rendered output - sa.Column('first_line', sa.Integer, nullable=False), - sa.Column('last_line', sa.Integer, nullable=False), - # log contents, including a terminating newline, encoded in utf-8 or, - # if 'compressed' is true, compressed with gzip - sa.Column('content', sa.LargeBinary(65536)), - sa.Column('compressed', sa.SmallInteger, nullable=False), - ) + logs = sautils.Table( + 'logs', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('name', sa.Text, nullable=False), + sa.Column('slug', sa.String(50), nullable=False), + sa.Column('stepid', sa.Integer, sa.ForeignKey('steps.id')), + sa.Column('complete', sa.SmallInteger, nullable=False), + sa.Column('num_lines', sa.Integer, nullable=False), + # 's' = stdio, 't' = text, 'h' = html + sa.Column('type', sa.String(1), nullable=False), + ) + + logchunks = sautils.Table( + 'logchunks', metadata, + sa.Column('logid', sa.Integer, sa.ForeignKey('logs.id')), + # 0-based line number range in this chunk (inclusive); note that for + # HTML logs, this counts lines of HTML, not lines of rendered output + sa.Column('first_line', sa.Integer, nullable=False), + sa.Column('last_line', sa.Integer, nullable=False), + # log contents, including a terminating newline, encoded in utf-8 or, + # if 'compressed' is true, compressed with gzip + sa.Column('content', sa.LargeBinary(65536)), + sa.Column('compressed', sa.SmallInteger, nullable=False), + ) # buildsets # This table contains input properties for buildsets - buildset_properties = sa.Table('buildset_properties', metadata, - sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), - nullable=False), - sa.Column('property_name', sa.String(256), nullable=False), - # JSON-encoded tuple of (value, source) - sa.Column('property_value', sa.Text, nullable=False), - ) + buildset_properties = sautils.Table( + 'buildset_properties', metadata, + sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), + nullable=False), + sa.Column('property_name', sa.String(256), nullable=False), + # JSON-encoded tuple of (value, source) + sa.Column('property_value', sa.Text, nullable=False), + ) # This table represents Buildsets - sets of BuildRequests that share the # same original cause and source information. - buildsets = sa.Table('buildsets', metadata, - sa.Column('id', sa.Integer, primary_key=True), - - # a simple external identifier to track down this buildset later, e.g., - # for try requests - sa.Column('external_idstring', sa.String(256)), - - # a short string giving the reason the buildset was created - sa.Column('reason', sa.String(256)), - sa.Column('submitted_at', sa.Integer, nullable=False), - - # if this is zero, then the build set is still pending - sa.Column('complete', sa.SmallInteger, nullable=False, - server_default=sa.DefaultClause("0")), - sa.Column('complete_at', sa.Integer), - - # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, - # etc - see master/buildbot/status/builder.py - sa.Column('results', sa.SmallInteger), - - # optional parent build, we use use_alter to prevent circular reference - # http://docs.sqlalchemy.org/en/latest/orm/relationships.html#rows-that-point-to-themselves-mutually-dependent-rows - sa.Column('parent_buildid', sa.Integer, - sa.ForeignKey('builds.id', use_alter=True, name='parent_buildid')), - # text describing what is the relationship with the build - # could be 'triggered from', 'rebuilt from', 'inherited from' - sa.Column('parent_relationship', sa.Text), - ) + buildsets = sautils.Table( + 'buildsets', metadata, + sa.Column('id', sa.Integer, primary_key=True), + + # a simple external identifier to track down this buildset later, e.g., + # for try requests + sa.Column('external_idstring', sa.String(256)), + + # a short string giving the reason the buildset was created + sa.Column('reason', sa.String(256)), + sa.Column('submitted_at', sa.Integer, nullable=False), + + # if this is zero, then the build set is still pending + sa.Column('complete', sa.SmallInteger, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('complete_at', sa.Integer), + + # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS, + # etc - see master/buildbot/status/builder.py + sa.Column('results', sa.SmallInteger), + + # optional parent build, we use use_alter to prevent circular reference + # http://docs.sqlalchemy.org/en/latest/orm/relationships.html#rows-that-point-to-themselves-mutually-dependent-rows + sa.Column('parent_buildid', sa.Integer, + sa.ForeignKey('builds.id', use_alter=True, name='parent_buildid')), + # text describing what is the relationship with the build + # could be 'triggered from', 'rebuilt from', 'inherited from' + sa.Column('parent_relationship', sa.Text), + ) # changesources # The changesources table gives a unique identifier to each ChangeSource. It # also links to other tables used to ensure only one master runs each # changesource - changesources = sa.Table('changesources', metadata, - sa.Column("id", sa.Integer, primary_key=True), + changesources = sautils.Table( + 'changesources', metadata, + sa.Column("id", sa.Integer, primary_key=True), - # name for this changesource, as given in the configuration, plus a hash - # of that name used for a unique index - sa.Column('name', sa.Text, nullable=False), - sa.Column('name_hash', sa.String(40), nullable=False), - ) + # name for this changesource, as given in the configuration, plus a hash + # of that name used for a unique index + sa.Column('name', sa.Text, nullable=False), + sa.Column('name_hash', sa.String(40), nullable=False), + ) # This links changesources to the master where they are running. A changesource # linked to a master that is inactive can be unlinked by any master. This # is a separate table so that we can "claim" changesources on a master by # inserting; this has better support in database servers for ensuring that # exactly one claim succeeds. - changesource_masters = sa.Table('changesource_masters', metadata, - sa.Column('changesourceid', sa.Integer, sa.ForeignKey('changesources.id'), - nullable=False, primary_key=True), - sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), - nullable=False), - ) + changesource_masters = sautils.Table( + 'changesource_masters', metadata, + sa.Column('changesourceid', sa.Integer, sa.ForeignKey('changesources.id'), + nullable=False, primary_key=True), + sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), + nullable=False), + ) # buildslaves - buildslaves = sa.Table("buildslaves", metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column("name", sa.String(50), nullable=False), - sa.Column("info", JsonObject, nullable=False), - ) + buildslaves = sautils.Table( + "buildslaves", metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("name", sa.String(50), nullable=False), + sa.Column("info", JsonObject, nullable=False), + ) # link buildslaves to all builder/master pairs for which they are # configured - configured_buildslaves = sa.Table('configured_buildslaves', metadata, - sa.Column('id', sa.Integer, primary_key=True, nullable=False), - sa.Column('buildermasterid', sa.Integer, - sa.ForeignKey('builder_masters.id'), nullable=False), - sa.Column('buildslaveid', sa.Integer, sa.ForeignKey('buildslaves.id'), - nullable=False), - ) + configured_buildslaves = sautils.Table( + 'configured_buildslaves', metadata, + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('buildermasterid', sa.Integer, + sa.ForeignKey('builder_masters.id'), nullable=False), + sa.Column('buildslaveid', sa.Integer, sa.ForeignKey('buildslaves.id'), + nullable=False), + ) # link buildslaves to the masters they are currently connected to - connected_buildslaves = sa.Table('connected_buildslaves', metadata, - sa.Column('id', sa.Integer, primary_key=True, nullable=False), - sa.Column('masterid', sa.Integer, - sa.ForeignKey('masters.id'), nullable=False), - sa.Column('buildslaveid', sa.Integer, sa.ForeignKey('buildslaves.id'), - nullable=False), - ) + connected_buildslaves = sautils.Table( + 'connected_buildslaves', metadata, + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('masterid', sa.Integer, + sa.ForeignKey('masters.id'), nullable=False), + sa.Column('buildslaveid', sa.Integer, sa.ForeignKey('buildslaves.id'), + nullable=False), + ) # changes # Files touched in changes - change_files = sa.Table('change_files', metadata, - sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), - nullable=False), - sa.Column('filename', sa.String(1024), nullable=False), - ) + change_files = sautils.Table( + 'change_files', metadata, + sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), + nullable=False), + sa.Column('filename', sa.String(1024), nullable=False), + ) # Properties for changes - change_properties = sa.Table('change_properties', metadata, - sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), - nullable=False), - sa.Column('property_name', sa.String(256), nullable=False), - # JSON-encoded tuple of (value, source) - sa.Column('property_value', sa.String(1024), nullable=False), - ) + change_properties = sautils.Table( + 'change_properties', metadata, + sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), + nullable=False), + sa.Column('property_name', sa.String(256), nullable=False), + # JSON-encoded tuple of (value, source) + sa.Column('property_value', sa.String(1024), nullable=False), + ) # users associated with this change; this allows multiple users for # situations where a version-control system can represent both an author # and committer, for example. - change_users = sa.Table("change_users", metadata, - sa.Column("changeid", sa.Integer, sa.ForeignKey('changes.changeid'), - nullable=False), - # uid for the author of the change with the given changeid - sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), - nullable=False) - ) + change_users = sautils.Table( + "change_users", metadata, + sa.Column("changeid", sa.Integer, sa.ForeignKey('changes.changeid'), + nullable=False), + # uid for the author of the change with the given changeid + sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), + nullable=False), + ) # Changes to the source code, produced by ChangeSources - changes = sa.Table('changes', metadata, - # changeid also serves as 'change number' - sa.Column('changeid', sa.Integer, primary_key=True), + changes = sautils.Table( + 'changes', metadata, + # changeid also serves as 'change number' + sa.Column('changeid', sa.Integer, primary_key=True), - # author's name (usually an email address) - sa.Column('author', sa.String(256), nullable=False), + # author's name (usually an email address) + sa.Column('author', sa.String(256), nullable=False), - # commit comment - sa.Column('comments', sa.Text, nullable=False), + # commit comment + sa.Column('comments', sa.Text, nullable=False), - # The branch where this change occurred. When branch is NULL, that - # means the main branch (trunk, master, etc.) - sa.Column('branch', sa.String(256)), + # The branch where this change occurred. When branch is NULL, that + # means the main branch (trunk, master, etc.) + sa.Column('branch', sa.String(256)), - # revision identifier for this change - sa.Column('revision', sa.String(256)), # CVS uses NULL + # revision identifier for this change + sa.Column('revision', sa.String(256)), # CVS uses NULL - sa.Column('revlink', sa.String(256)), + sa.Column('revlink', sa.String(256)), - # this is the timestamp of the change - it is usually copied from the - # version-control system, and may be long in the past or even in the - # future! - sa.Column('when_timestamp', sa.Integer, nullable=False), + # this is the timestamp of the change - it is usually copied from the + # version-control system, and may be long in the past or even in the + # future! + sa.Column('when_timestamp', sa.Integer, nullable=False), - # an arbitrary string used for filtering changes - sa.Column('category', sa.String(256)), + # an arbitrary string used for filtering changes + sa.Column('category', sa.String(256)), - # repository specifies, along with revision and branch, the - # source tree in which this change was detected. - sa.Column('repository', sa.String(length=512), nullable=False, - server_default=''), + # repository specifies, along with revision and branch, the + # source tree in which this change was detected. + sa.Column('repository', sa.String(length=512), nullable=False, + server_default=''), - # codebase is a logical name to specify what is in the repository - sa.Column('codebase', sa.String(256), nullable=False, - server_default=sa.DefaultClause("")), + # codebase is a logical name to specify what is in the repository + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), - # project names the project this source code represents. It is used - # later to filter changes - sa.Column('project', sa.String(length=512), nullable=False, - server_default=''), + # project names the project this source code represents. It is used + # later to filter changes + sa.Column('project', sa.String(length=512), nullable=False, + server_default=''), - # the sourcestamp this change brought the codebase to - sa.Column('sourcestampid', sa.Integer, - sa.ForeignKey('sourcestamps.id')), + # the sourcestamp this change brought the codebase to + sa.Column('sourcestampid', sa.Integer, + sa.ForeignKey('sourcestamps.id')), - # The parent of the change - # Even if for the moment there's only 1 parent for a change, we use plural here because - # somedays a change will have multiple parent. This way we don't need to change the API - sa.Column('parent_changeids', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=True), - ) + # The parent of the change + # Even if for the moment there's only 1 parent for a change, we use plural here because + # somedays a change will have multiple parent. This way we don't need to change the API + sa.Column('parent_changeids', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=True), + ) # sourcestamps # Patches for SourceStamps that were generated through the try mechanism - patches = sa.Table('patches', metadata, - sa.Column('id', sa.Integer, primary_key=True), + patches = sautils.Table( + 'patches', metadata, + sa.Column('id', sa.Integer, primary_key=True), - # number of directory levels to strip off (patch -pN) - sa.Column('patchlevel', sa.Integer, nullable=False), + # number of directory levels to strip off (patch -pN) + sa.Column('patchlevel', sa.Integer, nullable=False), - # base64-encoded version of the patch file - sa.Column('patch_base64', sa.Text, nullable=False), + # base64-encoded version of the patch file + sa.Column('patch_base64', sa.Text, nullable=False), - # patch author, if known - sa.Column('patch_author', sa.Text, nullable=False), + # patch author, if known + sa.Column('patch_author', sa.Text, nullable=False), - # patch comment - sa.Column('patch_comment', sa.Text, nullable=False), + # patch comment + sa.Column('patch_comment', sa.Text, nullable=False), - # subdirectory in which the patch should be applied; NULL for top-level - sa.Column('subdir', sa.Text), - ) + # subdirectory in which the patch should be applied; NULL for top-level + sa.Column('subdir', sa.Text), + ) # A sourcestamp identifies a particular instance of the source code. # Ideally, this would always be absolute, but in practice source stamps can # also mean "latest" (when revision is NULL), which is of course a # time-dependent definition. - sourcestamps = sa.Table('sourcestamps', metadata, - sa.Column('id', sa.Integer, primary_key=True), + sourcestamps = sautils.Table( + 'sourcestamps', metadata, + sa.Column('id', sa.Integer, primary_key=True), - # hash of the branch, revision, patchid, repository, codebase, and - # project, using hashColumns. - sa.Column('ss_hash', sa.String(40), nullable=False), + # hash of the branch, revision, patchid, repository, codebase, and + # project, using hashColumns. + sa.Column('ss_hash', sa.String(40), nullable=False), - # the branch to check out. When branch is NULL, that means - # the main branch (trunk, master, etc.) - sa.Column('branch', sa.String(256)), + # the branch to check out. When branch is NULL, that means + # the main branch (trunk, master, etc.) + sa.Column('branch', sa.String(256)), - # the revision to check out, or the latest if NULL - sa.Column('revision', sa.String(256)), + # the revision to check out, or the latest if NULL + sa.Column('revision', sa.String(256)), - # the patch to apply to generate this source code - sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), + # the patch to apply to generate this source code + sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), - # the repository from which this source should be checked out - sa.Column('repository', sa.String(length=512), nullable=False, - server_default=''), + # the repository from which this source should be checked out + sa.Column('repository', sa.String(length=512), nullable=False, + server_default=''), - # codebase is a logical name to specify what is in the repository - sa.Column('codebase', sa.String(256), nullable=False, - server_default=sa.DefaultClause("")), + # codebase is a logical name to specify what is in the repository + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), - # the project this source code represents - sa.Column('project', sa.String(length=512), nullable=False, - server_default=''), + # the project this source code represents + sa.Column('project', sa.String(length=512), nullable=False, + server_default=''), - # the time this sourcetamp was first seen (the first time it was added) - sa.Column('created_at', sa.Integer, nullable=False), - ) + # the time this sourcetamp was first seen (the first time it was added) + sa.Column('created_at', sa.Integer, nullable=False), + ) # a many-to-may relationship between buildsets and sourcestamps - buildset_sourcestamps = sa.Table('buildset_sourcestamps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('buildsetid', sa.Integer, - sa.ForeignKey('buildsets.id'), - nullable=False), - sa.Column('sourcestampid', sa.Integer, - sa.ForeignKey('sourcestamps.id'), - nullable=False), - ) + buildset_sourcestamps = sautils.Table( + 'buildset_sourcestamps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('buildsetid', sa.Integer, + sa.ForeignKey('buildsets.id'), + nullable=False), + sa.Column('sourcestampid', sa.Integer, + sa.ForeignKey('sourcestamps.id'), + nullable=False), + ) # schedulers @@ -423,14 +446,15 @@ class Model(base.DBConnectorComponent): # also links to other tables used to ensure only one master runs each # scheduler, and to track changes that a scheduler may trigger a build for # later. - schedulers = sa.Table('schedulers', metadata, - sa.Column("id", sa.Integer, primary_key=True), + schedulers = sautils.Table( + 'schedulers', metadata, + sa.Column("id", sa.Integer, primary_key=True), - # name for this scheduler, as given in the configuration, plus a hash - # of that name used for a unique index - sa.Column('name', sa.Text, nullable=False), - sa.Column('name_hash', sa.String(40), nullable=False), - ) + # name for this scheduler, as given in the configuration, plus a hash + # of that name used for a unique index + sa.Column('name', sa.Text, nullable=False), + sa.Column('name_hash', sa.String(40), nullable=False), + ) # This links schedulers to the master where they are running. A scheduler # linked to a master that is inactive can be unlinked by any master. This @@ -438,143 +462,152 @@ class Model(base.DBConnectorComponent): # inserting; this has better support in database servers for ensuring that # exactly one claim succeeds. The ID column is present for external users; # see bug #1053. - scheduler_masters = sa.Table('scheduler_masters', metadata, - sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id'), - nullable=False, primary_key=True), - sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), - nullable=False), - ) + scheduler_masters = sautils.Table( + 'scheduler_masters', metadata, + sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id'), + nullable=False, primary_key=True), + sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), + nullable=False), + ) # This table references "classified" changes that have not yet been # "processed". That is, the scheduler has looked at these changes and # determined that something should be done, but that hasn't happened yet. # Rows are deleted from this table as soon as the scheduler is done with # the change. - scheduler_changes = sa.Table('scheduler_changes', metadata, - sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id')), - sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), - # true (nonzero) if this change is important to this scheduler - sa.Column('important', sa.Integer), - ) + scheduler_changes = sautils.Table( + 'scheduler_changes', metadata, + sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.id')), + sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')), + # true (nonzero) if this change is important to this scheduler + sa.Column('important', sa.Integer), + ) # builders - builders = sa.Table('builders', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # builder's name - sa.Column('name', sa.Text, nullable=False), - # builder's description - sa.Column('description', sa.Text, nullable=True), - # sha1 of name; used for a unique index - sa.Column('name_hash', sa.String(40), nullable=False), - ) + builders = sautils.Table( + 'builders', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # builder's name + sa.Column('name', sa.Text, nullable=False), + # builder's description + sa.Column('description', sa.Text, nullable=True), + # sha1 of name; used for a unique index + sa.Column('name_hash', sa.String(40), nullable=False), + ) # This links builders to the master where they are running. A builder # linked to a master that is inactive can be unlinked by any master. Note # that builders can run on multiple masters at the same time. - builder_masters = sa.Table('builder_masters', metadata, - sa.Column('id', sa.Integer, primary_key=True, nullable=False), - sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'), - nullable=False), - sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), - nullable=False), - ) + builder_masters = sautils.Table( + 'builder_masters', metadata, + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'), + nullable=False), + sa.Column('masterid', sa.Integer, sa.ForeignKey('masters.id'), + nullable=False), + ) # tags - tags = sa.Table('tags', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # tag's name - sa.Column('name', sa.Text, nullable=False), - # sha1 of name; used for a unique index - sa.Column('name_hash', sa.String(40), nullable=False), - ) + tags = sautils.Table( + 'tags', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # tag's name + sa.Column('name', sa.Text, nullable=False), + # sha1 of name; used for a unique index + sa.Column('name_hash', sa.String(40), nullable=False), + ) # a many-to-may relationship between builders and tags - builders_tags = sa.Table('builders_tags', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('builderid', sa.Integer, - sa.ForeignKey('builders.id'), - nullable=False), - sa.Column('tagid', sa.Integer, - sa.ForeignKey('tags.id'), - nullable=False), - ) + builders_tags = sautils.Table( + 'builders_tags', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('builderid', sa.Integer, sa.ForeignKey('builders.id'), + nullable=False), + sa.Column('tagid', sa.Integer, sa.ForeignKey('tags.id'), + nullable=False), + ) # objects # This table uniquely identifies objects that need to maintain state across # invocations. - objects = sa.Table("objects", metadata, - # unique ID for this object - sa.Column("id", sa.Integer, primary_key=True), - # object's user-given name - sa.Column('name', sa.String(128), nullable=False), - # object's class name, basically representing a "type" for the state - sa.Column('class_name', sa.String(128), nullable=False), - ) + objects = sautils.Table( + "objects", metadata, + # unique ID for this object + sa.Column("id", sa.Integer, primary_key=True), + # object's user-given name + sa.Column('name', sa.String(128), nullable=False), + # object's class name, basically representing a "type" for the state + sa.Column('class_name', sa.String(128), nullable=False), + ) # This table stores key/value pairs for objects, where the key is a string # and the value is a JSON string. - object_state = sa.Table("object_state", metadata, - # object for which this value is set - sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), - nullable=False), - # name for this value (local to the object) - sa.Column("name", sa.String(length=256), nullable=False), - # value, as a JSON string - sa.Column("value_json", sa.Text, nullable=False), - ) + object_state = sautils.Table( + "object_state", metadata, + # object for which this value is set + sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), + nullable=False), + # name for this value (local to the object) + sa.Column("name", sa.String(length=256), nullable=False), + # value, as a JSON string + sa.Column("value_json", sa.Text, nullable=False), + ) # users # This table identifies individual users, and contains buildbot-specific # information about those users. - users = sa.Table("users", metadata, - # unique user id number - sa.Column("uid", sa.Integer, primary_key=True), + users = sautils.Table( + "users", metadata, + # unique user id number + sa.Column("uid", sa.Integer, primary_key=True), - # identifier (nickname) for this user; used for display - sa.Column("identifier", sa.String(256), nullable=False), + # identifier (nickname) for this user; used for display + sa.Column("identifier", sa.String(256), nullable=False), - # username portion of user credentials for authentication - sa.Column("bb_username", sa.String(128)), + # username portion of user credentials for authentication + sa.Column("bb_username", sa.String(128)), - # password portion of user credentials for authentication - sa.Column("bb_password", sa.String(128)), - ) + # password portion of user credentials for authentication + sa.Column("bb_password", sa.String(128)), + ) # This table stores information identifying a user that's related to a # particular interface - a version-control system, status plugin, etc. - users_info = sa.Table("users_info", metadata, - # unique user id number - sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), - nullable=False), + users_info = sautils.Table( + "users_info", metadata, + # unique user id number + sa.Column("uid", sa.Integer, sa.ForeignKey('users.uid'), + nullable=False), - # type of user attribute, such as 'git' - sa.Column("attr_type", sa.String(128), nullable=False), + # type of user attribute, such as 'git' + sa.Column("attr_type", sa.String(128), nullable=False), - # data for given user attribute, such as a commit string or password - sa.Column("attr_data", sa.String(128), nullable=False), - ) + # data for given user attribute, such as a commit string or password + sa.Column("attr_data", sa.String(128), nullable=False), + ) # masters - masters = sa.Table("masters", metadata, - # unique id per master - sa.Column('id', sa.Integer, primary_key=True), + masters = sautils.Table( + "masters", metadata, + # unique id per master + sa.Column('id', sa.Integer, primary_key=True), - # master's name (generally in the form hostname:basedir) - sa.Column('name', sa.Text, nullable=False), - # sha1 of name; used for a unique index - sa.Column('name_hash', sa.String(40), nullable=False), + # master's name (generally in the form hostname:basedir) + sa.Column('name', sa.Text, nullable=False), + # sha1 of name; used for a unique index + sa.Column('name_hash', sa.String(40), nullable=False), - # true if this master is running - sa.Column('active', sa.Integer, nullable=False), + # true if this master is running + sa.Column('active', sa.Integer, nullable=False), - # updated periodically by a running master, so silently failed masters - # can be detected by other masters - sa.Column('last_active', sa.Integer, nullable=False), - ) + # updated periodically by a running master, so silently failed masters + # can be detected by other masters + sa.Column('last_active', sa.Integer, nullable=False), + ) # indexes diff --git a/master/buildbot/test/unit/test_db_migrate_versions_011_add_buildrequest_claims.py b/master/buildbot/test/unit/test_db_migrate_versions_011_add_buildrequest_claims.py index 18b6f3fab23..ab02ba39c6c 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_011_add_buildrequest_claims.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_011_add_buildrequest_claims.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from sqlalchemy.engine import reflection from twisted.trial import unittest @@ -32,37 +33,39 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - self.buildsets = sa.Table('buildsets', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('external_idstring', sa.String(256)), - sa.Column('reason', sa.String(256)), - sa.Column('sourcestampid', sa.Integer, - nullable=False), # NOTE: foreign key omitted - sa.Column('submitted_at', sa.Integer, nullable=False), - sa.Column('complete', sa.SmallInteger, nullable=False, - server_default=sa.DefaultClause("0")), - sa.Column('complete_at', sa.Integer), - sa.Column('results', sa.SmallInteger), - ) + self.buildsets = sautils.Table( + 'buildsets', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('external_idstring', sa.String(256)), + sa.Column('reason', sa.String(256)), + sa.Column('sourcestampid', sa.Integer, + nullable=False), # NOTE: foreign key omitted + sa.Column('submitted_at', sa.Integer, nullable=False), + sa.Column('complete', sa.SmallInteger, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('complete_at', sa.Integer), + sa.Column('results', sa.SmallInteger), + ) self.buildsets.create(bind=conn) - self.buildrequests = sa.Table('buildrequests', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), - nullable=False), - sa.Column('buildername', sa.String(length=256), nullable=False), - sa.Column('priority', sa.Integer, nullable=False, - server_default=sa.DefaultClause("0")), - sa.Column('claimed_at', sa.Integer, - server_default=sa.DefaultClause("0")), - sa.Column('claimed_by_name', sa.String(length=256)), - sa.Column('claimed_by_incarnation', sa.String(length=256)), - sa.Column('complete', sa.Integer, - server_default=sa.DefaultClause("0")), - sa.Column('results', sa.SmallInteger), - sa.Column('submitted_at', sa.Integer, nullable=False), - sa.Column('complete_at', sa.Integer), - ) + self.buildrequests = sautils.Table( + 'buildrequests', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), + nullable=False), + sa.Column('buildername', sa.String(length=256), nullable=False), + sa.Column('priority', sa.Integer, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('claimed_at', sa.Integer, + server_default=sa.DefaultClause("0")), + sa.Column('claimed_by_name', sa.String(length=256)), + sa.Column('claimed_by_incarnation', sa.String(length=256)), + sa.Column('complete', sa.Integer, + server_default=sa.DefaultClause("0")), + sa.Column('results', sa.SmallInteger), + sa.Column('submitted_at', sa.Integer, nullable=False), + sa.Column('complete_at', sa.Integer), + ) self.buildrequests.create(bind=conn) idx = sa.Index('buildrequests_buildsetid', @@ -77,12 +80,13 @@ def create_tables_thd(self, conn): self.buildrequests.c.complete) idx.create() - self.objects = sa.Table("objects", metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column('name', sa.String(128), nullable=False), - sa.Column('class_name', sa.String(128), nullable=False), - sa.UniqueConstraint('name', 'class_name', name='object_identity'), - ) + self.objects = sautils.Table( + "objects", metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column('name', sa.String(128), nullable=False), + sa.Column('class_name', sa.String(128), nullable=False), + sa.UniqueConstraint('name', 'class_name', name='object_identity'), + ) self.objects.create(bind=conn) # tests diff --git a/master/buildbot/test/unit/test_db_migrate_versions_015_remove_bad_master_objectid.py b/master/buildbot/test/unit/test_db_migrate_versions_015_remove_bad_master_objectid.py index 904f72b517c..87933749552 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_015_remove_bad_master_objectid.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_015_remove_bad_master_objectid.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -29,19 +30,21 @@ def tearDown(self): def create_tables_thd(self, conn): metadata = sa.MetaData() - self.objects = sa.Table("objects", metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column('name', sa.String(128), nullable=False), - sa.Column('class_name', sa.String(128), nullable=False), - sa.UniqueConstraint('name', 'class_name', name='object_identity'), - ) - self.object_state = sa.Table("object_state", metadata, - sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), - nullable=False), - sa.Column("name", sa.String(length=256), nullable=False), - sa.Column("value_json", sa.Text, nullable=False), - sa.UniqueConstraint('objectid', 'name', name='name_per_object'), - ) + self.objects = sautils.Table( + "objects", metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column('name', sa.String(128), nullable=False), + sa.Column('class_name', sa.String(128), nullable=False), + sa.UniqueConstraint('name', 'class_name', name='object_identity'), + ) + self.object_state = sautils.Table( + "object_state", metadata, + sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), + nullable=False), + sa.Column("name", sa.String(length=256), nullable=False), + sa.Column("value_json", sa.Text, nullable=False), + sa.UniqueConstraint('objectid', 'name', name='name_per_object'), + ) self.objects.create(bind=conn) self.object_state.create(bind=conn) diff --git a/master/buildbot/test/unit/test_db_migrate_versions_016_restore_buildrequest_indices.py b/master/buildbot/test/unit/test_db_migrate_versions_016_restore_buildrequest_indices.py index f9f0074aa11..e79f9eee11b 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_016_restore_buildrequest_indices.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_016_restore_buildrequest_indices.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from sqlalchemy.engine import reflection from twisted.trial import unittest @@ -32,19 +33,20 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - self.buildrequests = sa.Table('buildrequests', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('buildsetid', sa.Integer, # foreign key removed - nullable=False), - sa.Column('buildername', sa.String(length=256), nullable=False), - sa.Column('priority', sa.Integer, nullable=False, - server_default=sa.DefaultClause("0")), - sa.Column('complete', sa.Integer, - server_default=sa.DefaultClause("0")), - sa.Column('results', sa.SmallInteger), - sa.Column('submitted_at', sa.Integer, nullable=False), - sa.Column('complete_at', sa.Integer), - ) + self.buildrequests = sautils.Table( + 'buildrequests', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('buildsetid', sa.Integer, # foreign key removed + nullable=False), + sa.Column('buildername', sa.String(length=256), nullable=False), + sa.Column('priority', sa.Integer, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('complete', sa.Integer, + server_default=sa.DefaultClause("0")), + sa.Column('results', sa.SmallInteger), + sa.Column('submitted_at', sa.Integer, nullable=False), + sa.Column('complete_at', sa.Integer), + ) self.buildrequests.create(bind=conn) # these indices should already exist everywhere but on sqlite diff --git a/master/buildbot/test/unit/test_db_migrate_versions_017_restore_other_indices.py b/master/buildbot/test/unit/test_db_migrate_versions_017_restore_other_indices.py index b3a0f3f2ead..e195b1c048a 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_017_restore_other_indices.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_017_restore_other_indices.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from sqlalchemy.engine import reflection from twisted.trial import unittest @@ -32,51 +33,56 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - self.changes = sa.Table('changes', metadata, - sa.Column('changeid', sa.Integer, primary_key=True), - sa.Column('author', sa.String(256), nullable=False), - sa.Column('comments', sa.String(1024), nullable=False), - sa.Column('is_dir', sa.SmallInteger, nullable=False), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('revlink', sa.String(256)), - sa.Column('when_timestamp', sa.Integer, nullable=False), - sa.Column('category', sa.String(256)), - sa.Column('repository', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('project', sa.String(length=512), nullable=False, - server_default=''), - ) + self.changes = sautils.Table( + 'changes', metadata, + sa.Column('changeid', sa.Integer, primary_key=True), + sa.Column('author', sa.String(256), nullable=False), + sa.Column('comments', sa.String(1024), nullable=False), + sa.Column('is_dir', sa.SmallInteger, nullable=False), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('revlink', sa.String(256)), + sa.Column('when_timestamp', sa.Integer, nullable=False), + sa.Column('category', sa.String(256)), + sa.Column('repository', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('project', sa.String(length=512), nullable=False, + server_default=''), + ) self.changes.create(bind=conn) - self.schedulers = sa.Table("schedulers", metadata, - sa.Column('schedulerid', sa.Integer, primary_key=True), - sa.Column('name', sa.String(128), nullable=False), - sa.Column('class_name', sa.String(128), nullable=False), - ) + self.schedulers = sautils.Table( + "schedulers", metadata, + sa.Column('schedulerid', sa.Integer, primary_key=True), + sa.Column('name', sa.String(128), nullable=False), + sa.Column('class_name', sa.String(128), nullable=False), + ) self.schedulers.create(bind=conn) - self.users = sa.Table("users", metadata, - sa.Column("uid", sa.Integer, primary_key=True), - sa.Column("identifier", sa.String(256), nullable=False), - sa.Column("bb_username", sa.String(128)), - sa.Column("bb_password", sa.String(128)), - ) + self.users = sautils.Table( + "users", metadata, + sa.Column("uid", sa.Integer, primary_key=True), + sa.Column("identifier", sa.String(256), nullable=False), + sa.Column("bb_username", sa.String(128)), + sa.Column("bb_password", sa.String(128)), + ) self.users.create(bind=conn) - self.objects = sa.Table("objects", metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column('name', sa.String(128), nullable=False), - sa.Column('class_name', sa.String(128), nullable=False), - ) + self.objects = sautils.Table( + "objects", metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column('name', sa.String(128), nullable=False), + sa.Column('class_name', sa.String(128), nullable=False), + ) self.objects.create() - self.object_state = sa.Table("object_state", metadata, - sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), - nullable=False), - sa.Column("name", sa.String(length=256), nullable=False), - sa.Column("value_json", sa.Text, nullable=False), - ) + self.object_state = sautils.Table( + "object_state", metadata, + sa.Column("objectid", sa.Integer, sa.ForeignKey('objects.id'), + nullable=False), + sa.Column("name", sa.String(length=256), nullable=False), + sa.Column("value_json", sa.Text, nullable=False), + ) self.object_state.create() # these indices should already exist everywhere but on sqlite @@ -101,8 +107,7 @@ def setup_thd(conn): def verify_thd(conn): insp = reflection.Inspector.from_engine(conn) - indexes = (insp.get_indexes('changes') - + insp.get_indexes('schedulers')) + indexes = (insp.get_indexes('changes') + insp.get_indexes('schedulers')) self.assertEqual( sorted([i['name'] for i in indexes]), sorted([ diff --git a/master/buildbot/test/unit/test_db_migrate_versions_018_add_sourcestampset.py b/master/buildbot/test/unit/test_db_migrate_versions_018_add_sourcestampset.py index d78ec12a0de..5ddf243c265 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_018_add_sourcestampset.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_018_add_sourcestampset.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from sqlalchemy.engine import reflection from twisted.python import log from twisted.trial import unittest @@ -33,41 +34,44 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - self.buildsets = sa.Table('buildsets', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('external_idstring', sa.String(256)), - sa.Column('reason', sa.String(256)), - sa.Column('sourcestampid', sa.Integer, - nullable=False), # NOTE: foreign key omitted - sa.Column('submitted_at', sa.Integer, nullable=False), - sa.Column('complete', sa.SmallInteger, nullable=False, - server_default=sa.DefaultClause("0")), - sa.Column('complete_at', sa.Integer), - sa.Column('results', sa.SmallInteger), - ) + self.buildsets = sautils.Table( + 'buildsets', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('external_idstring', sa.String(256)), + sa.Column('reason', sa.String(256)), + # NOTE: foreign key omitted: + sa.Column('sourcestampid', sa.Integer, nullable=False), + sa.Column('submitted_at', sa.Integer, nullable=False), + sa.Column('complete', sa.SmallInteger, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('complete_at', sa.Integer), + sa.Column('results', sa.SmallInteger), + ) self.buildsets.create(bind=conn) sa.Index('buildsets_complete', self.buildsets.c.complete).create() sa.Index('buildsets_submitted_at', self.buildsets.c.submitted_at).create() - self.patches = sa.Table('patches', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('patchlevel', sa.Integer, nullable=False), - sa.Column('patch_base64', sa.Text, nullable=False), - sa.Column('patch_author', sa.Text, nullable=False), - sa.Column('patch_comment', sa.Text, nullable=False), - sa.Column('subdir', sa.Text), - ) + self.patches = sautils.Table( + 'patches', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('patchlevel', sa.Integer, nullable=False), + sa.Column('patch_base64', sa.Text, nullable=False), + sa.Column('patch_author', sa.Text, nullable=False), + sa.Column('patch_comment', sa.Text, nullable=False), + sa.Column('subdir', sa.Text), + ) self.patches.create(bind=conn) - self.sourcestamps = sa.Table('sourcestamps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), - sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), - sa.Column('project', sa.String(length=512), nullable=False, server_default=''), - sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id')), - ) + self.sourcestamps = sautils.Table( + 'sourcestamps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), + sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), + sa.Column('project', sa.String(length=512), nullable=False, server_default=''), + sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id')), + ) self.sourcestamps.create(bind=conn) def fill_tables_with_testdata(self, conn, testdata): diff --git a/master/buildbot/test/unit/test_db_migrate_versions_019_merge_schedulers_to_objects.py b/master/buildbot/test/unit/test_db_migrate_versions_019_merge_schedulers_to_objects.py index 313ecc76450..54733da46c9 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_019_merge_schedulers_to_objects.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_019_merge_schedulers_to_objects.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -31,34 +32,38 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - changes = sa.Table('changes', metadata, - sa.Column('changeid', sa.Integer, primary_key=True), - # the rest is unimportant - ) + changes = sautils.Table( + 'changes', metadata, + sa.Column('changeid', sa.Integer, primary_key=True), + # the rest is unimportant + ) changes.create() - buildsets = sa.Table('buildsets', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # the rest is unimportant - ) + buildsets = sautils.Table( + 'buildsets', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # the rest is unimportant + ) buildsets.create() - self.schedulers = sa.Table("schedulers", metadata, - sa.Column('schedulerid', sa.Integer, primary_key=True), - sa.Column('name', sa.String(128), nullable=False), - sa.Column('class_name', sa.String(128), nullable=False), - ) + self.schedulers = sautils.Table( + "schedulers", metadata, + sa.Column('schedulerid', sa.Integer, primary_key=True), + sa.Column('name', sa.String(128), nullable=False), + sa.Column('class_name', sa.String(128), nullable=False), + ) self.schedulers.create(bind=conn) sa.Index('name_and_class', self.schedulers.c.name, self.schedulers.c.class_name).create() - self.scheduler_changes = sa.Table('scheduler_changes', metadata, - sa.Column('schedulerid', sa.Integer, - sa.ForeignKey('schedulers.schedulerid')), - sa.Column('changeid', sa.Integer, - sa.ForeignKey('changes.changeid')), - sa.Column('important', sa.SmallInteger), - ) + self.scheduler_changes = sautils.Table( + 'scheduler_changes', metadata, + sa.Column('schedulerid', sa.Integer, + sa.ForeignKey('schedulers.schedulerid')), + sa.Column('changeid', sa.Integer, + sa.ForeignKey('changes.changeid')), + sa.Column('important', sa.SmallInteger), + ) self.scheduler_changes.create() sa.Index('scheduler_changes_schedulerid', self.scheduler_changes.c.schedulerid).create() @@ -68,7 +73,7 @@ def create_tables_thd(self, conn): self.scheduler_changes.c.schedulerid, self.scheduler_changes.c.changeid, unique=True).create() - self.scheduler_upstream_buildsets = sa.Table( + self.scheduler_upstream_buildsets = sautils.Table( 'scheduler_upstream_buildsets', metadata, sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id')), sa.Column('schedulerid', sa.Integer, @@ -81,11 +86,12 @@ def create_tables_thd(self, conn): sa.Index('scheduler_upstream_buildsets_schedulerid', self.scheduler_upstream_buildsets.c.schedulerid).create() - self.objects = sa.Table("objects", metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column('name', sa.String(128), nullable=False), - sa.Column('class_name', sa.String(128), nullable=False), - ) + self.objects = sautils.Table( + "objects", metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column('name', sa.String(128), nullable=False), + sa.Column('class_name', sa.String(128), nullable=False), + ) self.objects.create(bind=conn) sa.Index('object_identity', self.objects.c.name, diff --git a/master/buildbot/test/unit/test_db_migrate_versions_020_remove_change_links.py b/master/buildbot/test/unit/test_db_migrate_versions_020_remove_change_links.py index b69d9838cf0..79eb94c2a40 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_020_remove_change_links.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_020_remove_change_links.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -31,18 +32,20 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - changes = sa.Table('changes', metadata, - sa.Column('changeid', sa.Integer, primary_key=True), - # the rest is unimportant - ) + changes = sautils.Table( + 'changes', metadata, + sa.Column('changeid', sa.Integer, primary_key=True), + # the rest is unimportant + ) changes.create() # Links (URLs) for changes - change_links = sa.Table('change_links', metadata, - sa.Column('changeid', sa.Integer, - sa.ForeignKey('changes.changeid'), nullable=False), - sa.Column('link', sa.String(1024), nullable=False), - ) + change_links = sautils.Table( + 'change_links', metadata, + sa.Column('changeid', sa.Integer, + sa.ForeignKey('changes.changeid'), nullable=False), + sa.Column('link', sa.String(1024), nullable=False), + ) change_links.create() sa.Index('change_links_changeid', change_links.c.changeid).create() diff --git a/master/buildbot/test/unit/test_db_migrate_versions_021_fix_postgres_sequences.py b/master/buildbot/test/unit/test_db_migrate_versions_021_fix_postgres_sequences.py index 9be2f1a2ae4..bb94484f157 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_021_fix_postgres_sequences.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_021_fix_postgres_sequences.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -51,8 +52,9 @@ def setup_thd(conn): # one table to test that corner case for i, col in enumerate(self.cols): tbl_name, col_name = col.split('.') - tbl = sa.Table(tbl_name, metadata, - sa.Column(col_name, sa.Integer, primary_key=True)) + tbl = sautils.Table( + tbl_name, metadata, + sa.Column(col_name, sa.Integer, primary_key=True)) tbl.create() if i > 1: conn.execute(tbl.insert(), {col_name: i}) @@ -65,8 +67,9 @@ def verify_thd(conn): # is as expected for i, col in enumerate(self.cols): tbl_name, col_name = col.split('.') - tbl = sa.Table(tbl_name, metadata, - sa.Column(col_name, sa.Integer, primary_key=True)) + tbl = sautils.Table( + tbl_name, metadata, + sa.Column(col_name, sa.Integer, primary_key=True)) r = conn.execute(tbl.insert(), {}) if i > 1: exp = i + 1 diff --git a/master/buildbot/test/unit/test_db_migrate_versions_022_add_codebase.py b/master/buildbot/test/unit/test_db_migrate_versions_022_add_codebase.py index 37d79731148..895342f9af9 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_022_add_codebase.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_022_add_codebase.py @@ -19,6 +19,7 @@ from buildbot.test.util import migration from buildbot.util import UTC from buildbot.util import datetime2epoch +from buildbot.util import sautils from twisted.trial import unittest @@ -35,30 +36,32 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - self.sourcestamps = sa.Table('sourcestamps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('patchid', sa.Integer), - sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), - sa.Column('project', sa.String(length=512), nullable=False, server_default=''), - sa.Column('sourcestampsetid', sa.Integer), - ) + self.sourcestamps = sautils.Table( + 'sourcestamps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('patchid', sa.Integer), + sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), + sa.Column('project', sa.String(length=512), nullable=False, server_default=''), + sa.Column('sourcestampsetid', sa.Integer), + ) self.sourcestamps.create(bind=conn) - self.changes = sa.Table('changes', metadata, - sa.Column('changeid', sa.Integer, primary_key=True), - sa.Column('author', sa.String(256), nullable=False), - sa.Column('comments', sa.String(1024), nullable=False), - sa.Column('is_dir', sa.SmallInteger, nullable=False), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('revlink', sa.String(256)), - sa.Column('when_timestamp', sa.Integer, nullable=False), - sa.Column('category', sa.String(256)), - sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), - sa.Column('project', sa.String(length=512), nullable=False, server_default=''), - ) + self.changes = sautils.Table( + 'changes', metadata, + sa.Column('changeid', sa.Integer, primary_key=True), + sa.Column('author', sa.String(256), nullable=False), + sa.Column('comments', sa.String(1024), nullable=False), + sa.Column('is_dir', sa.SmallInteger, nullable=False), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('revlink', sa.String(256)), + sa.Column('when_timestamp', sa.Integer, nullable=False), + sa.Column('category', sa.String(256)), + sa.Column('repository', sa.String(length=512), nullable=False, server_default=''), + sa.Column('project', sa.String(length=512), nullable=False, server_default=''), + ) self.changes.create(bind=conn) def reload_tables_after_migration(self, conn): diff --git a/master/buildbot/test/unit/test_db_migrate_versions_023_increase_comments_property_lengths.py b/master/buildbot/test/unit/test_db_migrate_versions_023_increase_comments_property_lengths.py index 8c32907cd22..717a277ee77 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_023_increase_comments_property_lengths.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_023_increase_comments_property_lengths.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -36,43 +37,46 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - changes = sa.Table('changes', metadata, - sa.Column('changeid', sa.Integer, primary_key=True), - sa.Column('author', sa.String(256), nullable=False), - sa.Column('comments', sa.String(1024), nullable=False), - sa.Column('is_dir', sa.SmallInteger, nullable=False), # old, for CVS - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), # CVS uses NULL - sa.Column('revlink', sa.String(256)), - sa.Column('when_timestamp', sa.Integer, nullable=False), - sa.Column('category', sa.String(256)), - sa.Column('repository', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('codebase', sa.String(256), nullable=False, - server_default=sa.DefaultClause("")), - sa.Column('project', sa.String(length=512), nullable=False, - server_default=''), - ) + changes = sautils.Table( + 'changes', metadata, + sa.Column('changeid', sa.Integer, primary_key=True), + sa.Column('author', sa.String(256), nullable=False), + sa.Column('comments', sa.String(1024), nullable=False), + sa.Column('is_dir', sa.SmallInteger, nullable=False), # old, for CVS + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), # CVS uses NULL + sa.Column('revlink', sa.String(256)), + sa.Column('when_timestamp', sa.Integer, nullable=False), + sa.Column('category', sa.String(256)), + sa.Column('repository', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), + sa.Column('project', sa.String(length=512), nullable=False, + server_default=''), + ) changes.create() - buildsets = sa.Table('buildsets', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('external_idstring', sa.String(256)), - sa.Column('reason', sa.String(256)), - sa.Column('submitted_at', sa.Integer, nullable=False), - sa.Column('complete', sa.SmallInteger, nullable=False, - server_default=sa.DefaultClause("0")), - sa.Column('complete_at', sa.Integer), - sa.Column('results', sa.SmallInteger), - sa.Column('sourcestampsetid', sa.Integer) # foreign key omitted - ) + buildsets = sautils.Table( + 'buildsets', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('external_idstring', sa.String(256)), + sa.Column('reason', sa.String(256)), + sa.Column('submitted_at', sa.Integer, nullable=False), + sa.Column('complete', sa.SmallInteger, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('complete_at', sa.Integer), + sa.Column('results', sa.SmallInteger), + sa.Column('sourcestampsetid', sa.Integer), # foreign key omitted + ) buildsets.create() - buildset_properties = sa.Table('buildset_properties', metadata, - sa.Column('buildsetid', sa.Integer, nullable=False), - sa.Column('property_name', sa.String(256), nullable=False), - sa.Column('property_value', sa.String(1024), nullable=False), - ) + buildset_properties = sautils.Table( + 'buildset_properties', metadata, + sa.Column('buildsetid', sa.Integer, nullable=False), + sa.Column('property_name', sa.String(256), nullable=False), + sa.Column('property_value', sa.String(1024), nullable=False), + ) buildset_properties.create() # tests diff --git a/master/buildbot/test/unit/test_db_migrate_versions_025_add_master_table.py b/master/buildbot/test/unit/test_db_migrate_versions_025_add_master_table.py index 16800e7c773..0b4f1696bf8 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_025_add_master_table.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_025_add_master_table.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -28,27 +29,30 @@ def tearDown(self): return self.tearDownMigrateTest() def _createTables_thd(self, conn, metadata): - objects = sa.Table("objects", metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column('name', sa.String(128), nullable=False), - sa.Column('class_name', sa.String(128), nullable=False), - ) + objects = sautils.Table( + "objects", metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column('name', sa.String(128), nullable=False), + sa.Column('class_name', sa.String(128), nullable=False), + ) objects.create() - buildrequest_claims = sa.Table('buildrequest_claims', metadata, - sa.Column('brid', sa.Integer, index=True, unique=True), - sa.Column('objectid', sa.Integer, index=True, nullable=True), - sa.Column('claimed_at', sa.Integer, nullable=False), - ) + buildrequest_claims = sautils.Table( + 'buildrequest_claims', metadata, + sa.Column('brid', sa.Integer, index=True, unique=True), + sa.Column('objectid', sa.Integer, index=True, nullable=True), + sa.Column('claimed_at', sa.Integer, nullable=False), + ) buildrequest_claims.create() sa.Index('buildrequest_claims_brids', buildrequest_claims.c.brid, unique=True).create() - buildrequests = sa.Table('buildrequests', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # .. - ) + buildrequests = sautils.Table( + 'buildrequests', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # .. + ) buildrequests.create() def test_empty_migration(self): diff --git a/master/buildbot/test/unit/test_db_migrate_versions_026_add_schedulers_table.py b/master/buildbot/test/unit/test_db_migrate_versions_026_add_schedulers_table.py index 472d3e8647c..391c352f9e4 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_026_add_schedulers_table.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_026_add_schedulers_table.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -31,22 +32,25 @@ def test_migration(self): def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - scheduler_changes = sa.Table('scheduler_changes', metadata, - sa.Column('objectid', sa.Integer), - sa.Column('changeid', sa.Integer), - # .. - ) + scheduler_changes = sautils.Table( + 'scheduler_changes', metadata, + sa.Column('objectid', sa.Integer), + sa.Column('changeid', sa.Integer), + # .. + ) scheduler_changes.create() - sa.Table('masters', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # .. - ).create() - - sa.Table('changes', metadata, - sa.Column('changeid', sa.Integer, primary_key=True), - # .. - ).create() + sautils.Table( + 'masters', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # .. + ).create() + + sautils.Table( + 'changes', metadata, + sa.Column('changeid', sa.Integer, primary_key=True), + # .. + ).create() idx = sa.Index('scheduler_changes_objectid', scheduler_changes.c.objectid) diff --git a/master/buildbot/test/unit/test_db_migrate_versions_027_builders_table.py b/master/buildbot/test/unit/test_db_migrate_versions_027_builders_table.py index 5b4eb9fbf05..2caf6d035cc 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_027_builders_table.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_027_builders_table.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -31,10 +32,11 @@ def test_migration(self): def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - sa.Table('masters', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # .. - ).create() + sautils.Table( + 'masters', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # .. + ).create() def verify_thd(conn): metadata = sa.MetaData() diff --git a/master/buildbot/test/unit/test_db_migrate_versions_028_sourcestamps_refactor.py b/master/buildbot/test/unit/test_db_migrate_versions_028_sourcestamps_refactor.py index d9f94f0e82c..c4910a3aead 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_028_sourcestamps_refactor.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_028_sourcestamps_refactor.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -32,72 +33,78 @@ def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - patches = sa.Table('patches', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # ... - ) + patches = sautils.Table( + 'patches', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # ... + ) patches.create() - sourcestampsets = sa.Table('sourcestampsets', metadata, - sa.Column('id', sa.Integer, primary_key=True), - ) + sourcestampsets = sautils.Table( + 'sourcestampsets', metadata, + sa.Column('id', sa.Integer, primary_key=True), + ) sourcestampsets.create() - sourcestamps = sa.Table('sourcestamps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), - sa.Column('repository', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('codebase', sa.String(256), nullable=False, - server_default=sa.DefaultClause("")), - sa.Column('project', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('sourcestampsetid', sa.Integer, - sa.ForeignKey('sourcestampsets.id')), - ) + sourcestamps = sautils.Table( + 'sourcestamps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), + sa.Column('repository', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), + sa.Column('project', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('sourcestampsetid', sa.Integer, + sa.ForeignKey('sourcestampsets.id')), + ) sourcestamps.create() - buildsets = sa.Table('buildsets', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('external_idstring', sa.String(256)), - sa.Column('reason', sa.String(256)), - sa.Column('submitted_at', sa.Integer, nullable=False), - sa.Column('complete', sa.SmallInteger, nullable=False, - server_default=sa.DefaultClause("0")), - sa.Column('complete_at', sa.Integer), - sa.Column('results', sa.SmallInteger), - sa.Column('sourcestampsetid', sa.Integer, - sa.ForeignKey('sourcestampsets.id')), - ) + buildsets = sautils.Table( + 'buildsets', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('external_idstring', sa.String(256)), + sa.Column('reason', sa.String(256)), + sa.Column('submitted_at', sa.Integer, nullable=False), + sa.Column('complete', sa.SmallInteger, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('complete_at', sa.Integer), + sa.Column('results', sa.SmallInteger), + sa.Column('sourcestampsetid', sa.Integer, + sa.ForeignKey('sourcestampsets.id')), + ) buildsets.create() - changes = sa.Table('changes', metadata, - sa.Column('changeid', sa.Integer, primary_key=True), - sa.Column('author', sa.String(256), nullable=False), - sa.Column('comments', sa.String(1024), nullable=False), - sa.Column('is_dir', sa.SmallInteger, nullable=False), # old, for CVS - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), # CVS uses NULL - sa.Column('revlink', sa.String(256)), - sa.Column('when_timestamp', sa.Integer, nullable=False), - sa.Column('category', sa.String(256)), - sa.Column('repository', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('codebase', sa.String(256), nullable=False, - server_default=sa.DefaultClause("")), - sa.Column('project', sa.String(length=512), nullable=False, - server_default=''), - ) + changes = sautils.Table( + 'changes', metadata, + sa.Column('changeid', sa.Integer, primary_key=True), + sa.Column('author', sa.String(256), nullable=False), + sa.Column('comments', sa.String(1024), nullable=False), + sa.Column('is_dir', sa.SmallInteger, nullable=False), # old, for CVS + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), # CVS uses NULL + sa.Column('revlink', sa.String(256)), + sa.Column('when_timestamp', sa.Integer, nullable=False), + sa.Column('category', sa.String(256)), + sa.Column('repository', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), + sa.Column('project', sa.String(length=512), nullable=False, + server_default=''), + ) changes.create() - sourcestamp_changes = sa.Table('sourcestamp_changes', metadata, - sa.Column('sourcestampid', sa.Integer, - sa.ForeignKey('sourcestamps.id'), nullable=False), - sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), - nullable=False), - ) + sourcestamp_changes = sautils.Table( + 'sourcestamp_changes', metadata, + sa.Column('sourcestampid', sa.Integer, + sa.ForeignKey('sourcestamps.id'), nullable=False), + sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), + nullable=False), + ) sourcestamp_changes.create() # now insert some data.. diff --git a/master/buildbot/test/unit/test_db_migrate_versions_029_replace_builds_table.py b/master/buildbot/test/unit/test_db_migrate_versions_029_replace_builds_table.py index 325673c583e..d31908b6268 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_029_replace_builds_table.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_029_replace_builds_table.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -32,29 +33,33 @@ def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - buildrequests = sa.Table('buildrequests', metadata, - sa.Column('id', sa.Integer, primary_key=True), - ) + buildrequests = sautils.Table( + 'buildrequests', metadata, + sa.Column('id', sa.Integer, primary_key=True), + ) buildrequests.create() - builders = sa.Table('builders', metadata, - sa.Column('id', sa.Integer, primary_key=True), - ) + builders = sautils.Table( + 'builders', metadata, + sa.Column('id', sa.Integer, primary_key=True), + ) builders.create() - masters = sa.Table("masters", metadata, - sa.Column('id', sa.Integer, primary_key=True), - ) + masters = sautils.Table( + "masters", metadata, + sa.Column('id', sa.Integer, primary_key=True), + ) masters.create() - builds = sa.Table('builds', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('number', sa.Integer, nullable=False), - sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), - nullable=False), - sa.Column('start_time', sa.Integer, nullable=False), - sa.Column('finish_time', sa.Integer), - ) + builds = sautils.Table( + 'builds', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('number', sa.Integer, nullable=False), + sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), + nullable=False), + sa.Column('start_time', sa.Integer, nullable=False), + sa.Column('finish_time', sa.Integer), + ) builds.create() def verify_thd(conn): diff --git a/master/buildbot/test/unit/test_db_migrate_versions_030_statusdb_tables.py b/master/buildbot/test/unit/test_db_migrate_versions_030_statusdb_tables.py index 083f448952d..8946ec9389f 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_030_statusdb_tables.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_030_statusdb_tables.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -32,9 +33,10 @@ def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - sa.Table('builds', metadata, - sa.Column('id', sa.Integer, primary_key=True), - ).create() + sautils.Table( + 'builds', metadata, + sa.Column('id', sa.Integer, primary_key=True), + ).create() def verify_thd(conn): r = conn.execute("select * from steps") diff --git a/master/buildbot/test/unit/test_db_migrate_versions_031_add_changesources_table.py b/master/buildbot/test/unit/test_db_migrate_versions_031_add_changesources_table.py index d7a8de2e894..b67afd743fb 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_031_add_changesources_table.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_031_add_changesources_table.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -32,10 +33,11 @@ def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - sa.Table('masters', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # .. - ).create() + sautils.Table( + 'masters', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # .. + ).create() def verify_thd(conn): metadata = sa.MetaData() diff --git a/master/buildbot/test/unit/test_db_migrate_versions_032_slave_connections.py b/master/buildbot/test/unit/test_db_migrate_versions_032_slave_connections.py index 95e564b444a..c661e0596b3 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_032_slave_connections.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_032_slave_connections.py @@ -17,6 +17,7 @@ from buildbot.db.types.json import JsonObject from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -33,19 +34,22 @@ def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - sa.Table('builder_masters', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # .. - ).create() - sa.Table('masters', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # .. - ).create() - buildslaves = sa.Table("buildslaves", metadata, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column("name", sa.String(256), nullable=False), - sa.Column("info", JsonObject, nullable=False), - ) + sautils.Table( + 'builder_masters', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # .. + ).create() + sautils.Table( + 'masters', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # .. + ).create() + buildslaves = sautils.Table( + "buildslaves", metadata, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("name", sa.String(256), nullable=False), + sa.Column("info", JsonObject, nullable=False), + ) buildslaves.create() conn.execute(buildslaves.insert(), { 'id': 29, diff --git a/master/buildbot/test/unit/test_db_migrate_versions_033_buildrequests_waited_for.py b/master/buildbot/test/unit/test_db_migrate_versions_033_buildrequests_waited_for.py index db398b28562..9ba50431a31 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_033_buildrequests_waited_for.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_033_buildrequests_waited_for.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -32,19 +33,20 @@ def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - buildrequests = sa.Table('buildrequests', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('buildsetid', sa.Integer, nullable=False), - sa.Column('buildername', sa.String(length=256), - nullable=False), - sa.Column('priority', sa.Integer, nullable=False, - server_default=sa.DefaultClause("0")), - sa.Column('complete', sa.Integer, - server_default=sa.DefaultClause("0")), - sa.Column('results', sa.SmallInteger), - sa.Column('submitted_at', sa.Integer, nullable=False), - sa.Column('complete_at', sa.Integer), - ) + buildrequests = sautils.Table( + 'buildrequests', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('buildsetid', sa.Integer, nullable=False), + sa.Column('buildername', sa.String(length=256), + nullable=False), + sa.Column('priority', sa.Integer, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('complete', sa.Integer, + server_default=sa.DefaultClause("0")), + sa.Column('results', sa.SmallInteger), + sa.Column('submitted_at', sa.Integer, nullable=False), + sa.Column('complete_at', sa.Integer), + ) buildrequests.create() conn.execute(buildrequests.insert(), [ diff --git a/master/buildbot/test/unit/test_db_migrate_versions_034_log_slug.py b/master/buildbot/test/unit/test_db_migrate_versions_034_log_slug.py index f08bd20f1fa..6b39413ae3e 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_034_log_slug.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_034_log_slug.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -32,7 +33,7 @@ def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - steps = sa.Table( + steps = sautils.Table( 'steps', metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('number', sa.Integer, nullable=False), @@ -47,7 +48,7 @@ def setup_thd(conn): ) steps.create() - logs = sa.Table( + logs = sautils.Table( 'logs', metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('name', sa.String(50), nullable=False), @@ -59,7 +60,7 @@ def setup_thd(conn): ) logs.create() - logchunks = sa.Table( + logchunks = sautils.Table( 'logchunks', metadata, sa.Column('logid', sa.Integer, sa.ForeignKey('logs.id')), # 0-based line number range in this chunk (inclusive); note that for diff --git a/master/buildbot/test/unit/test_db_migrate_versions_036_build_parent.py b/master/buildbot/test/unit/test_db_migrate_versions_036_build_parent.py index 39a1dc0b6dd..30c192fcbf2 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_036_build_parent.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_036_build_parent.py @@ -19,6 +19,7 @@ from buildbot.test.util import migration from buildbot.util import datetime2epoch +from buildbot.util import sautils from twisted.trial import unittest @@ -35,45 +36,41 @@ def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn # This table contains basic information about each build. - builds = sa.Table('builds', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('number', sa.Integer, nullable=False), - sa.Column('builderid', sa.Integer), - # note that there is 1:N relationship here. - # In case of slave loss, build has results RETRY - # and buildrequest is unclaimed - sa.Column('buildrequestid', sa.Integer, - nullable=False), - # slave which performed this build - # TODO: ForeignKey to buildslaves table, named buildslaveid - # TODO: keep nullable to support slave-free - # builds - sa.Column('buildslaveid', sa.Integer), - # master which controlled this build - sa.Column('masterid', sa.Integer, - nullable=False), - # start/complete times - sa.Column( - 'started_at', sa.Integer, nullable=False), - sa.Column('complete_at', sa.Integer), - # a list of strings describing the build's state - sa.Column( - 'state_strings_json', sa.Text, nullable=False), - sa.Column('results', sa.Integer), - ) + builds = sautils.Table( + 'builds', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('number', sa.Integer, nullable=False), + sa.Column('builderid', sa.Integer), + # note that there is 1:N relationship here. + # In case of slave loss, build has results RETRY + # and buildrequest is unclaimed + sa.Column('buildrequestid', sa.Integer, nullable=False), + # slave which performed this build + # TODO: ForeignKey to buildslaves table, named buildslaveid + # TODO: keep nullable to support slave-free + # builds + sa.Column('buildslaveid', sa.Integer), + # master which controlled this build + sa.Column('masterid', sa.Integer, nullable=False), + # start/complete times + sa.Column('started_at', sa.Integer, nullable=False), + sa.Column('complete_at', sa.Integer), + # a list of strings describing the build's state + sa.Column('state_strings_json', sa.Text, nullable=False), + sa.Column('results', sa.Integer), + ) builds.create() - buildsets = sa.Table('buildsets', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column( - 'external_idstring', sa.String(256)), - sa.Column('reason', sa.String(256)), - sa.Column( - 'submitted_at', sa.Integer, nullable=False), - sa.Column('complete', sa.SmallInteger, nullable=False, - server_default=sa.DefaultClause("0")), - sa.Column('complete_at', sa.Integer), - sa.Column('results', sa.SmallInteger), - ) + buildsets = sautils.Table( + 'buildsets', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('external_idstring', sa.String(256)), + sa.Column('reason', sa.String(256)), + sa.Column('submitted_at', sa.Integer, nullable=False), + sa.Column('complete', sa.SmallInteger, nullable=False, + server_default=sa.DefaultClause("0")), + sa.Column('complete_at', sa.Integer), + sa.Column('results', sa.SmallInteger), + ) buildsets.create() diff --git a/master/buildbot/test/unit/test_db_migrate_versions_037_buildrequests_builderid.py b/master/buildbot/test/unit/test_db_migrate_versions_037_buildrequests_builderid.py index d4daea60a29..a315645c4ff 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_037_buildrequests_builderid.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_037_buildrequests_builderid.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -32,15 +33,15 @@ def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - builders = sa.Table( + builders = sautils.Table( 'builders', metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('name', sa.Text, nullable=False), sa.Column('name_hash', sa.String(40), nullable=False), - ) + ) builders.create() - buildsets = sa.Table( + buildsets = sautils.Table( 'buildsets', metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('external_idstring', sa.String(256)), @@ -52,10 +53,10 @@ def setup_thd(conn): sa.Column('results', sa.SmallInteger), sa.Column('parent_buildid', sa.Integer), sa.Column('parent_relationship', sa.Text), - ) + ) buildsets.create() - buildrequests = sa.Table( + buildrequests = sautils.Table( 'buildrequests', metadata, sa.Column('id', sa.Integer, primary_key=True), sa.Column('buildsetid', sa.Integer, @@ -71,7 +72,7 @@ def setup_thd(conn): sa.Column('complete_at', sa.Integer), sa.Column('waited_for', sa.SmallInteger, server_default=sa.DefaultClause("0")), - ) + ) buildrequests.create() idx = sa.Index('buildrequests_buildsetid', @@ -115,7 +116,7 @@ def verify_thd(conn): 'buildrequests_builderid', 'buildrequests_buildsetid', 'buildrequests_complete', - ]) + ]) # get the new builderid bldr2_id = conn.execute( diff --git a/master/buildbot/test/unit/test_db_migrate_versions_038_state_string.py b/master/buildbot/test/unit/test_db_migrate_versions_038_state_string.py index e560f62079e..a46b3917b21 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_038_state_string.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_038_state_string.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -31,31 +32,33 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - builds = sa.Table('builds', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('number', sa.Integer, nullable=False), - sa.Column('builderid', sa.Integer), - sa.Column('buildrequestid', sa.Integer, nullable=False), - sa.Column('buildslaveid', sa.Integer), - sa.Column('masterid', sa.Integer, nullable=False), - sa.Column('started_at', sa.Integer, nullable=False), - sa.Column('complete_at', sa.Integer), - sa.Column('state_strings_json', sa.Text, nullable=False), - sa.Column('results', sa.Integer), - ) + builds = sautils.Table( + 'builds', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('number', sa.Integer, nullable=False), + sa.Column('builderid', sa.Integer), + sa.Column('buildrequestid', sa.Integer, nullable=False), + sa.Column('buildslaveid', sa.Integer), + sa.Column('masterid', sa.Integer, nullable=False), + sa.Column('started_at', sa.Integer, nullable=False), + sa.Column('complete_at', sa.Integer), + sa.Column('state_strings_json', sa.Text, nullable=False), + sa.Column('results', sa.Integer), + ) builds.create() - steps = sa.Table('steps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('number', sa.Integer, nullable=False), - sa.Column('name', sa.String(50), nullable=False), - sa.Column('buildid', sa.Integer, sa.ForeignKey('builds.id')), - sa.Column('started_at', sa.Integer), - sa.Column('complete_at', sa.Integer), - sa.Column('state_strings_json', sa.Text, nullable=False), - sa.Column('results', sa.Integer), - sa.Column('urls_json', sa.Text, nullable=False), - ) + steps = sautils.Table( + 'steps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('number', sa.Integer, nullable=False), + sa.Column('name', sa.String(50), nullable=False), + sa.Column('buildid', sa.Integer, sa.ForeignKey('builds.id')), + sa.Column('started_at', sa.Integer), + sa.Column('complete_at', sa.Integer), + sa.Column('state_strings_json', sa.Text, nullable=False), + sa.Column('results', sa.Integer), + sa.Column('urls_json', sa.Text, nullable=False), + ) steps.create() # tests diff --git a/master/buildbot/test/unit/test_db_migrate_versions_039_add_builder_description.py b/master/buildbot/test/unit/test_db_migrate_versions_039_add_builder_description.py index 9254a395204..b890006db9e 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_039_add_builder_description.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_039_add_builder_description.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -31,12 +32,14 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - builders = sa.Table('builders', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # builder's name - sa.Column('name', sa.Text, nullable=False), - # sha1 of name; used for a unique index - sa.Column('name_hash', sa.String(40), nullable=False),) + builders = sautils.Table( + 'builders', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # builder's name + sa.Column('name', sa.Text, nullable=False), + # sha1 of name; used for a unique index + sa.Column('name_hash', sa.String(40), nullable=False), + ) builders.create() conn.execute(builders.insert(), [ diff --git a/master/buildbot/test/unit/test_db_migrate_versions_040_add_builder_tags.py b/master/buildbot/test/unit/test_db_migrate_versions_040_add_builder_tags.py index c23883cade9..4978cac74e7 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_040_add_builder_tags.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_040_add_builder_tags.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -31,13 +32,15 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - builders = sa.Table('builders', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # builder's name - sa.Column('name', sa.Text, nullable=False), - sa.Column('description', sa.Text, nullable=True), - # sha1 of name; used for a unique index - sa.Column('name_hash', sa.String(40), nullable=False),) + builders = sautils.Table( + 'builders', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # builder's name + sa.Column('name', sa.Text, nullable=False), + sa.Column('description', sa.Text, nullable=True), + # sha1 of name; used for a unique index + sa.Column('name_hash', sa.String(40), nullable=False), + ) builders.create() conn.execute(builders.insert(), [ diff --git a/master/buildbot/test/unit/test_db_migrate_versions_041_add_N_N_tagsbuilders.py b/master/buildbot/test/unit/test_db_migrate_versions_041_add_N_N_tagsbuilders.py index 5868f053647..622067db0fa 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_041_add_N_N_tagsbuilders.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_041_add_N_N_tagsbuilders.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -31,14 +32,16 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - builders = sa.Table('builders', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # builder's name - sa.Column('name', sa.Text, nullable=False), - sa.Column('tags', sa.Text), - sa.Column('description', sa.Text, nullable=True), - # sha1 of name; used for a unique index - sa.Column('name_hash', sa.String(40), nullable=False),) + builders = sautils.Table( + 'builders', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # builder's name + sa.Column('name', sa.Text, nullable=False), + sa.Column('tags', sa.Text), + sa.Column('description', sa.Text, nullable=True), + # sha1 of name; used for a unique index + sa.Column('name_hash', sa.String(40), nullable=False), + ) builders.create() conn.execute(builders.insert(), [ diff --git a/master/buildbot/test/unit/test_db_migrate_versions_042_add_build_properties_table.py b/master/buildbot/test/unit/test_db_migrate_versions_042_add_build_properties_table.py index bb5f5c53f2e..5af2c2b49fc 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_042_add_build_properties_table.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_042_add_build_properties_table.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -32,10 +33,11 @@ def setup_thd(conn): metadata = sa.MetaData() metadata.bind = conn - sa.Table('builds', metadata, - sa.Column('id', sa.Integer, primary_key=True), - # .. - ).create() + sautils.Table( + 'builds', metadata, + sa.Column('id', sa.Integer, primary_key=True), + # .. + ).create() def verify_thd(conn): metadata = sa.MetaData() diff --git a/master/buildbot/test/unit/test_db_migrate_versions_043_add_changes_parent.py b/master/buildbot/test/unit/test_db_migrate_versions_043_add_changes_parent.py index e9c6a73a1aa..0f47e8a9bd1 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_043_add_changes_parent.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_043_add_changes_parent.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -31,46 +32,51 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - patches = sa.Table('patches', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('patchlevel', sa.Integer, nullable=False), - sa.Column('patch_base64', sa.Text, nullable=False), - sa.Column('patch_author', sa.Text, nullable=False), - sa.Column('patch_comment', sa.Text, nullable=False), - sa.Column('subdir', sa.Text), - ) + patches = sautils.Table( + 'patches', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('patchlevel', sa.Integer, nullable=False), + sa.Column('patch_base64', sa.Text, nullable=False), + sa.Column('patch_author', sa.Text, nullable=False), + sa.Column('patch_comment', sa.Text, nullable=False), + sa.Column('subdir', sa.Text), + ) - sourcestamps = sa.Table('sourcestamps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('ss_hash', sa.String(40), nullable=False), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), - sa.Column('repository', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('codebase', sa.String(256), nullable=False, - server_default=sa.DefaultClause("")), - sa.Column('project', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('created_at', sa.Integer, nullable=False)) + sourcestamps = sautils.Table( + 'sourcestamps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('ss_hash', sa.String(40), nullable=False), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')), + sa.Column('repository', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), + sa.Column('project', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('created_at', sa.Integer, nullable=False), + ) - changes = sa.Table('changes', metadata, - sa.Column('changeid', sa.Integer, primary_key=True), - sa.Column('author', sa.String(256), nullable=False), - sa.Column('comments', sa.Text, nullable=False), - sa.Column('branch', sa.String(256)), - sa.Column('revision', sa.String(256)), - sa.Column('revlink', sa.String(256)), - sa.Column('when_timestamp', sa.Integer, nullable=False), - sa.Column('category', sa.String(256)), - sa.Column('repository', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('codebase', sa.String(256), nullable=False, - server_default=sa.DefaultClause("")), - sa.Column('project', sa.String(length=512), nullable=False, - server_default=''), - sa.Column('sourcestampid', sa.Integer, - sa.ForeignKey('sourcestamps.id'))) + changes = sautils.Table( + 'changes', metadata, + sa.Column('changeid', sa.Integer, primary_key=True), + sa.Column('author', sa.String(256), nullable=False), + sa.Column('comments', sa.Text, nullable=False), + sa.Column('branch', sa.String(256)), + sa.Column('revision', sa.String(256)), + sa.Column('revlink', sa.String(256)), + sa.Column('when_timestamp', sa.Integer, nullable=False), + sa.Column('category', sa.String(256)), + sa.Column('repository', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('codebase', sa.String(256), nullable=False, + server_default=sa.DefaultClause("")), + sa.Column('project', sa.String(length=512), nullable=False, + server_default=''), + sa.Column('sourcestampid', sa.Integer, + sa.ForeignKey('sourcestamps.id')), + ) patches.create() sourcestamps.create() changes.create() diff --git a/master/buildbot/test/unit/test_db_migrate_versions_044_add_step_hidden.py b/master/buildbot/test/unit/test_db_migrate_versions_044_add_step_hidden.py index ac0baaca18a..8289d1861be 100644 --- a/master/buildbot/test/unit/test_db_migrate_versions_044_add_step_hidden.py +++ b/master/buildbot/test/unit/test_db_migrate_versions_044_add_step_hidden.py @@ -16,6 +16,7 @@ import sqlalchemy as sa from buildbot.test.util import migration +from buildbot.util import sautils from twisted.trial import unittest @@ -31,16 +32,18 @@ def create_tables_thd(self, conn): metadata = sa.MetaData() metadata.bind = conn - steps = sa.Table('steps', metadata, - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('number', sa.Integer, nullable=False), - sa.Column('name', sa.String(50), nullable=False), - sa.Column('buildid', sa.Integer), - sa.Column('started_at', sa.Integer), - sa.Column('complete_at', sa.Integer), - sa.Column('state_string', sa.Text, nullable=False, server_default=''), - sa.Column('results', sa.Integer), - sa.Column('urls_json', sa.Text, nullable=False)) + steps = sautils.Table( + 'steps', metadata, + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('number', sa.Integer, nullable=False), + sa.Column('name', sa.String(50), nullable=False), + sa.Column('buildid', sa.Integer), + sa.Column('started_at', sa.Integer), + sa.Column('complete_at', sa.Integer), + sa.Column('state_string', sa.Text, nullable=False, server_default=''), + sa.Column('results', sa.Integer), + sa.Column('urls_json', sa.Text, nullable=False), + ) steps.create() conn.execute(steps.insert(), [ diff --git a/master/buildbot/test/util/migration.py b/master/buildbot/test/util/migration.py index e36d866a934..f44ed5960d5 100644 --- a/master/buildbot/test/util/migration.py +++ b/master/buildbot/test/util/migration.py @@ -23,6 +23,7 @@ from buildbot.test.util import db from buildbot.test.util import dirs from buildbot.test.util import querylog +from buildbot.util import sautils from twisted.internet import defer from twisted.python import log @@ -59,11 +60,12 @@ def do_test_migration(self, base_version, target_version, def setup_thd(conn): metadata = sa.MetaData() - table = sa.Table('migrate_version', metadata, - sa.Column('repository_id', sa.String(250), - primary_key=True), - sa.Column('repository_path', sa.Text), - sa.Column('version', sa.Integer)) + table = sautils.Table( + 'migrate_version', metadata, + sa.Column('repository_id', sa.String(250), primary_key=True), + sa.Column('repository_path', sa.Text), + sa.Column('version', sa.Integer), + ) table.create(bind=conn) conn.execute(table.insert(), repository_id='Buildbot', diff --git a/master/buildbot/util/sautils.py b/master/buildbot/util/sautils.py index 02e9d9b4008..bdf2968ab58 100644 --- a/master/buildbot/util/sautils.py +++ b/master/buildbot/util/sautils.py @@ -50,3 +50,11 @@ def tryint(s): return -1 return tuple(map(tryint, sa.__version__.split('.'))) return (0, 0, 0) # "it's old" + + +def Table(*args, **kwargs): + """Wrap table creation to add any necessary dialect-specific options""" + # work aorund the case where a database was created for us with + # a non-utf8 character set (mysql's default) + kwargs['mysql_character_set'] = 'utf8' + return sa.Table(*args, **kwargs)