Skip to content

Commit

Permalink
bug 1310187: Make Scheduled Changes conditions configurable (#151). r…
Browse files Browse the repository at this point in the history 
…=jlorenzo,nthomas
  • Loading branch information
@bhearsum
bhearsum committed Nov 22, 2016
1 parent 6ad44d5 commit 3c7256b
Show file tree
Hide file tree
Showing 2 changed files with 192 additions and 25 deletions.
66 changes: 41 additions & 25 deletions auslib/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ class AUSTable(object):
"""

def __init__(self, db, dialect, history=True, versioned=True, scheduled_changes=False,
onInsert=None, onUpdate=None, onDelete=None):
scheduled_changes_kwargs={}, onInsert=None, onUpdate=None, onDelete=None):
self.db = db
self.t = self.table
# Enable versioning, if required
Expand All @@ -213,7 +213,7 @@ def __init__(self, db, dialect, history=True, versioned=True, scheduled_changes=
self.history = None
# Set-up a scheduled changes table if required
if scheduled_changes:
self.scheduled_changes = ScheduledChangeTable(db, dialect, self.t.metadata, self)
self.scheduled_changes = ScheduledChangeTable(db, dialect, self.t.metadata, self, **scheduled_changes_kwargs)
else:
self.scheduled_changes = None
self.log = logging.getLogger(self.__class__.__name__)
Expand Down Expand Up @@ -760,40 +760,56 @@ class ConditionsTable(AUSTable):
# conditions require mulitple arguments. This data structure defines
# each type of condition, and groups their args together for easier
# processing.
condition_groups = (
("when",),
("telemetry_product", "telemetry_channel", "telemetry_uptake"),
)
condition_groups = {
"time": ("when",),
"uptake": ("telemetry_product", "telemetry_channel", "telemetry_uptake"),
}

def __init__(self, db, dialect, metadata, baseName, conditions, history=True):
if not conditions:
raise ValueError("No conditions enabled, cannot initialize conditions for for {}".format(baseName))
if set(conditions) - set(self.condition_groups):
raise ValueError("Unknown conditions in: {}".format(conditions))

self.enabled_condition_groups = {k: v for k, v in self.condition_groups.iteritems() if k in conditions}

def __init__(self, db, dialect, metadata, baseName, history=True):
self.table = Table("{}_conditions".format(baseName), metadata,
Column("sc_id", Integer, primary_key=True),
Column("telemetry_product", String(15)),
Column("telemetry_channel", String(75)),
Column("telemetry_uptake", Integer),
)
if dialect == "sqlite":
self.table.append_column(Column("when", Integer))
else:
self.table.append_column(Column("when", BigInteger))

if "uptake" in conditions:
self.table.append_column(Column("telemetry_product", String(15)))
self.table.append_column(Column("telemetry_channel", String(75)))
self.table.append_column(Column("telemetry_uptake", Integer))

if "time" in conditions:
if dialect == "sqlite":
self.table.append_column(Column("when", Integer))
else:
self.table.append_column(Column("when", BigInteger))

super(ConditionsTable, self).__init__(db, dialect, history=history, versioned=True)

def validate(self, conditions):
# Filter out conditions whose values are none before processing.
conditions = {k: v for k, v in conditions.iteritems() if conditions[k]}
if not conditions:
raise ValueError("No conditions found")

for c in conditions:
if c not in itertools.chain(*self.condition_groups):
for condition, args in self.condition_groups.iteritems():
if c in args:
if c in itertools.chain(*self.enabled_condition_groups.values()):
break
else:
raise ValueError("{} condition is disabled".format(condition))
else:
raise ValueError("Invalid condition: %s", c)

for group in self.condition_groups:
for group in self.enabled_condition_groups.values():
if set(group) == set(conditions.keys()):
break
else:
raise ValueError("Invalid combination of conditions: %s", conditions.keys())
raise ValueError("Invalid combination of conditions: {}".format(conditions.keys()))

if "when" in conditions:
try:
Expand All @@ -811,15 +827,15 @@ class ScheduledChangeTable(AUSTable):
columns of its base, and adding the necessary ones to provide the schedule.
By default, ScheduledChangeTables enable History on themselves."""

def __init__(self, db, dialect, metadata, baseTable, history=True):
def __init__(self, db, dialect, metadata, baseTable, conditions=("time", "uptake"), history=True):
table_name = "{}_scheduled_changes".format(baseTable.t.name)
self.baseTable = baseTable
self.table = Table(table_name, metadata,
Column("sc_id", Integer, primary_key=True, autoincrement=True),
Column("scheduled_by", String(100), nullable=False),
Column("complete", Boolean, default=False),
)
self.conditions = ConditionsTable(db, dialect, metadata, table_name)
self.conditions = ConditionsTable(db, dialect, metadata, table_name, conditions)

# The primary key column(s) are used in construct "where" clauses for
# existing rows.
Expand Down Expand Up @@ -870,11 +886,11 @@ def _splitColumns(self, columns):
are stored in the conditions table in a few different places."""
base_columns = {}
condition_columns = {}
for k in columns:
if k in itertools.chain(*self.conditions.condition_groups):
condition_columns[k] = columns[k]
for cond_type in columns:
if cond_type in itertools.chain(*self.conditions.condition_groups.values()):
condition_columns[cond_type] = columns[cond_type]
else:
base_columns[k] = columns[k]
base_columns[cond_type] = columns[cond_type]

return base_columns, condition_columns

Expand Down Expand Up @@ -953,7 +969,7 @@ def select(self, where=None, transaction=None, **kwargs):
if "sc_id" not in kwargs["columns"] and self.sc_id not in kwargs["columns"]:
kwargs["columns"].append(self.sc_id)
for row in super(ScheduledChangeTable, self).select(where=where, transaction=transaction, **kwargs):
columns = [getattr(self.conditions, c) for c in itertools.chain(*self.conditions.condition_groups)]
columns = [getattr(self.conditions, c) for c in itertools.chain(*self.conditions.enabled_condition_groups.values())]
conditions = self.conditions.select([self.conditions.sc_id == row["sc_id"]], transaction=transaction, columns=columns)
row.update(conditions[0])
ret.append(row)
Expand Down
151 changes: 151 additions & 0 deletions auslib/test/test_db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,8 @@ def testAllTablesCreated(self):
self.assertTrue(self.table.history)
self.assertTrue(self.table.scheduled_changes)
self.assertTrue(self.table.scheduled_changes.history)
self.assertTrue(self.table.scheduled_changes.conditions)
self.assertTrue(self.table.scheduled_changes.conditions.history)

def testTablesHaveCorrectColumns(self):
sc_columns = [c.name for c in self.sc_table.t.get_children()]
Expand Down Expand Up @@ -1118,6 +1120,155 @@ def testMergeUpdateWithConflict(self):
self.assertRaises(UpdateMergeError, self.sc_table.mergeUpdate, old_row, what, changed_by="bob")


class TestScheduledChangesWithConfigurableConditions(unittest.TestCase, MemoryDatabaseMixin):

def setUp(self):
MemoryDatabaseMixin.setUp(self)
self.db = AUSDatabase(self.dburi)
self.db.create()
self.engine = self.db.engine
self.metadata = self.db.metadata

class TestTable(AUSTable):

def __init__(self, db, metadata):
self.table = Table("test_table", metadata, Column("fooid", Integer, primary_key=True, autoincrement=True),
Column("foo", String(15), nullable=False),
Column("bar", String(15)))
super(TestTable, self).__init__(db, "sqlite", scheduled_changes=True, scheduled_changes_kwargs={"conditions": ["time"]},
history=True, versioned=True)

self.table = TestTable(self.db, self.metadata)
self.sc_table = self.table.scheduled_changes
self.metadata.create_all()
self.table.t.insert().execute(fooid=10, foo="h", data_version=1)
self.table.t.insert().execute(fooid=11, foo="i", bar="j", data_version=1)
self.sc_table.t.insert().execute(sc_id=1, scheduled_by="bob", base_fooid=10, base_foo="h", base_bar="bbb", base_data_version=1, data_version=1)
self.sc_table.conditions.t.insert().execute(sc_id=1, when=87000, data_version=1)
self.db.permissions.t.insert().execute(permission="admin", username="bob", data_version=1)

def testAllTablesCreated(self):
self.assertTrue(self.table)
self.assertTrue(self.table.history)
self.assertTrue(self.table.scheduled_changes)
self.assertTrue(self.table.scheduled_changes.history)
self.assertTrue(self.table.scheduled_changes.conditions)
self.assertTrue(self.table.scheduled_changes.conditions.history)

def testSCTableHasCorrectColumns(self):
sc_columns = [c.name for c in self.sc_table.t.get_children()]
self.assertTrue("sc_id" in sc_columns)
self.assertTrue("scheduled_by" in sc_columns)
self.assertTrue("complete" in sc_columns)
self.assertTrue("data_version" in sc_columns)
self.assertTrue("base_fooid" in sc_columns)
self.assertTrue("base_foo" in sc_columns)
self.assertTrue("base_bar" in sc_columns)
self.assertTrue("base_data_version" in sc_columns)
self.assertTrue("telemetry_product" not in sc_columns)
self.assertTrue("telemetry_channel" not in sc_columns)
self.assertTrue("telemetry_uptake" not in sc_columns)
self.assertTrue("when" not in sc_columns)

cond_columns = [c.name for c in self.sc_table.conditions.t.get_children()]
self.assertTrue("sc_id" in cond_columns)
self.assertTrue("telemetry_product" not in cond_columns)
self.assertTrue("telemetry_channel" not in cond_columns)
self.assertTrue("telemetry_uptake" not in cond_columns)
self.assertTrue("when" in cond_columns)

def testSCTableWithNoConditions(self):
class TestTable2(AUSTable):

def __init__(self, db, metadata):
self.table = Table("test_table3", metadata, Column("fooid", Integer, primary_key=True, autoincrement=True),
Column("foo", String(15), nullable=False),
Column("bar", String(15)))
super(TestTable2, self).__init__(db, "sqlite", scheduled_changes=True, scheduled_changes_kwargs={"conditions": []},
history=True, versioned=True)

self.assertRaisesRegexp(ValueError, "No conditions enabled", TestTable2, self.db, self.metadata)

def testSCTableWithBadConditions(self):
class TestTable3(AUSTable):

def __init__(self, db, metadata):
self.table = Table("test_table3", metadata, Column("fooid", Integer, primary_key=True, autoincrement=True),
Column("foo", String(15), nullable=False),
Column("bar", String(15)))
super(TestTable3, self).__init__(db, "sqlite", scheduled_changes=True, scheduled_changes_kwargs={"conditions": ["time", "blech"]},
history=True, versioned=True)

self.assertRaisesRegexp(ValueError, "Unknown conditions", TestTable3, self.db, self.metadata)

def testValidateConditionsNone(self):
self.assertRaisesRegexp(ValueError, "No conditions found", self.sc_table.conditions.validate, {})

@mock.patch("time.time", mock.MagicMock(return_value=200))
def testValidateConditionsJustWhen(self):
self.sc_table.conditions.validate({"when": 12345678})

def testValidateConditionsTelemetryRaisesError(self):
conditions = {
"telemetry_product": "Firefox",
"telemetry_channel": "nightly",
"telemetry_uptake": "200000",
}
self.assertRaisesRegexp(ValueError, "uptake condition is disabled", self.sc_table.conditions.validate, conditions)

@mock.patch("time.time", mock.MagicMock(return_value=200))
def testInsertWithEnabledCondition(self):
what = {"fooid": 11, "foo": "i", "bar": "jjj", "data_version": 1, "when": 909000}
self.sc_table.insert(changed_by="bob", **what)
row = self.sc_table.t.select().where(self.sc_table.sc_id == 2).execute().fetchall()[0]
cond_row = self.sc_table.conditions.t.select().where(self.sc_table.conditions.sc_id == 2).execute().fetchall()[0]
self.assertEquals(row.scheduled_by, "bob")
self.assertEquals(row.data_version, 1)
self.assertEquals(row.base_fooid, 11)
self.assertEquals(row.base_foo, "i")
self.assertEquals(row.base_bar, "jjj")
self.assertEquals(row.base_data_version, 1)
self.assertEquals(cond_row.when, 909000)
self.assertEquals(cond_row.data_version, 1)

def testInsertWithDisabledCondition(self):
what = {"fooid": 11, "foo": "i", "bar": "jjj", "data_version": 1, "telemetry_product": "aa",
"telemetry_channel": "bb", "telemetry_uptake": 34567}
self.assertRaisesRegexp(ValueError, "uptake condition is disabled", self.sc_table.insert, changed_by="bob", **what)

@mock.patch("time.time", mock.MagicMock(return_value=200))
def testUpdateWithNewValueForEnabledCondition(self):
where = [self.sc_table.sc_id == 1]
what = {"when": 1000300, "bar": "ccc"}
self.sc_table.update(where, what, changed_by="bob", old_data_version=1)
row = self.sc_table.t.select().where(self.sc_table.sc_id == 1).execute().fetchall()[0]
cond_row = self.sc_table.conditions.t.select().where(self.sc_table.conditions.sc_id == 1).execute().fetchall()[0]
history_row = self.sc_table.history.t.select().where(self.sc_table.history.sc_id == 1).execute().fetchall()[0]
cond_history_row = self.sc_table.conditions.history.t.select().where(self.sc_table.conditions.history.sc_id == 1).execute().fetchall()[0]
self.assertEquals(row.scheduled_by, "bob")
self.assertEquals(row.data_version, 2)
self.assertEquals(row.base_fooid, 10)
self.assertEquals(row.base_foo, "h")
self.assertEquals(row.base_bar, "ccc")
self.assertEquals(row.base_data_version, 1)
self.assertEquals(history_row.changed_by, "bob")
self.assertEquals(history_row.scheduled_by, "bob")
self.assertEquals(history_row.data_version, 2)
self.assertEquals(history_row.base_fooid, 10)
self.assertEquals(history_row.base_foo, "h")
self.assertEquals(history_row.base_bar, "ccc")
self.assertEquals(history_row.base_data_version, 1)
self.assertEquals(cond_row.when, 1000300)
self.assertEquals(cond_row.data_version, 2)
self.assertEquals(cond_history_row.changed_by, "bob")
self.assertEquals(cond_history_row.when, 1000300)
self.assertEquals(cond_history_row.data_version, 2)

def testUpdateChangeToDisabledCondition(self):
where = [self.sc_table.sc_id == 1]
what = {"telemetry_product": "pro", "telemetry_channel": "cha", "telemetry_uptake": 3456, "bar": "ccc", "when": None}
self.assertRaisesRegexp(ValueError, "uptake condition is disabled", self.sc_table.update, where, what, changed_by="bob", old_data_version=1)

# In https://bugzilla.mozilla.org/show_bug.cgi?id=1284481, we changed the sampled data to be a true
# production dump, which doesn't import properly into sqlite. We should uncomment this test in the
# future, when we have the ability to run it againts a mysql database.
Expand Down

0 comments on commit 3c7256b

Please sign in to comment.