Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
bug 945179: Match on a list of locales, rather than exact only. r=nth…
…omas
  • Loading branch information
bhearsum committed Mar 16, 2015
1 parent 88617f7 commit d15f6cd
Show file tree
Hide file tree
Showing 6 changed files with 104 additions and 13 deletions.
4 changes: 2 additions & 2 deletions auslib/admin/views/forms.py
Expand Up @@ -100,7 +100,7 @@ class RuleForm(Form):
version = NullableStringField('Version', validators=[Length(0,10) ])
build_id = NullableStringField('BuildID', validators=[Length(0,20) ])
channel = NullableStringField('Channel', validators=[Length(0,75) ])
locale = NullableStringField('Locale', validators=[Length(0,10) ])
locale = NullableStringField('Locale', validators=[Length(0,200) ])
distribution = NullableStringField('Distribution', validators=[Length(0,100) ])
build_target = NullableStringField('Build Target', validators=[Length(0,75) ])
os_version = NullableStringField('OS Version', validators=[Length(0,1000) ])
Expand All @@ -117,7 +117,7 @@ class EditRuleForm(DbEditableForm):
version = NullableStringField('Version', validators=[Optional(), Length(0,10) ])
build_id = NullableStringField('BuildID', validators=[Optional(), Length(0,20) ])
channel = NullableStringField('Channel', validators=[Optional(), Length(0,75) ])
locale = NullableStringField('Locale', validators=[Optional(), Length(0,10) ])
locale = NullableStringField('Locale', validators=[Optional(), Length(0,200) ])
distribution = NullableStringField('Distribution', validators=[Optional(), Length(0,100) ])
build_target = NullableStringField('Build Target', validators=[Optional(), Length(0,75) ])
os_version = NullableStringField('OS Version', validators=[Optional(), Length(0,1000) ])
Expand Down
22 changes: 19 additions & 3 deletions auslib/blobs/apprelease.py
Expand Up @@ -222,6 +222,14 @@ def _getPatchesXML(self, localeData, updateQuery, whitelistedDomains, specialFor


class ReleaseBlobV1(ReleaseBlobBase, SingleUpdateXMLMixin, SeparatedFileUrlsMixin):
""" This is the legacy format for apps based on Gecko 1.8.0 to 1.9.2, which
translates to
* Firefox 1.5 to 3.6.x
* Thunderbird 1.5 to 3.1.y
It was deprecated by https://bugzilla.mozilla.org/show_bug.cgi?id=530872 during
Gecko 2.0 development (aka 1.9.3).
"""
format_ = {
'name': None,
'schema_version': None,
Expand Down Expand Up @@ -413,7 +421,11 @@ def _getUpdateLineXML(self, buildTarget, locale, update_type):


class ReleaseBlobV2(ReleaseBlobBase, NewStyleVersionsMixin, SingleUpdateXMLMixin, SeparatedFileUrlsMixin):
""" Changes from ReleaseBlobV1:
""" Client-side changes in
https://bugzilla.mozilla.org/show_bug.cgi?id=530872
were introduced at Gecko 1.9.3a3, requiring this new blob class.
Changed parameters from ReleaseBlobV1:
* appv, extv become appVersion, platformVersion, displayVersion
Added:
* actions, billboardURL, openURL, notificationURL,
Expand Down Expand Up @@ -555,7 +567,9 @@ def _getPatchesXML(self, localeData, updateQuery, whitelistedDomains, specialFor


class ReleaseBlobV3(ReleaseBlobBase, NewStyleVersionsMixin, MultipleUpdatesXMLMixin, SeparatedFileUrlsMixin):
""" Changes from ReleaseBlobV2:
""" This is an internal change to add functionality to Balrog.
Changes from ReleaseBlobV2:
* support multiple partials
* remove "partial" and "complete" from locale level
* add "partials" and "completes" to locale level, ftpFilenames, and bouncerProducts
Expand Down Expand Up @@ -705,7 +719,9 @@ def _getUrl(self, updateQuery, patchKey, patch, specialForceHosts):


class ReleaseBlobV4(ReleaseBlobBase, NewStyleVersionsMixin, MultipleUpdatesXMLMixin, UnifiedFileUrlsMixin):
""" Changes from ReleaseBlobV4:
""" This is an internal change to add functionality to Balrog.
Changes from ReleaseBlobV3:
* Support pushing release builds to the beta channel with bouncer support (bug 1021026)
** Combine fileUrls, bouncerProducts, and ftpFilenames into a larger data structure,
still called "fileUrls". (See below for a more detailed description.)
Expand Down
46 changes: 40 additions & 6 deletions auslib/db.py
Expand Up @@ -632,7 +632,7 @@ def __init__(self, metadata, dialect):
Column('channel', String(75)),
Column('buildTarget', String(75)),
Column('buildID', String(20)),
Column('locale', String(10)),
Column('locale', String(200)),
Column('osVersion', String(1000)),
Column('distribution', String(100)),
Column('distVersion', String(100)),
Expand Down Expand Up @@ -663,6 +663,17 @@ def _channelMatchesRule(self, ruleChannel, queryChannel, fallbackChannel):
if self._matchesRegex(ruleChannel, fallbackChannel):
return True

def _matchesList(self, ruleString, queryString):
"""Decides whether a ruleString from a rule matches an incoming string.
The rule may specify multiple matches, delimited by a comma. Once
split we look for an exact match against the string from the queries.
We want an exact match so (eg) we only get the locales we specify"""
if ruleString is None:
return True
for subString in ruleString.split(','):
if subString == queryString:
return True

def _versionMatchesRule(self, ruleVersion, queryVersion):
"""Decides whether a version from the rules matches an incoming version.
If the ruleVersion is null, we match any queryVersion. If it's not
Expand Down Expand Up @@ -694,6 +705,11 @@ def _osVersionMatchesRule(self, ruleOsVersion, queryOsVersion):
if os in queryOsVersion:
return True

def _localeMatchesRule(self, ruleLocales, queryLocale):
"""Decides if a comma seperated list of locales in a rule matches an
update request"""
return self._matchesList(ruleLocales, queryLocale)

def addRule(self, changed_by, what, transaction=None):
ret = self.insert(changed_by=changed_by, transaction=transaction, **what)
return ret.inserted_primary_key[0]
Expand All @@ -714,7 +730,6 @@ def getRulesMatchingQuery(self, updateQuery, fallbackChannel, transaction=None):
where=[
((self.product==updateQuery['product']) | (self.product==None)) &
((self.buildTarget==updateQuery['buildTarget']) | (self.buildTarget==None)) &
((self.locale==updateQuery['locale']) | (self.locale==None)) &
((self.headerArchitecture==updateQuery['headerArchitecture']) | (self.headerArchitecture==None))
]
# Query version 2 doesn't have distribution information, and to keep
Expand Down Expand Up @@ -755,6 +770,10 @@ def getRulesMatchingQuery(self, updateQuery, fallbackChannel, transaction=None):
if not self._osVersionMatchesRule(rule['osVersion'], updateQuery['osVersion']):
self.log.debug("%s doesn't match %s", rule['osVersion'], updateQuery['osVersion'])
continue
# Locales may be a comma delimited rule too, exact matches only
if not self._localeMatchesRule(rule['locale'], updateQuery['locale']):
self.log.debug("%s doesn't match %s", rule['locale'], updateQuery['locale'])
continue
matchingRules.append(rule)
self.log.debug("Reduced matches:")
if self.log.isEnabledFor(logging.DEBUG):
Expand Down Expand Up @@ -1180,25 +1199,40 @@ def onUpdate(table, who, where, what):
cef_event('Human modification', CEF_ALERT, user=who, what=what, where=where, table=table.name, type='update')
return onInsert, onDelete, onUpdate


# A helper that sets sql_mode. This should only be used with MySQL, and
# lets us put the database in a stricter mode that will disallow things like
# automatic data truncation.
# From http://www.enricozini.org/2012/tips/sa-sqlmode-traditional/
from sqlalchemy.interfaces import PoolListener
class SetSqlMode(PoolListener):
def connect(self, dbapi_con, connection_record):
cur = dbapi_con.cursor()
cur.execute("SET SESSION sql_mode='TRADITIONAL'")


class AUSDatabase(object):
engine = None
migrate_repo = path.join(path.dirname(__file__), "migrate")

def __init__(self, dburi=None):
def __init__(self, dburi=None, mysql_traditional_mode=False):
"""Create a new AUSDatabase. Before this object is useful, dburi must be
set, either through the constructor or setDburi()"""
if dburi:
self.setDburi(dburi)
self.setDburi(dburi, mysql_traditional_mode)
self.log = logging.getLogger(self.__class__.__name__)

def setDburi(self, dburi):
def setDburi(self, dburi, mysql_traditional_mode=False):
"""Setup the database connection. Note that SQLAlchemy only opens a connection
to the database when it needs to, however."""
if self.engine:
raise AlreadySetupError()
self.dburi = dburi
self.metadata = MetaData()
self.engine = create_engine(self.dburi, pool_recycle=60)
listeners = []
if mysql_traditional_mode and "mysql" in dburi:
listeners.append(SetSqlMode())
self.engine = create_engine(self.dburi, pool_recycle=60, listeners=listeners)
dialect = self.engine.name
self.rulesTable = Rules(self.metadata, dialect)
self.releasesTable = Releases(self.metadata, dialect)
Expand Down
11 changes: 11 additions & 0 deletions auslib/migrate/versions/007_longer_locales.py
@@ -0,0 +1,11 @@
from sqlalchemy import String, MetaData, Table

def upgrade(migrate_engine):
metadata = MetaData(bind=migrate_engine)
Table('rules', metadata, autoload=True).c.locale.alter(type=String(200))
Table('rules_history', metadata, autoload=True).c.locale.alter(type=String(200))

def downgrade(migrate_engine):
metadata = MetaData(bind=migrate_engine)
Table('rules', metadata, autoload=True).c.locale.alter(type=String(10))
Table('rules_history', metadata, autoload=True).c.locale.alter(type=String(10))
32 changes: 31 additions & 1 deletion auslib/test/test_db.py
Expand Up @@ -542,6 +542,7 @@ def setUp(self):
self.paths.t.insert().execute(id=5, priority=80, buildTarget='d', version='3.3', backgroundRate=0, mapping='c', update_type='z', data_version=1)
self.paths.t.insert().execute(id=6, priority=100, buildTarget='d', mapping='a', backgroundRate=100, osVersion='foo 1', update_type='z', data_version=1)
self.paths.t.insert().execute(id=7, priority=100, buildTarget='d', mapping='a', backgroundRate=100, osVersion='foo 2,blah 6', update_type='z', data_version=1)
self.paths.t.insert().execute(id=8, priority=100, buildTarget='e', mapping='d', backgroundRate=100, locale='foo,bar-baz', update_type='z', data_version=1)

def testGetOrderedRules(self):
rules = self._stripNullColumns(self.paths.getOrderedRules())
Expand All @@ -550,6 +551,7 @@ def testGetOrderedRules(self):
dict(rule_id=5, priority=80, backgroundRate=0, version='3.3', buildTarget='d', mapping='c', update_type='z', data_version=1),
dict(rule_id=6, priority=100, buildTarget='d', mapping='a', backgroundRate=100, osVersion='foo 1', update_type='z', data_version=1),
dict(rule_id=7, priority=100, buildTarget='d', mapping='a', backgroundRate=100, osVersion='foo 2,blah 6', update_type='z', data_version=1),
dict(rule_id=8, priority=100, buildTarget='e', mapping='d', backgroundRate=100, locale='foo,bar-baz', update_type='z', data_version=1),
dict(rule_id=2, priority=100, backgroundRate=100, version='3.3', buildTarget='d', mapping='b', update_type='z', data_version=1),
dict(rule_id=3, priority=100, backgroundRate=100, version='3.5', buildTarget='a', mapping='a', update_type='z', data_version=1),
dict(rule_id=1, priority=100, backgroundRate=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1),
Expand Down Expand Up @@ -666,6 +668,34 @@ def testGetRulesMatchingQueryOsVersionMultipleSubstring(self):
]
self.assertEquals(rules, expected)

def testGetRulesMatchingQueryLocale(self):
rules = self.paths.getRulesMatchingQuery(
dict(product='', version='', channel='', buildTarget='e',
buildID='', locale='foo', osVersion='', distribution='',
distVersion='', headerArchitecture='', force=False,
queryVersion=3,
),
fallbackChannel='',
)
rules = self._stripNullColumns(rules)
expected = [
dict(rule_id=8, priority=100, buildTarget='e', mapping='d', backgroundRate=100, locale='foo,bar-baz', update_type='z', data_version=1)
]
self.assertEquals(rules, expected)

def testGetRulesMatchingQueryLocaleNoPartialMatch(self):
rules = self.paths.getRulesMatchingQuery(
dict(product='', version='5', channel='', buildTarget='e',
buildID='', locale='bar', osVersion='', distribution='',
distVersion='', headerArchitecture='', force=False,
queryVersion=3,
),
fallbackChannel='',
)
rules = self._stripNullColumns(rules)
expected = []
self.assertEquals(rules, expected)

def testGetRuleById(self):
rule = self._stripNullColumns([self.paths.getRuleById(1)])
expected = [dict(rule_id=1, priority=100, backgroundRate=100, version='3.5', buildTarget='d', mapping='c', update_type='z', data_version=1)]
Expand Down Expand Up @@ -705,7 +735,7 @@ def testDeleteRule(self):
self.assertEquals(rule, [])

def testGetNumberOfRules(self):
self.assertEquals(self.paths.countRules(), 7)
self.assertEquals(self.paths.countRules(), 8)


class TestRulesSpecial(unittest.TestCase, RulesTestMixin, MemoryDatabaseMixin):
Expand Down
2 changes: 1 addition & 1 deletion scripts/manage-db.py
Expand Up @@ -32,7 +32,7 @@

action = args[0]

db = AUSDatabase(options.db)
db = AUSDatabase(options.db, mysql_traditional_mode=True)
if action == 'create':
db.create(options.version)
elif action == 'upgrade':
Expand Down

0 comments on commit d15f6cd

Please sign in to comment.