Bug 1312499 - figure out how to match multiple substrings in one rule…

… field (#280). r=bhearsum
mozilla-releng · Apr 10, 2017 · 946ccd1 · 946ccd1
1 parent b74d1fb
commit 946ccd1
Show file tree

Hide file tree

Showing 3 changed files with 224 additions and 42 deletions.
diff --git a/auslib/db.py b/auslib/db.py
@@ -1539,18 +1539,40 @@ def _buildIDMatchesRule(self, ruleBuildID, queryBuildID):
             return True
         return string_compare(queryBuildID, ruleBuildID)
 
-    def _csvMatchesRule(self, ruleString, queryString, substring=True):
-        """Decides whether a column from a rule matches an incoming one.
-           Some columns in a rule may specify multiple values delimited by a
-           comma. Once split we do a full or substring match against the query
-           string. Because we support substring matches, there's no need
-           to support globbing as well."""
+    def _simpleBooleanMatchesSubRule(self, subRuleString, queryString, substring):
+        """Performs the actual logical 'AND' operation on a rule as well as partial/full string matching
+           for each section of a rule.
+           If all parts of the subRuleString match the queryString, then we have successfully resolved the
+           logical 'AND' operation and return True.
+           Partial matching makes use of Python's "<substring> in <string>" functionality, giving us the ability
+           for an incoming rule to match only a substring of a rule.
+           Full matching makes use of Python's "<string> in <list>" functionality, giving us the ability for
+           an incoming rule to exactly match the whole rule. Currently, incoming rules are comma-separated strings."""
+        for rule in subRuleString:
+            if substring and rule not in queryString:
+                return False
+            elif not substring and rule not in queryString.split(','):
+                return False
+        return True
+
+    def _simpleBooleanMatchesRule(self, ruleString, queryString, substring=True):
+        """Decides whether a column from a rule matches an incoming one using simplified boolean logic.
+           Only two operators are supported: '&&' (and), ',' (or). A rule like 'AMD,SSE' will match incoming
+           rules that contain either 'AMD' or 'SSE'. A rule like 'AMD&&SSE' will only match incoming rules
+           that contain both 'AMD' and 'SSE'.
+           This function can do substring matching or full string matching. When doing substring matching, a rule
+           specifying 'AMD,Windows 10' WILL match an incoming rule such as 'Windows 10.1.2'. When doing full string
+           matching, a rule specifying 'AMD,SSE' will NOT match an incoming rule that contains 'SSE3', but WILL match
+           an incoming rule that contains either 'AMD' or 'SSE3'."""
         if ruleString is None:
             return True
-        for part in ruleString.split(','):
-            if substring and part in queryString:
-                return True
-            elif part == queryString:
+
+        decomposedRules = [[rule.strip() for rule in subRule.split('&&')] for subRule in ruleString.split(',')]
+
+        for subRule in decomposedRules:
+            if self._simpleBooleanMatchesSubRule(subRule, queryString, substring):
+                # We can immediately return True on the first match because this loop is iterating over an OR expression
+                # so we need just one match to pass.
                 return True
         return False
 
@@ -1645,11 +1667,11 @@ def getRawMatches():
             # To help keep the rules table compact, multiple OS versions may be
             # specified in a single rule. They are comma delimited, so we need to
             # break them out and create clauses for each one.
-            if not self._csvMatchesRule(rule['osVersion'], updateQuery['osVersion']):
+            if not self._simpleBooleanMatchesRule(rule['osVersion'], updateQuery['osVersion']):
                 self.log.debug("%s doesn't match %s", rule['osVersion'], updateQuery['osVersion'])
                 continue
             # Same deal for system capabilities
-            if not self._csvMatchesRule(rule['systemCapabilities'], updateQuery.get('systemCapabilities', ""), substring=False):
+            if not self._simpleBooleanMatchesRule(rule['systemCapabilities'], updateQuery.get('systemCapabilities', ""), substring=False):
                 self.log.debug("%s doesn't match %s", rule['systemCapabilities'], updateQuery.get('systemCapabilities'))
                 continue
             # Locales may be a comma delimited rule too, exact matches only

diff --git a/auslib/test/test_db.py b/auslib/test/test_db.py
@@ -1892,6 +1892,16 @@ def setUp(self):
                                       product="foo", channel="foo*", data_version=1)
         self.paths.t.insert().execute(rule_id=10, priority=100, buildTarget="g", mapping="g", fallbackMapping='fallback', backgroundRate=100,
                                       update_type="z", product="foo", channel="foo", data_version=1)
+
+        self.paths.t.insert().execute(rule_id=11, priority=100, buildTarget='h', mapping='h', systemCapabilities='GenuineIntel, SSE2', update_type='z',
+                                      product='a', channel='a', data_version=1)
+        self.paths.t.insert().execute(rule_id=12, priority=100, buildTarget='i', mapping='i', systemCapabilities='GenuineIntel && SSE2', update_type='z',
+                                      product='a', channel='a', data_version=1, version='5.0')
+        self.paths.t.insert().execute(rule_id=13, priority=100, buildTarget='j', mapping='j', systemCapabilities='GenuineIntel && SSE3, SSE2', update_type='z',
+                                      product='a', channel='a', data_version=1)
+        self.paths.t.insert().execute(rule_id=14, priority=100, buildTarget='k', mapping='k', systemCapabilities='GenuineIntel, SSE', update_type='z',
+                                      product='a', channel='a', data_version=1)
+
         self.db.permissions.t.insert().execute(permission="admin", username="bill", data_version=1)
         self.db.permissions.user_roles.t.insert(username="bill", role="bar", data_version=1)
         self.db.permissions.user_roles.t.insert(username="jane", role="bar", data_version=1)
@@ -1908,27 +1918,34 @@ def testAllTablesCreated(self):
     def testGetOrderedRules(self):
         rules = self._stripNullColumns(self.paths.getOrderedRules())
         expected = [
-            dict(rule_id=4, alias="gandalf", priority=80, backgroundRate=100, buildTarget='d', mapping='a', update_type='z',
-                 channel="a", data_version=1),
-            dict(rule_id=5, priority=80, backgroundRate=0, version='3.3', buildTarget='d', mapping='c', update_type='z',
-                 data_version=1),
-            dict(rule_id=6, alias="radagast", priority=100, buildTarget='d', mapping='a', backgroundRate=100, osVersion='foo 1', update_type='z',
-                 product="a", channel="a", data_version=1),
-            dict(rule_id=7, priority=100, buildTarget='d', mapping='a', backgroundRate=100, osVersion='foo 2,blah 6', update_type='z',
-                 product="a", channel="a", data_version=1),
-            dict(rule_id=8, priority=100, buildTarget='e', mapping='d', backgroundRate=100, locale='foo,bar-baz', update_type='z',
-                 product="a", channel="a", data_version=1),
-            dict(rule_id=9, priority=100, buildTarget="f", mapping="f", backgroundRate=100, systemCapabilities="S", update_type="z",
-                 product="foo", channel="foo*", data_version=1),
-            dict(rule_id=10, priority=100, buildTarget="g", mapping="g", fallbackMapping='fallback', backgroundRate=100, update_type="z",
-                 product="foo", channel="foo", data_version=1),
-            dict(rule_id=2, priority=100, backgroundRate=100, version='3.3', buildTarget='d', mapping='b', update_type='z',
-                 product="a", channel="a", data_version=1),
-            dict(rule_id=3, priority=100, backgroundRate=100, version='3.5', buildTarget='a', mapping='a', update_type='z',
-                 product="a", data_version=1),
-            dict(rule_id=1, priority=100, backgroundRate=100, version='3.5', buildTarget='d', mapping='c', update_type='z',
-                 product="a", channel="a", data_version=1),
+            dict(alias='gandalf', backgroundRate=100, buildTarget='d', channel='a', data_version=1, mapping='a', priority=80, rule_id=4, update_type='z'),
+            dict(backgroundRate=0, buildTarget='d', data_version=1, mapping='c', priority=80, rule_id=5, update_type='z', version='3.3'),
+            dict(alias='radagast', backgroundRate=100, buildTarget='d', channel='a', data_version=1, mapping='a', osVersion='foo 1', priority=100, product='a',
+                 rule_id=6, update_type='z'),
+            dict(backgroundRate=100, buildTarget='d', channel='a', data_version=1, mapping='a', osVersion='foo 2,blah 6', priority=100, product='a', rule_id=7,
+                 update_type='z'),
+            dict(backgroundRate=100, buildTarget='e', channel='a', data_version=1, locale='foo,bar-baz', mapping='d', priority=100, product='a', rule_id=8,
+                 update_type='z'),
+            dict(backgroundRate=100, buildTarget='f', channel='foo*', data_version=1, mapping='f', priority=100, product='foo', rule_id=9,
+                 systemCapabilities='S', update_type='z'),
+            dict(backgroundRate=100, buildTarget='g', channel='foo', data_version=1, fallbackMapping='fallback', mapping='g', priority=100, product='foo',
+                 rule_id=10, update_type='z'),
+            dict(buildTarget='h', channel='a', data_version=1, mapping='h', priority=100, product='a', rule_id=11,
+                 systemCapabilities='GenuineIntel, SSE2', update_type='z'),
+            dict(buildTarget='j', channel='a', data_version=1, mapping='j', priority=100, product='a', rule_id=13,
+                 systemCapabilities='GenuineIntel && SSE3, SSE2', update_type='z'),
+            dict(buildTarget='k', channel='a', data_version=1, mapping='k', priority=100, product='a', rule_id=14,
+                 systemCapabilities='GenuineIntel, SSE', update_type='z'),
+            dict(backgroundRate=100, buildTarget='d', channel='a', data_version=1, mapping='b', priority=100, product='a', rule_id=2,
+                 update_type='z', version='3.3'),
+            dict(backgroundRate=100, buildTarget='a', data_version=1, mapping='a', priority=100, product='a', rule_id=3,
+                 update_type='z', version='3.5'),
+            dict(backgroundRate=100, buildTarget='d', channel='a', data_version=1, mapping='c', priority=100, product='a', rule_id=1,
+                 update_type='z', version='3.5'),
+            dict(buildTarget='i', channel='a', data_version=1, mapping='i', priority=100, product='a', rule_id=12,
+                 systemCapabilities='GenuineIntel && SSE2', update_type='z', version='5.0')
         ]
+
         self.assertEquals(rules, expected)
 
     def testGetOrderedRulesWithCondition(self):
@@ -2042,7 +2059,7 @@ def testGetRulesMatchingQueryOsVersionSubstring(self):
     def testGetRulesMatchingQueryOsVersionSubstringNotAtStart(self):
         rules = self.paths.getRulesMatchingQuery(
             dict(product="a", version='5.0', channel="a", buildTarget='d',
-                 buildID='', locale='', osVersion='bbb foo 1.2.3', distribution='',
+                 buildID='', locale='', osVersion='bbb,foo 1.2.3', distribution='',
                  distVersion='', headerArchitecture='', force=False,
                  queryVersion=3,
                  ),
@@ -2103,6 +2120,149 @@ def testGetRulesMatchingQuerySystemCapabilitiesNoSubstringMatch(self):
         rules = self._stripNullColumns(rules)
         self.assertEquals(rules, [])
 
+    def testGetRulesMatchingQuerySystemCapabilitiesOrs(self):
+        match_1 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='h', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='GenuineIntel',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        match_2 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='h', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='SSE2', version='5.0'),
+            fallbackChannel=''
+        )
+        no_match_1 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='h', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='SSE3', version='5.0'),
+            fallbackChannel=''
+        )
+        no_match_2 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='h', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='SSE', version='5.0'),
+            fallbackChannel=''
+        )
+
+        self.assertEqual(1, len(match_1))
+        self.assertEqual(1, len(match_2))
+        self.assertEqual(11, match_1[0]['rule_id'])
+        self.assertEqual(11, match_2[0]['rule_id'])
+        self.assertEquals(no_match_1, [])
+        self.assertEquals(no_match_2, [])
+
+    def testGetRulesMatchingQuerySystemCapabilitiesBooleanAnds(self):
+        match = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='i', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='GenuineIntel,SSE2',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        no_match_1 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='i', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='AMD,SSE2',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        no_match_2 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='i', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='GenuineIntel,SSE3',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        no_match_3 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='i', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='GenuineIntel,SSE',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+
+        self.assertEqual(len(match), 1)
+        self.assertEqual(match[0]['rule_id'], 12)
+        self.assertEqual(no_match_1, [])
+        self.assertEqual(no_match_2, [])
+        self.assertEqual(no_match_3, [])
+
+    def testGetRulesMatchingQuerySystemCapabilitiesBooleanExact(self):
+        match = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='k', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='SSE',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        no_match = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='k', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='SSE2',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+
+        self.assertEqual(len(match), 1)
+        self.assertEqual(match[0]['rule_id'], 14)
+        self.assertEqual(no_match, [])
+
+    def testGetRulesMatchingQuerySystemCapabilitiesBooleanMixed(self):
+        match_1 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='j', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='GenuineIntel,SSE2',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        match_2 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='j', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='AMD,SSE2',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        match_3 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='j', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='GenuineIntel,SSE3',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        no_match_1 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='j', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='AMD,SSE3',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        no_match_2 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='j', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='GenuineIntel,SSE',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        no_match_3 = self.paths.getRulesMatchingQuery(
+            dict(product='a', channel='a', buildTarget='j', buildID='', locale='',
+                 osVersion='', distribution='', distVersion='', headerArchitecture='',
+                 force=False, queryVersion=6, systemCapabilities='AMD,SSE',
+                 version='5.0'),
+            fallbackChannel=''
+        )
+        self.assertEqual(len(match_1), 1)
+        self.assertEqual(match_1[0]['rule_id'], 13)
+        self.assertEqual(len(match_2), 1)
+        self.assertEqual(match_2[0]['rule_id'], 13)
+        self.assertEqual(len(match_3), 1)
+        self.assertEqual(match_3[0]['rule_id'], 13)
+        self.assertEqual(no_match_1, [])
+        self.assertEqual(no_match_2, [])
+        self.assertEqual(no_match_3, [])
+
     def testGetRulesMatchingQueryFallbackMapping(self):
         rules = self.paths.getRulesMatchingQuery(
             dict(product="foo", version="5.0", channel="foo", buildTarget="g",
@@ -2254,7 +2414,7 @@ def testDeleteRuleThatRequiresSignoffWithChannelGlob(self):
         self.assertRaises(SignoffRequiredError, self.paths.delete, {"rule_id": 9}, changed_by="bill", old_data_version=1)
 
     def testGetNumberOfRules(self):
-        self.assertEquals(self.paths.countRules(), 10)
+        self.assertEquals(self.paths.countRules(), 14)
 
 
 class TestRulesSpecial(unittest.TestCase, RulesTestMixin, MemoryDatabaseMixin):