bug 1021021: add schema for h264 updates - reorganize blob code. r=nt…

…homas

auslib/admin/views/forms.py

@@ -2,7 +2,7 @@
 
 from flaskext.wtf import Form, TextField, Required, TextInput, FileInput, IntegerField, SelectField, validators, HiddenInput, Optional
 
-from auslib.blob import createBlob
+from auslib.blobs.base import createBlob
 
 import logging
 log = logging.getLogger(__name__)

auslib/admin/views/releases.py

@@ -3,7 +3,7 @@
 from flask import render_template, Response, jsonify, make_response, request
 
 from auslib import dbo
-from auslib.blob import createBlob
+from auslib.blobs.base import createBlob
 from auslib.db import OutdatedDataError
 from auslib.log import cef_event, CEF_WARN, CEF_ALERT
 from auslib.util import getPagination

auslib/blob.py → auslib/blobs/apprelease.py

@@ -5,80 +5,12 @@
 log = logging.getLogger(__name__)
 
 from auslib import dbo
-from auslib.AUS import isSpecialURL, containsForbiddenDomain, getFallbackChannel
+from auslib.AUS import containsForbiddenDomain, getFallbackChannel
+from auslib.blobs.base import Blob
 from auslib.util.versions import MozillaVersion
 
 
-def isValidBlob(format_, blob, topLevel=True):
-    """Decides whether or not 'blob' is valid based on the format provided.
-       Validation follows these rules:
-       1) If there's no format at all, the blob is valid.
-       2) If the format contains a '*' key, all key names are accepted.
-       3) If the format doesn't contain a '*' key, all keys in the blob must
-          also be present in the format.
-       3) If the value for the key is None, all values for that key are valid.
-       4) If the value for the key is a dictionary, validate it.
-    """
-    # If there's no format at all, we assume the blob is valid.
-    if not format_:
-        return True
-    # If the blob isn't a dictionary-like or list-like object, it's not valid!
-    if not isinstance(blob, (dict,list)):
-        return False
-    # If the blob format has a schema_version then that's a mandatory int
-    if topLevel and 'schema_version' in format_:
-        if 'schema_version' not in blob or not isinstance(blob['schema_version'], int):
-            log.debug("blob is not valid because schema_version is not defined, or non-integer")
-            return False
-    # check the blob against the format
-    if isinstance(blob, dict):
-        for key in blob.keys():
-            # A '*' key in the format means that all key names in the blob are accepted.
-            if '*' in format_:
-                # But we still need to validate the sub-blob, if it exists.
-                if format_['*'] and not isValidBlob(format_['*'], blob[key], topLevel=False):
-                    log.debug("blob is not valid because of key '%s'" % key)
-                    return False
-            # If there's no '*' key, we need to make sure the key name is valid
-            # and the sub-blob is valid, if it exists.
-            elif key not in format_ or not isValidBlob(format_[key], blob[key], topLevel=False):
-                log.debug("blob is not valid because of key '%s'" % key)
-                return False
-    else:
-        # Empty lists are not allowed. These can be represented by leaving out the key entirely.
-        if len(blob) == 0:
-            return False
-        for subBlob in blob:
-            # Other than the empty list check above, we can hand off the rest
-            # of the validation to another isValidBlob call!
-            if not isValidBlob(format_[0], subBlob, topLevel=False):
-                return False
-    return True
-
-def createBlob(data):
-    """Takes a string form of a blob (eg from DB or API) and converts into an
-    actual blob, taking care to notice the schema"""
-    data = json.loads(data)
-    try:
-        if data['schema_version'] == 1:
-            return ReleaseBlobV1(**data)
-        elif data['schema_version'] == 2:
-            return ReleaseBlobV2(**data)
-        elif data['schema_version'] == 3:
-            return ReleaseBlobV3(**data)
-        else:
-            raise ValueError("schema_version is unknown")
-    except KeyError:
-        raise ValueError("schema_version is not set")
-
-class Blob(dict):
-    """See isValidBlob for details on how format is used to validate blobs."""
-    format_ = {}
-
-    def __init__(self, *args, **kwargs):
-        self.log = logging.getLogger(self.__class__.__name__)
-        dict.__init__(self, *args, **kwargs)
-
+class ReleaseBlobBase(Blob):
     def matchesUpdateQuery(self, updateQuery):
         self.log.debug("Trying to match update query to %s" % self["name"])
         buildTarget = updateQuery["buildTarget"]
@@ -96,21 +28,6 @@ def matchesUpdateQuery(self, updateQuery):
                 self.log.debug("Query matched!")
                 return True
 
-    def isValid(self):
-        """Decides whether or not this blob is valid based."""
-        self.log.debug('Validating blob %s' % self)
-        return isValidBlob(self.format_, self)
-
-    def loadJSON(self, data):
-        """Replaces this blob's contents with parsed contents of the json
-           string provided."""
-        self.clear()
-        self.update(json.loads(data))
-
-    def getJSON(self):
-        """Returns a JSON formatted version of this blob."""
-        return json.dumps(self)
-
     def getResolvedPlatform(self, platform):
         return self['platforms'][platform].get('alias', platform)
 
@@ -161,11 +78,8 @@ def _getUrl(self, updateQuery, patch, specialForceHosts, ftpFilename, bouncerPro
             url = url.replace('%PRODUCT%', bouncerProduct)
             url = url.replace('%OS_BOUNCER%', platformData['OS_BOUNCER'])
         # pass on forcing for special hosts (eg download.m.o for mozilla metrics)
-        if updateQuery['force'] and isSpecialURL(url, specialForceHosts):
-            if '?' in url:
-                url += '&force=1'
-            else:
-                url += '?force=1'
+        if updateQuery['force']:
+            url = self.processSpecialForceHosts(url, specialForceHosts)
 
         return url
 
@@ -275,7 +189,7 @@ def _getPatchesXML(self, localeData, updateQuery, whitelistedDomains, specialFor
         return patches
 
 
-class ReleaseBlobV1(Blob, SingleUpdateXMLMixin):
+class ReleaseBlobV1(ReleaseBlobBase, SingleUpdateXMLMixin):
     format_ = {
         'name': None,
         'schema_version': None,
@@ -457,7 +371,7 @@ def _getUpdateLineXML(self, buildTarget, locale, update_type):
         return updateLine
 
 
-class ReleaseBlobV2(Blob, NewStyleVersionsMixin, SingleUpdateXMLMixin):
+class ReleaseBlobV2(ReleaseBlobBase, NewStyleVersionsMixin, SingleUpdateXMLMixin):
     """ Changes from ReleaseBlobV1:
          * appv, extv become appVersion, platformVersion, displayVersion
         Added:
@@ -612,7 +526,7 @@ def _getPatchesXML(self, localeData, updateQuery, whitelistedDomains, specialFor
         return patches
 
 
-class ReleaseBlobV3(Blob, NewStyleVersionsMixin, MultipleUpdatesXMLMixin):
+class ReleaseBlobV3(ReleaseBlobBase, NewStyleVersionsMixin, MultipleUpdatesXMLMixin):
     """ Changes from ReleaseBlobV2:
          * support multiple partials
            * remove "partial" and "complete" from locale level

auslib/blobs/base.py

@@ -0,0 +1,106 @@
+import simplejson as json
+
+import logging
+log = logging.getLogger(__name__)
+
+from auslib.AUS import isSpecialURL
+
+def isValidBlob(format_, blob, topLevel=True):
+    """Decides whether or not 'blob' is valid based on the format provided.
+       Validation follows these rules:
+       1) If there's no format at all, the blob is valid.
+       2) If the format contains a '*' key, all key names are accepted.
+       3) If the format doesn't contain a '*' key, all keys in the blob must
+          also be present in the format.
+       3) If the value for the key is None, all values for that key are valid.
+       4) If the value for the key is a dictionary, validate it.
+    """
+    # If there's no format at all, we assume the blob is valid.
+    if not format_:
+        return True
+    # If the blob isn't a dictionary-like or list-like object, it's not valid!
+    if not isinstance(blob, (dict,list)):
+        return False
+    # If the blob format has a schema_version then that's a mandatory int
+    if topLevel and 'schema_version' in format_:
+        if 'schema_version' not in blob or not isinstance(blob['schema_version'], int):
+            log.debug("blob is not valid because schema_version is not defined, or non-integer")
+            return False
+    # check the blob against the format
+    if isinstance(blob, dict):
+        for key in blob.keys():
+            # A '*' key in the format means that all key names in the blob are accepted.
+            if '*' in format_:
+                # But we still need to validate the sub-blob, if it exists.
+                if format_['*'] and not isValidBlob(format_['*'], blob[key], topLevel=False):
+                    log.debug("blob is not valid because of key '%s'" % key)
+                    return False
+            # If there's no '*' key, we need to make sure the key name is valid
+            # and the sub-blob is valid, if it exists.
+            elif key not in format_ or not isValidBlob(format_[key], blob[key], topLevel=False):
+                log.debug("blob is not valid because of key '%s'" % key)
+                return False
+    else:
+        # Empty lists are not allowed. These can be represented by leaving out the key entirely.
+        if len(blob) == 0:
+            return False
+        for subBlob in blob:
+            # Other than the empty list check above, we can hand off the rest
+            # of the validation to another isValidBlob call!
+            if not isValidBlob(format_[0], subBlob, topLevel=False):
+                return False
+    return True
+
+def createBlob(data):
+    # These imports need to be done here to avoid errors due to circular
+    # between this module and specific blob modules like apprelease.
+    from auslib.blobs.apprelease import ReleaseBlobV1, ReleaseBlobV2, ReleaseBlobV3
+
+    """Takes a string form of a blob (eg from DB or API) and converts into an
+    actual blob, taking care to notice the schema"""
+    data = json.loads(data)
+    try:
+        if data['schema_version'] == 1:
+            return ReleaseBlobV1(**data)
+        elif data['schema_version'] == 2:
+            return ReleaseBlobV2(**data)
+        elif data['schema_version'] == 3:
+            return ReleaseBlobV3(**data)
+        else:
+            raise ValueError("schema_version is unknown")
+    except KeyError:
+        raise ValueError("schema_version is not set")
+
+class Blob(dict):
+    """See isValidBlob for details on how format is used to validate blobs."""
+    format_ = {}
+
+    def __init__(self, *args, **kwargs):
+        self.log = logging.getLogger(self.__class__.__name__)
+        dict.__init__(self, *args, **kwargs)
+
+    def isValid(self):
+        """Decides whether or not this blob is valid based."""
+        self.log.debug('Validating blob %s' % self)
+        return isValidBlob(self.format_, self)
+
+    def loadJSON(self, data):
+        """Replaces this blob's contents with parsed contents of the json
+           string provided."""
+        self.clear()
+        self.update(json.loads(data))
+
+    def getJSON(self):
+        """Returns a JSON formatted version of this blob."""
+        return json.dumps(self)
+
+    def shouldServeUpdate(self, updateQuery):
+        raise NotImplementedError()
+
+    def processSpecialForceHosts(self, url, specialForceHosts):
+        if isSpecialURL(url, specialForceHosts):
+            if '?' in url:
+                url += '&force=1'
+            else:
+                url += '?force=1'
+        return url

auslib/db.py

@@ -13,7 +13,7 @@
 import migrate.versioning.schema
 import migrate.versioning.api
 
-from auslib.blob import createBlob
+from auslib.blobs.base import createBlob
 from auslib.log import cef_event, CEF_ALERT
 from auslib.util.comparison import string_compare, version_compare
 

auslib/test/test_blob.py → auslib/test/blobs/test_apprelease.py

@@ -2,65 +2,14 @@
 from xml.dom import minidom
 
 from auslib import dbo
-from auslib.blob import Blob, ReleaseBlobV1, ReleaseBlobV2, ReleaseBlobV3
+from auslib.blobs.apprelease import ReleaseBlobBase, ReleaseBlobV1, ReleaseBlobV2, \
+    ReleaseBlobV3
 
-class SimpleBlob(Blob):
+class SimpleBlob(ReleaseBlobBase):
     format_ = {'foo': None}
 
-class MultiLevelBlob(Blob):
-    format_ = {
-        'foo': {
-            'bar': {
-                'baz': None
-            }
-        }
-    }
-
-class BlobWithWildcard(Blob):
-    format_ = {
-        'foo': {
-            '*': None
-        }
-    }
-
-class BlobWithList(Blob):
-    format_ = {
-        'foo': [
-            {
-                'bar': None
-            }
-        ]
-    }
-
-class TestBlob(unittest.TestCase):
-    def testSimpleValid(self):
-        blob = SimpleBlob(foo='bar')
-        self.assertTrue(blob.isValid())
-
-    def testSimpleInvalid(self):
-        blob = SimpleBlob(bar='foo')
-        self.assertFalse(blob.isValid())
-
-    def testMultiLevelValid(self):
-        blob = MultiLevelBlob(foo=dict(bar=dict(baz='abc')))
-        self.assertTrue(blob.isValid())
-
-    def testMultiLevelInvalid(self):
-        blob = MultiLevelBlob(foo=dict(baz=dict(bar='abc')))
-        self.assertFalse(blob.isValid())
-
-    def testWildcardValid(self):
-        blob = BlobWithWildcard(foo=dict(bar='abc', baz=123))
-        self.assertTrue(blob.isValid())
-
-    def testWildcardInvalid(self):
-        blob = BlobWithWildcard(bar=dict(foo='abc'))
-        self.assertFalse(blob.isValid())
-
-    def testWildcardWrongType(self):
-        blob = BlobWithWildcard(foo='abc')
-        self.assertFalse(blob.isValid())
 
+class TestReleaseBlobBase(unittest.TestCase):
     def testGetResolvedPlatform(self):
         blob = SimpleBlob(platforms=dict(a=dict(), b=dict(alias='a')))
         self.assertEquals('a', blob.getResolvedPlatform('a'))
@@ -99,21 +48,6 @@ def testGetBuildIDMissingLocaleBuildIDAtPlatform(self):
         self.assertRaises(KeyError, blob.getBuildID, 'c', 'a')
     # XXX: should we support the locale overriding the platform? this should probably be invalid
 
-    def testBlobWithList(self):
-        blob = BlobWithList(foo=[dict(bar=1)])
-        self.assertTrue(blob.isValid())
-
-    def testBlobWithEmptyList(self):
-        blob = BlobWithList(foo=[])
-        self.assertFalse(blob.isValid())
-
-    def testBlobWithMissingList(self):
-        blob = BlobWithList()
-        self.assertTrue(blob.isValid())
-
-    def testBlobWithInvalidSublist(self):
-        blob = BlobWithList(foo=[dict(blah=2)])
-        self.assertFalse(blob.isValid())
 
 class TestReleaseBlobV1(unittest.TestCase):
     def testGetAppv(self):