bug 1021021: add schema for h264 updates - add gmp blob. r=nthomas

auslib/blobs/base.py

@@ -52,24 +52,30 @@ def isValidBlob(format_, blob, topLevel=True):
     return True
 
 def createBlob(data):
+    """Takes a string form of a blob (eg from DB or API) and converts into an
+    actual blob, taking care to notice the schema"""
     # These imports need to be done here to avoid errors due to circular
     # between this module and specific blob modules like apprelease.
     from auslib.blobs.apprelease import ReleaseBlobV1, ReleaseBlobV2, ReleaseBlobV3
+    from auslib.blobs.gmp import GMPBlobV1
+
+    blob_map = {
+        1:    ReleaseBlobV1,
+        2:    ReleaseBlobV2,
+        3:    ReleaseBlobV3,
+        1000: GMPBlobV1,
+    }
 
-    """Takes a string form of a blob (eg from DB or API) and converts into an
-    actual blob, taking care to notice the schema"""
     data = json.loads(data)
-    try:
-        if data['schema_version'] == 1:
-            return ReleaseBlobV1(**data)
-        elif data['schema_version'] == 2:
-            return ReleaseBlobV2(**data)
-        elif data['schema_version'] == 3:
-            return ReleaseBlobV3(**data)
-        else:
-            raise ValueError("schema_version is unknown")
-    except KeyError:
+    schema_version = data.get("schema_version")
+
+    if not schema_version:
         raise ValueError("schema_version is not set")
+    if schema_version not in blob_map:
+        raise ValueError("schema_version is unknown")
+
+    return blob_map[schema_version](**data)
+
 
 class Blob(dict):
     """See isValidBlob for details on how format is used to validate blobs."""

auslib/blobs/gmp.py

@@ -0,0 +1,75 @@
+import re
+
+from auslib.AUS import containsForbiddenDomain
+from auslib.blobs.base import Blob
+
+
+class GMPBlobV1(Blob):
+    format_ = {
+        "name": None,
+        "schema_version": None,
+        "hashFunction": None,
+        "vendors": {
+            "*": {
+                "version": None,
+                "platforms": {
+                    "*": {
+                        "alias": None,
+                        "filesize": None,
+                        "hashValue": None,
+                        "fileUrl": None
+                    }
+                }
+            }
+        }
+    }
+
+    def __init__(self, **kwargs):
+        Blob.__init__(self, **kwargs)
+        if "schema_version" not in self:
+            self["schema_version"] = 1000
+
+    def getVendorsForPlatform(self, platform):
+        for v in self["vendors"]:
+            if platform in self["vendors"][v]["platforms"]:
+                yield v
+
+    def getResolvedPlatform(self, vendor, platform):
+        return self['vendors'][vendor]['platforms'][platform].get('alias', platform)
+
+    def getPlatformData(self, vendor, platform):
+        platform = self.getResolvedPlatform(vendor, platform)
+        return self['vendors'][vendor]['platforms'][platform]
+
+    def shouldServeUpdate(self, updateQuery):
+        # GMP updates should always be returned. It is the responsibility
+        # of the client to decide whether or not any action needs to be taken.
+        return True
+
+    # Because specialForceHosts is only relevant to our own internal servers,
+    # and these type of updates are always served externally, we don't process
+    # them in GMP blobs.
+    def createXML(self, updateQuery, update_type, whitelistedDomains, specialForceHosts):
+        buildTarget = updateQuery["buildTarget"]
+
+        vendorXML = []
+        for vendor in self.getVendorsForPlatform(buildTarget):
+            vendorInfo = self["vendors"][vendor]
+            platformData = self.getPlatformData(vendor, buildTarget)
+
+            url = platformData["fileUrl"]
+            if containsForbiddenDomain(url, whitelistedDomains):
+                continue
+            vendorXML.append('        <addon id="%s" URL="%s" hashFunction="%s" hashValue="%s" size="%d" version="%s"/>' % \
+                (vendor, url, self["hashFunction"], platformData["hashValue"],
+                    platformData["filesize"], vendorInfo["version"]))
+
+        xml = ['<?xml version="1.0"?>']
+        xml.append('<updates>')
+        if vendorXML:
+            xml.append('    <addons>')
+            xml.extend(vendorXML)
+            xml.append('    </addons>')
+        xml.append('</updates>')
+        # ensure valid xml by using the right entity for ampersand
+        return re.sub('&(?!amp;)','&amp;', '\n'.join(xml))

auslib/test/admin/views/test_releases.py

@@ -387,6 +387,7 @@ def testGetNonExistentReleaseBlob(self):
         ret = self.client.get("/releases/huetno/data")
         self.assertStatusCode(ret, 404)
 
+
 class TestReleasesAPI_HTML(ViewTest, HTMLTestMixin):
 
     def testGetReleases(self):
@@ -418,7 +419,6 @@ def testNewReleasePut(self):
 }
 """))
 
-                                                        #json.dumps(newReleaseFile.getvalue())))
         self.assertEquals(ret.status_code, 201, "Status Code: %d, Data: %s" % (ret.status_code, ret.data))
         r = dbo.releases.t.select().where(dbo.releases.name=='new_release').execute().fetchall()
         self.assertEquals(len(r), 1)
@@ -440,6 +440,61 @@ def testNewReleasePut(self):
 }
 """))
 
+    def testGMPReleasePut(self):
+
+        ret = self._put('/releases/gmprel', data=dict(name='gmprel', version='5', product='GMP',
+                                                            blob="""
+{
+    "name": "gmprel",
+    "schema_version": 1000,
+    "hashFunction": "sha512",
+    "vendors": {
+        "foo": {
+            "version": "1",
+            "platforms": {
+                "a": {
+                    "filesize": "2",
+                    "hashValue": "3",
+                    "fileUrl": "http://good.com/4"
+                },
+                "a2": {
+                    "alias": "a"
+                }
+            }
+        }
+    }
+}
+"""))
+
+        self.assertEquals(ret.status_code, 201, "Status Code: %d, Data: %s" % (ret.status_code, ret.data))
+        r = dbo.releases.t.select().where(dbo.releases.name=='gmprel').execute().fetchall()
+        self.assertEquals(len(r), 1)
+        self.assertEquals(r[0]['name'], 'gmprel')
+        self.assertEquals(r[0]['version'], '5')
+        self.assertEquals(r[0]['product'], 'GMP')
+        self.assertEquals(json.loads(r[0]['data']), json.loads("""
+{
+    "name": "gmprel",
+    "schema_version": 1000,
+    "hashFunction": "sha512",
+    "vendors": {
+        "foo": {
+            "version": "1",
+            "platforms": {
+                "a": {
+                    "filesize": "2",
+                    "hashValue": "3",
+                    "fileUrl": "http://good.com/4"
+                },
+                "a2": {
+                    "alias": "a"
+                }
+            }
+        }
+    }
+}
+"""))
+
 
 class TestReleaseHistoryView(ViewTest, HTMLTestMixin):
     def testGetNoRevisions(self):

auslib/test/blobs/test_gmp.py

@@ -0,0 +1,149 @@
+import mock
+from tempfile import mkstemp
+import unittest
+from xml.dom import minidom
+
+from auslib.blobs.gmp import GMPBlobV1
+import auslib.log
+
+
+class TestSchema1Blob(unittest.TestCase):
+    maxDiff = 2000
+
+    def setUp(self):
+        self.cef_patcher = mock.patch("auslib.log.cef_event")
+        self.cef_patcher.start()
+        self.specialForceHosts = ["http://a.com"]
+        self.whitelistedDomains = ["a.com", "boring.com"]
+        self.blob = GMPBlobV1()
+        self.blob.loadJSON("""
+{
+    "name": "fake",
+    "schema_version": 1000,
+    "hashFunction": "SHA512",
+    "vendors": {
+        "c": {
+            "version": "1",
+            "platforms": {
+                "p": {
+                    "filesize": 2,
+                    "hashValue": "3",
+                    "fileUrl": "http://a.com/blah"
+                },
+                "q": {
+                    "filesize": 4,
+                    "hashValue": "5",
+                    "fileUrl": "http://boring.com/blah"
+                },
+                "q2": {
+                    "alias": "q"
+                }
+            }
+        },
+        "d": {
+            "version": "5",
+            "platforms": {
+                "q": {
+                    "filesize": 10,
+                    "hashValue": "11",
+                    "fileUrl": "http://boring.com/foo"
+                },
+                "r": {
+                    "filesize": 666,
+                    "hashValue": "666",
+                    "fileUrl": "http://evil.com/fire"
+                }
+            }
+        }
+    }
+}
+""")
+
+    def tearDown(self):
+        self.cef_patcher.stop()
+
+    def testGetVendorsForPlatform(self):
+        vendors = set([v for v in self.blob.getVendorsForPlatform("q")])
+        self.assertEquals(set(["c", "d"]), vendors)
+
+    def testGetVendorsForPlatformOnlyInOne(self):
+        vendors = set([v for v in self.blob.getVendorsForPlatform("r")])
+        self.assertEquals(set(["d"]), vendors)
+
+    def testGetResolvedPlatform(self):
+        self.assertEquals("q", self.blob.getResolvedPlatform("c", "q2"))
+
+    def testGetPlatformData(self):
+        expected = {
+            "filesize": 4,
+            "hashValue": "5",
+            "fileUrl": "http://boring.com/blah",
+        }
+        self.assertEquals(self.blob.getPlatformData("c", "q2"), expected)
+
+    def testGMPUpdate(self):
+        updateQuery = {
+            "product": "gg", "version": "3", "buildID": "1",
+            "buildTarget": "p", "locale": "l", "channel": "a",
+            "osVersion": "a", "distribution": "a", "distVersion": "a",
+            "force": 0
+        }
+        returned = self.blob.createXML(updateQuery, "minor", self.whitelistedDomains, self.specialForceHosts)
+        returned = minidom.parseString(returned)
+        expected = minidom.parseString("""<?xml version="1.0"?>
+<updates>
+    <addons>
+        <addon id="c" URL="http://a.com/blah" hashFunction="SHA512" hashValue="3" size="2" version="1"/>
+    </addons>
+</updates>
+""")
+        self.assertEqual(returned.toxml(), expected.toxml())
+
+    def testGMPUpdateWithAlias(self):
+        updateQuery = {
+            "product": "gg", "version": "3", "buildID": "1",
+            "buildTarget": "q2", "locale": "l", "channel": "a",
+            "osVersion": "a", "distribution": "a", "distVersion": "a",
+            "force": 0
+        }
+        returned = self.blob.createXML(updateQuery, "minor", self.whitelistedDomains, self.specialForceHosts)
+        returned = minidom.parseString(returned)
+        expected = minidom.parseString("""<?xml version="1.0"?>
+<updates>
+    <addons>
+        <addon id="c" URL="http://boring.com/blah" hashFunction="SHA512" hashValue="5" size="4" version="1"/>
+    </addons>
+</updates>
+""")
+        self.assertEqual(returned.toxml(), expected.toxml())
+
+    def testGMPUpdateMultipleAddons(self):
+        updateQuery = {
+            "product": "gg", "version": "3", "buildID": "1",
+            "buildTarget": "q", "locale": "l", "channel": "a",
+            "osVersion": "a", "distribution": "a", "distVersion": "a",
+            "force": 0
+        }
+        returned = self.blob.createXML(updateQuery, "minor", self.whitelistedDomains, self.specialForceHosts)
+        returned = minidom.parseString(returned)
+        expected = minidom.parseString("""<?xml version="1.0"?>
+<updates>
+    <addons>
+        <addon id="c" URL="http://boring.com/blah" hashFunction="SHA512" hashValue="5" size="4" version="1"/>
+        <addon id="d" URL="http://boring.com/foo" hashFunction="SHA512" hashValue="11" size="10" version="5"/>
+    </addons>
+</updates>
+""")
+        self.assertEqual(returned.toxml(), expected.toxml())
+
+    def testGMPWithForbiddenDomain(self):
+        updateQuery = {
+            "product": "gg", "version": "3", "buildID": "1",
+            "buildTarget": "r", "locale": "l", "channel": "a",
+            "osVersion": "a", "distribution": "a", "distVersion": "a",
+            "force": 0
+        }
+        with mock.patch("auslib.AUS.cef_event") as c:
+            returned = self.blob.createXML(updateQuery, "minor", self.whitelistedDomains, self.specialForceHosts)
+            returned = minidom.parseString(returned)
+            self.assertEqual(returned.getElementsByTagName('updates')[0].firstChild.nodeValue, '\n')