Merge from master

AUS-server.py

@@ -1,3 +1,5 @@
+from migrate import DatabaseAlreadyControlledError
+
 import logging
 
 log = logging.getLogger(__name__)
@@ -30,7 +32,7 @@
 
     AUS.setDb(options.db)
     try:
-        db.create()
+        AUS.db.create()
     except DatabaseAlreadyControlledError:
         pass
 

Makefile

@@ -31,16 +31,16 @@ test.done: $(ALL_PY_FILES) $(if $(ALWAYS_RUN_TESTS), FORCE)
 	@echo Running unit tests
 ifdef COVERAGE
 	$(RM) -r .coverage htmlcov
-	$(PYTHON_ENV) $(COVERAGE_BIN) run $(NOSE) $(NOSE_ARGS)
+	-$(PYTHON_ENV) $(COVERAGE_BIN) run $(NOSE) $(NOSE_ARGS)
 else
-	$(PYTHON_ENV) $(NOSE) $(NOSE_ARGS)
+	-$(PYTHON_ENV) $(NOSE) $(NOSE_ARGS)
 endif
 	@echo Running rules tests
 ifdef COVERAGE
-	$(COVERAGE_BIN) run -a test-rules.py $(TEST_ARGS)
+	-$(COVERAGE_BIN) run -a test-rules.py $(TEST_ARGS)
 	$(COVERAGE_BIN) html --include='*auslib*'
 else
-	$(PYTHON) test-rules.py $(TEST_ARGS)
+	-$(PYTHON) test-rules.py $(TEST_ARGS)
 endif
 	touch $@
 

auslib/AUS.py

@@ -47,25 +47,23 @@ def isSpecialURL(self, url):
                 return True
         return False
 
-    def identifyRequest(self, updateQuery):
-        self.log.debug("Got updateQuery: %s", updateQuery)
+    def queryMatchesRelease(self, updateQuery, release):
+        """Check if the updateQuery given is the same as the release."""
+        self.log.debug("Trying to match update query to %s" % release['name'])
         buildTarget = updateQuery['buildTarget']
         buildID = updateQuery['buildID']
         locale = updateQuery['locale']
 
-        for release in self.releases.getReleases(product=updateQuery['product'], version=updateQuery['version']):
-            self.log.debug("Trying to match request to %s", release['name'])
-            if buildTarget in release['data']['platforms']:
-                try:
-                    releaseBuildID = release['data'].getBuildID(buildTarget, locale)
-                except KeyError:
-                    continue
-                self.log.debug("releasePlat buildID is: %s", releaseBuildID)
-                if buildID == releaseBuildID:
-                    self.log.debug("Identified query as %s", release['name'])
-                    return release['name']
-        self.log.debug("Couldn't identify query")
-        return None
+        if buildTarget in release['data']['platforms']:
+            try:
+                releaseBuildID = release['data'].getBuildID(buildTarget, locale)
+            # Platform doesn't exist in release, clearly it's not a match!
+            except KeyError:
+                return False
+            self.log.debug("releasePlat buildID is: %s", releaseBuildID)
+            if buildID == releaseBuildID:
+                self.log.debug("Query matched!")
+                return True
 
     def evaluateRules(self, updateQuery):
         self.log.debug("Looking for rules that apply to:")
@@ -76,57 +74,59 @@ def evaluateRules(self, updateQuery):
         )
 
         ### XXX throw any N->N update rules and keep the highest priority remaining one
-        if len(rules) >= 1:
-            rules = sorted(rules,key=lambda rule: rule['priority'], reverse=True)
-            rule = rules[0]
-
-            # for background checks (force=1 missing from query), we might not
-            # serve every request an update
-            # throttle=100 means all requests are served
-            # throttle=25 means only one quarter of requests are served
-            if not updateQuery['force'] and rule['throttle'] < 100:
-                self.log.debug("throttle < 100, rolling the dice")
-                if self.rand.getInt() >= rule['throttle']:
-                    self.log.debug("request was dropped")
-                    rule = None
-
-            self.log.debug("Returning rule:")
-            self.log.debug("%s", rule)
-            return rule
-        return None
+        if len(rules) < 1:
+            return None, None
+
+        rules = sorted(rules,key=lambda rule: rule['priority'], reverse=True)
+        rule = rules[0]
+        self.log.debug("Matching rule: %s" % rule)
+
+        # There's a few cases where we have a matching rule but don't want
+        # to serve an update:
+        # 1) No mapping.
+        if not rule['mapping']:
+            self.log.debug("Matching rule points at null mapping.")
+            return None, None
+
+        # 2) For background checks (force=1 missing from query), we might not
+        # serve every request an update
+        # throttle=100 means all requests are served
+        # throttle=25 means only one quarter of requests are served
+        if not updateQuery['force'] and rule['throttle'] < 100:
+            self.log.debug("throttle < 100, rolling the dice")
+            if self.rand.getInt() >= rule['throttle']:
+                self.log.debug("request was dropped")
+                return None, None
+
+        # 3) Mapping points at the incoming release.
+        release = self.releases.getReleases(name=rule['mapping'], limit=1)[0]
+        if self.queryMatchesRelease(updateQuery, release):
+            self.log.debug("Incoming query is the same as matching rule's mapping.")
+            return None, None
+
+        self.log.debug("Returning release %s", release['name'])
+        return release['data'], rule['update_type']
 
     def getFallbackChannel(self, channel):
         return channel.split('-cck-')[0]
 
-    def expandRelease(self, updateQuery, rule):
-        if not rule or not rule['mapping']:
-            self.log.debug("Couldn't find rule or mapping for %s" % rule)
-            return None
-        # read data from releases table
-        try:
-            res = self.releases.getReleases(name=rule['mapping'], limit=1)[0]
-        except IndexError:
-            # need to log some sort of data inconsistency error here
-            self.log.debug("Failed to get release data from db for:")
-            self.log.debug(rule['mapping'])
-            return None
-        relData = res['data']
+    def expandRelease(self, updateQuery, relData, update_type):
         updateData = defaultdict(list)
 
-        # platforms may be aliased to another platform in the case
-        # of identical data, minimizing the json size
         buildTarget = updateQuery['buildTarget']
-        relDataPlat = relData.getPlatformData(buildTarget)
         locale = updateQuery['locale']
-
         # return early if we don't have an update for this platform
-        if buildTarget not in relData['platforms']:
-            self.log.debug("No platform %s in release %s", buildTarget, rule['mapping'])
+        if buildTarget not in relData.get('platforms', {}):
+            self.log.debug("No platform %s in release %s", buildTarget, relData['name'])
             return updateData
 
+        # platforms may be aliased to another platform in the case
+        # of identical data, minimizing the json size
+        relDataPlat = relData.getPlatformData(buildTarget)
+
         # return early if we don't have an update for this locale
         if locale not in relDataPlat['locales']:
-            self.log.debug("No update to %s for %s/%s", rule['mapping'], buildTarget, locale)
+            self.log.debug("No update to %s for %s/%s", relData['name'], buildTarget, locale)
             return updateData
         else:
             relDataPlatLoc = relDataPlat['locales'][locale]
@@ -143,7 +143,7 @@ def expandRelease(self, updateQuery, rule):
                     updateData[attr] = relData[attr]
 
         # general properties
-        updateData['type'] = rule['update_type']
+        updateData['type'] = update_type
         updateData['schema_version'] = relData['schema_version']
         updateData['build'] = relData.getBuildID(buildTarget, locale)
         if 'detailsUrl' in relData:
@@ -157,7 +157,15 @@ def expandRelease(self, updateQuery, rule):
                 self.log.debug("Skipping patchKey '%s'", patchKey)
                 continue
             patch = relDataPlatLoc[patchKey]
-            if patch['from'] == updateQuery['name'] or patch['from'] == '*':
+            # This is factored out to avoid querying the db when from is '*'
+            def hasAPartial():
+                try:
+                    release = self.releases.getReleases(name=patch['from'], limit=1)[0]
+                    return self.queryMatchesRelease(updateQuery, release)
+                except IndexError:
+                    return False
+
+            if patch['from'] == '*' or hasAPartial():
                 if 'fileUrl' in patch:
                     url = patch['fileUrl']
                 else:
@@ -202,24 +210,22 @@ def expandRelease(self, updateQuery, rule):
         self.log.debug("Returning %s", updateData)
         return updateData
 
-    def createSnippet(self, updateQuery, release):
-        rel = self.expandRelease(updateQuery, release)
-        if not rel:
-            # handle this better, both for prod and debugging
-            self.log.debug("Couldn't expand rule for update target")
+    def createSnippet(self, updateQuery, release, update_type):
+        if not release:
             # XXX: Not sure we should be specifying patch types here, but it's
             # required for tests that have null snippets in them at the time
             # of writing.
             return {"partial": "", "complete": ""}
 
+        rel = self.expandRelease(updateQuery, release, update_type)
         if rel['schema_version'] == 1:
-            return self.createSnippetV1(updateQuery, rel)
+            return self.createSnippetV1(updateQuery, rel, update_type)
         elif rel['schema_version'] == 2:
-            return self.createSnippetV2(updateQuery, rel)
+            return self.createSnippetV2(updateQuery, rel, update_type)
         else:
             return {"partial": "", "complete": ""}
 
-    def createSnippetV1(self, updateQuery, rel):
+    def createSnippetV1(self, updateQuery, rel, update_type):
         snippets = {}
         for patch in rel['patches']:
             snippet  = ["version=1",
@@ -244,7 +250,7 @@ def createSnippetV1(self, updateQuery, rel):
             self.log.debug('%s\n%s' % (s, snippets[s].rstrip()))
         return snippets
 
-    def createSnippetV2(self, updateQuery, rel):
+    def createSnippetV2(self, updateQuery, rel, update_type):
         snippets = {}
         for patch in rel['patches']:
             snippet  = ["version=2",
@@ -274,13 +280,12 @@ def createSnippetV2(self, updateQuery, rel):
             self.log.debug('%s\n%s' % (s, snippets[s].rstrip()))
         return snippets
 
-    def createXML(self, updateQuery, release):
-        rel = self.expandRelease(updateQuery, release)
-
+    def createXML(self, updateQuery, release, update_type):
         # this will fall down all sorts of interesting ways by hardcoding fields
         xml = ['<?xml version="1.0"?>']
         xml.append('<updates>')
-        if rel:
+        if release:
+            rel = self.expandRelease(updateQuery, release, update_type)
             if rel['schema_version'] == 1:
                 updateLine='    <update type="%s" version="%s" extensionVersion="%s" buildID="%s"' % \
                            (rel['type'], rel['appv'], rel['extv'], rel['build'])

auslib/admin/base.py

@@ -16,9 +16,8 @@
   SingleReleaseView, ReleasesPageView, ReleaseHistoryView
 from auslib.admin.views.rules import RulesPageView, RulesAPIView, \
   SingleRuleView, RuleHistoryView
-from auslib.admin.views.index import IndexPageView, RecentChangesTableView
 from auslib.admin.views.history import DiffView, FieldView
-from auslib.admin.views.index import IndexPageView
+from auslib.admin.views.index import IndexPageView, RecentChangesTableView
 
 @app.errorhandler(500)
 def isa(error):

auslib/admin/views/forms.py

@@ -92,8 +92,10 @@ class ReleaseForm(Form):
     data_version = IntegerField('data_version', widget=HiddenInput())
     product = TextField('Product', validators=[Required()])
     version = TextField('Version', validators=[Required()])
+    hashFunction = TextField('Hash Function')
     data = JSONTextField('Data', validators=[Required()])
     copyTo = JSONTextField('Copy To', default=list)
+    alias = JSONTextField('Alias', default=list)
 
 class RuleForm(Form):
     throttle = IntegerField('Throttle', validators=[Required(), validators.NumberRange(0, 100) ])

auslib/admin/views/index.py

@@ -1,5 +1,4 @@
 import collections
-import time
 
 from flask import render_template, request
 

auslib/admin/views/releases.py

@@ -1,7 +1,5 @@
 import simplejson as json
 
-from sqlalchemy.exc import SQLAlchemyError
-
 from flask import render_template, Response, jsonify, make_response, request
 
 from auslib.blob import ReleaseBlobV1, CURRENT_SCHEMA_VERSION
@@ -69,8 +67,10 @@ def changeRelease(release, changed_by, transaction, existsCallback, commitCallba
         return Response(status=400, response=form.errors)
     product = form.product.data
     version = form.version.data
+    hashFunction = form.hashFunction.data
     incomingData = form.data.data
     copyTo = form.copyTo.data
+    alias = form.alias.data
     old_data_version = form.data_version.data
 
     allReleases = [release]
@@ -91,13 +91,18 @@ def changeRelease(release, changed_by, transaction, existsCallback, commitCallba
                 # If the product we're given doesn't match the one in the DB, panic.
                 if product != releaseInfo['product']:
                     return Response(status=400, response="Product name '%s' doesn't match the one on the release object ('%s') for release '%s'" % (product, releaseInfo['product'], rel))
+                if 'hashFunction' in releaseInfo['data'] and hashFunction != releaseInfo['data']['hashFunction']:
+                    return Response(status=400, response="hashFunction '%s' doesn't match the one on the release object ('%s') for release '%s'" % (hashFunction, releaseInfo['data']['hashFunction'], rel))
             # If this isn't the release in the URL...
             else:
                 # Use the data_version we just grabbed from the db.
                 old_data_version = releaseInfo['data_version']
         except IndexError:
             # If the release doesn't already exist, create it, and set old_data_version appropriately.
-            releaseInfo = createRelease(rel, product, version, changed_by, transaction, dict(name=rel))
+            newReleaseData = dict(name=rel)
+            if hashFunction:
+                newReleaseData['hashFunction'] = hashFunction
+            releaseInfo = createRelease(rel, product, version, changed_by, transaction, newReleaseData)
             old_data_version = 1
 
         # If the version doesn't match, just update it. This will be the case for nightlies
@@ -109,7 +114,10 @@ def changeRelease(release, changed_by, transaction, existsCallback, commitCallba
                 transaction=transaction)
             old_data_version += 1
 
-        commitCallback(rel, product, version, incomingData, releaseInfo['data'], old_data_version)
+        extraArgs = {}
+        if alias:
+            extraArgs['alias'] = alias
+        commitCallback(rel, product, version, incomingData, releaseInfo['data'], old_data_version, extraArgs)
 
     new_data_version = db.releases.getReleases(name=release, transaction=transaction)[0]['data_version']
     if new:
@@ -147,9 +155,9 @@ def exists(rel, product, version):
                     locale=locale, transaction=transaction)
             return False
 
-        def commit(rel, product, version, localeData, releaseData, old_data_version):
+        def commit(rel, product, version, localeData, releaseData, old_data_version, extraArgs):
             return db.releases.addLocaleToRelease(name=rel, platform=platform,
-                locale=locale, data=localeData, old_data_version=old_data_version,
+                locale=locale, data=localeData, alias=extraArgs.get('alias'), old_data_version=old_data_version,
                 changed_by=changed_by, transaction=transaction)
 
         return changeRelease(release, changed_by, transaction, exists, commit, self.log)
@@ -197,7 +205,7 @@ def exists(rel, product, version):
                 return True
             return False
 
-        def commit(rel, product, version, newReleaseData, releaseData, old_data_version):
+        def commit(rel, product, version, newReleaseData, releaseData, old_data_version, extraArgs):
             releaseData.update(newReleaseData)
             return db.releases.updateRelease(name=rel, blob=releaseData,
                 changed_by=changed_by, old_data_version=old_data_version,

auslib/admin/views/rules.py

@@ -1,8 +1,6 @@
 import json
 from flask import render_template, Response, make_response, request
 
-from sqlalchemy.exc import SQLAlchemyError
-
 from auslib.util import getPagination
 from auslib.admin.base import db
 from auslib.admin.views.base import (