Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
bug 693593: create API to enable nightly build submissions to balrog.…
… r=nthomas,rail
  • Loading branch information
bhearsum committed Dec 16, 2011
1 parent eea939e commit ae253f6
Show file tree
Hide file tree
Showing 18 changed files with 740 additions and 32 deletions.
9 changes: 9 additions & 0 deletions admin.ini-dist
@@ -0,0 +1,9 @@
[database]
;Database to be used by the Admin application. Must be r/w.
;dburi=sqlite:////var/www/aus/update.db
;dburi=mysql://user:password@host/database

[logging]
;Where to put the application log. No rotation is done on this file.
logfile=/var/log/aus.log
;level=ERROR
46 changes: 37 additions & 9 deletions admin.py
@@ -1,13 +1,41 @@
import logging
from os import path
import site
import sys

from paste.auth.basic import AuthBasicHandler

from auslib.web.base import app, db
mydir = path.dirname(path.abspath(__file__))
site.addsitedir(mydir)
site.addsitedir(path.join(mydir, 'vendor/lib/python'))

if __name__ == '__main__':
from optparse import OptionParser
parser = OptionParser()
parser.set_defaults(
db='sqlite:///update.db',
port=9000,
)

parser.add_option("-d", "--db", dest="db", help="database to use, relative to inputdir")
parser.add_option("-p", "--port", dest="port", type="int", help="port for server")
parser.add_option("-v", "--verbose", dest="verbose", action="store_true",
help="Verbose output")
options, args = parser.parse_args()

log_level = logging.INFO
if options.verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level, format="%(asctime)s: %(message)s")

from auslib.web.base import app, db

db.setDburi('sqlite:////tmp/bhearsum.db')
db.createTables()
app.config['SECRET_KEY'] = 'abc123'
app.config['DEBUG'] = True
db.setDburi(options.db)
db.createTables()

def auth(environ, username, password):
return username == password
app.wsgi_app = AuthBasicHandler(app.wsgi_app, "Balrog standalone auth", auth)
app.run()
app.config['SECRET_KEY'] = 'abc123'
app.config['DEBUG'] = True
def auth(environ, username, password):
return username == password
app.wsgi_app = AuthBasicHandler(app.wsgi_app, "Balrog standalone auth", auth)
app.run(port=options.port)
22 changes: 22 additions & 0 deletions admin.wsgi
@@ -0,0 +1,22 @@
import logging
from os import path
import site
import sys

mydir = path.dirname(path.abspath(__file__))
site.addsitedir(mydir)
site.addsitedir(path.join(mydir, 'vendor/lib/python'))

from auslib.web.base import db, app as application
from auslib.config import AUSConfig

cfg = AUSConfig('/etc/aus/admin.ini')
errors = cfg.validate()
if errors:
print >>sys.stderr, "Invalid configuration file:"
for err in errors:
print >>sys.stderr, err
sys.exit(1)

logging.basicConfig(filename=cfg.getLogfile(), level=cfg.getLogLevel())
db.setDburi(cfg.getDburi())
99 changes: 99 additions & 0 deletions auslib/blob.py
@@ -0,0 +1,99 @@
import simplejson as json

import logging
log = logging.getLogger(__name__)

CURRENT_SCHEMA_VERSION=1

def isValidBlob(format, blob):
"""Decides whether or not 'blob' is valid based on the format provided.
Validation follows these rules:
1) If there's no format at all, the blob is valid.
2) If the format contains a '*' key, all key names are accepted.
3) If the format doesn't contain a '*' key, all keys in the blob must
also be present in the format.
3) If the value for the key is None, all values for that key are valid.
4) If the value for the key is a dictionary, validate it.
"""
# If there's no format at all, we assume the blob is valid.
if not format:
return True
# If the blob isn't a dictionary-like object, it's not valid!
if not hasattr(blob, 'keys') or not callable(blob.keys):
return False
for key in blob.keys():
# A '*' key in the format means that all key names in the blob are accepted.
if '*' in format:
# But we still need to validate the sub-blob, if it exists.
if format['*'] and not isValidBlob(format['*'], blob[key]):
log.debug("blob is not valid because of key '%s'" % key)
return False
# If there's no '*' key, we need to make sure the key name is valid
# and the sub-blob is valid, if it exists.
elif key not in format or not isValidBlob(format[key], blob[key]):
log.debug("blob is not valid because of key '%s'" % key)
return False
return True

class Blob(dict):
"""See isValidBlob for details on how format is used to validate blobs."""
format = {}

def isValid(self):
"""Decides whether or not this blob is valid based."""
return isValidBlob(self.format, self)

def loadJSON(self, data):
"""Replaces this blob's contents with parsed contents of the json
string provided."""
self.clear()
self.update(json.loads(data))

def getJSON(self):
"""Returns a JSON formatted version of this blob."""
return json.dumps(self)

class ReleaseBlobV1(Blob):
format = {
'name': None,
'schema_version': None,
'detailsUrl': None,
'fileUrls': {
'*': None
},
'ftpFilenames': {
'*': None
},
'bouncerProducts': {
'*': None
},
'hashFunction': None,
'fakePartials': None,
'extv': None,
'appv': None,
'platforms': {
'*': {
'alias': None,
'buildID': None,
'OS_BOUNCER': None,
'OS_FTP': None,
'locales': {
'*': {
'partial': {
'filesize': None,
'from': None,
'hashValue': None,
'fileUrl': None
},
'complete': {
'filesize': None,
'from': None,
'hashValue': None,
'fileUrl': None
}
}
}
}
}
}

66 changes: 63 additions & 3 deletions auslib/db.py
Expand Up @@ -8,6 +8,8 @@
CheckConstraint, create_engine, select, BigInteger
from sqlalchemy.exc import SQLAlchemyError

from auslib.blob import ReleaseBlobV1

import logging
log = logging.getLogger(__name__)

Expand Down Expand Up @@ -329,7 +331,7 @@ def _prepareUpdate(self, trans, where, what, changed_by, old_data_version):
if self.history:
trans.execute(self.history.forUpdate(row, changed_by))
if ret.rowcount != 1:
raise OutdatedDataError("Failed to delet row, old_data_version doesn't match current data_version")
raise OutdatedDataError("Failed to update row, old_data_version doesn't match current data_version")
return ret

def update(self, where, what, changed_by=None, old_data_version=None):
Expand Down Expand Up @@ -555,9 +557,65 @@ def getReleases(self, name=None, product=None, version=None, limit=None):
where.append(self.version==version)
rows = self.select(where=where, limit=limit)
for row in rows:
row['data'] = json.loads(row['data'])
blob = ReleaseBlobV1()
blob.loadJSON(row['data'])
row['data'] = blob
return rows

def getReleaseBlob(self, name):
try:
row = self.select(where=[self.name==name], columns=[self.data], limit=1)[0]
except IndexError:
raise KeyError("Couldn't find release with name '%s'" % name)
blob = ReleaseBlobV1()
blob.loadJSON(row['data'])
return blob

def addRelease(self, name, product, version, blob, changed_by):
if not blob.isValid():
log.debug("Releases.addRelease: invalid blob is %s" % blob)
raise ValueError("Release blob is invalid.")
columns = dict(name=name, product=product, version=version, data=blob.getJSON())
# Raises DuplicateDataError if the release already exists.
self.insert(changed_by, **columns)

def updateRelease(self, name, changed_by, old_data_version, product=None, version=None):
what = {}
if product:
what['product'] = product
if version:
what['version'] = version
self.update(where=[self.name==name], what=what, changed_by=changed_by, old_data_version=old_data_version)

def addLocaleToRelease(self, name, platform, locale, blob, old_data_version, changed_by):
"""Adds or update's the existing data for a specific platform + locale
combination, in the release identified by 'name'. The data is
validated before commiting it, and a ValueError is raised if it is
invalid.
"""
releaseBlob = self.getReleaseBlob(name)
if 'platforms' not in releaseBlob:
releaseBlob['platforms'] = {
platform: {
'locales': {
}
}
}
releaseBlob['platforms'][platform]['locales'][locale] = blob
if not releaseBlob.isValid():
log.debug("Releases.addLocaleToRelease: invalid releaseBlob is %s" % releaseBlob)
raise ValueError("New release blob is invalid.")
where = [self.name==name]
what = dict(data=releaseBlob.getJSON())
self.update(where, what, changed_by, old_data_version)

def getLocale(self, name, platform, locale):
try:
blob = self.getReleaseBlob(name)
return blob['platforms'][platform]['locales'][locale]
except KeyError:
raise KeyError("Couldn't find locale identified by: %s, %s, %s" % (name, platform ,locale))

class Permissions(AUSTable):
"""allPermissions defines the structure and possible options for all
available permissions. Most permissions are identified by an URL,
Expand Down Expand Up @@ -664,7 +722,9 @@ def getOptions(self, username, permission):

def hasUrlPermission(self, username, url, method, urlOptions={}):
"""Check if a user has access to an URL via a specific HTTP method.
GETs are always allowed."""
GETs are always allowed, and admins can always access everything."""
if self.select(where=[self.username==username, self.permission=='admin']):
return True
try:
options = self.getOptions(username, url)
except ValueError:
Expand Down
51 changes: 51 additions & 0 deletions auslib/test/test_blob.py
@@ -0,0 +1,51 @@
import unittest

from auslib.blob import Blob

class SimpleBlob(Blob):
format = {'foo': None}

class MultiLevelBlob(Blob):
format = {
'foo': {
'bar': {
'baz': None
}
}
}

class BlobWithWildcard(Blob):
format = {
'foo': {
'*': None
}
}

class TestBlob(unittest.TestCase):
def testSimpleValid(self):
blob = SimpleBlob(foo='bar')
self.assertTrue(blob.isValid())

def testSimpleInvalid(self):
blob = SimpleBlob(bar='foo')
self.assertFalse(blob.isValid())

def testMultiLevelValid(self):
blob = MultiLevelBlob(foo=dict(bar=dict(baz='abc')))
self.assertTrue(blob.isValid())

def testMultiLevelInvalid(self):
blob = MultiLevelBlob(foo=dict(baz=dict(bar='abc')))
self.assertFalse(blob.isValid())

def testWildcardValid(self):
blob = BlobWithWildcard(foo=dict(bar='abc', baz=123))
self.assertTrue(blob.isValid())

def testWildcardInvalid(self):
blob = BlobWithWildcard(bar=dict(foo='abc'))
self.assertFalse(blob.isValid())

def testWildcardWrongType(self):
blob = BlobWithWildcard(foo='abc')
self.assertFalse(blob.isValid())

0 comments on commit ae253f6

Please sign in to comment.