Skip to content

Commit

Permalink
datastore: implementation of auto updating dojo store
Browse files Browse the repository at this point in the history
using websocket+rest api.

master test is now passing, we can detect active to inactive state.

also cleanup the tabs from the js.

Signed-off-by: Pierre Tardy <pierre.tardy@intel.com>t
  • Loading branch information
Pierre Tardy committed Dec 4, 2012
1 parent 7006097 commit ca1fe0e
Show file tree
Hide file tree
Showing 26 changed files with 971 additions and 772 deletions.
2 changes: 1 addition & 1 deletion master/buildbot/scripts/uitestserver.py
Expand Up @@ -31,7 +31,7 @@ def _uitestserver(config):
reactor.stop()
raise defer.returnValue(1)
master = yield fakemaster.make_master_for_uitest(int(config['port']), public_html)
webbrowser.open(master.config.www['url']+"static/bb/tests/runner.html")
webbrowser.open(master.config.www['url']+"app/base/bb/tests/runner.html")

def uitestserver(config):
def async():
Expand Down
14 changes: 12 additions & 2 deletions master/buildbot/test/scenarios/base.py
Expand Up @@ -15,6 +15,7 @@

from buildbot.data import testhooks
from buildbot.test.fake import fakedb
from twisted.internet import defer
import mock

class BaseScenario(testhooks.TestHooksScenario):
Expand All @@ -38,6 +39,15 @@ def populateBaseDb(self):
project=u'world-domination'),
])
def stopMaster(self):
print "stopping master 14"
return self.master.data.updates.masterStopped(name=u'master', masterid=14)

@defer.inlineCallbacks
def addChanges(self):
for rev in xrange(1,2000):
yield self.master.data.updates.addChange(
author=u'warner', branch=u'warnerdb',
category=u'devel', comments=u'fix whitespace',
files=[u'master/buildbot/__init__.py'],
project=u'Buildbot', repository=u'git://warner',
revision=u'0e92a098b'+str(rev), revlink=u'http://warner/0e92a098b'+str(rev),
when_timestamp=256738404,
properties={u'foo': 20})
2 changes: 1 addition & 1 deletion master/buildbot/www/index.html
Expand Up @@ -91,7 +91,7 @@
and fill out the navbar
-->
<script>var dojoConfig = %(dojoConfigJson)s;</script>
<script src="%(baseUrl)s/app/base/dojo/dojo.js"></script>
<script src="%(baseUrl)s/app/base/dojo/dojo.js" data-dojo-config=''></script>
<script>require(["bb/router"], function(router) { router(); });</script>
</body>
</html>
32 changes: 31 additions & 1 deletion master/buildbot/www/rest.py
Expand Up @@ -74,9 +74,28 @@ class V2RootResource(resource.Resource):
knownArgs = set(['as_text', 'filter', 'compact', 'callback'])
def decodeUrlEncoding(self, request):
# calculate the request options
reqOptions = {}
reqOptions = {"start":0,"count":100}
for option in set(request.args) - self.knownArgs:
reqOptions[option] = request.args[option][0]
_range = request.getHeader('X-Range') or request.getHeader('Range') or ""
if _range.startswith("items="):
try:
start, end = map(int,_range.split("=")[1].split("-"))
reqOptions["start"] = start
reqOptions["count"] = end-start
except:
raise ValueError("bad Range/X-Range header")

if "sort" in reqOptions:
def convert(s):
s = s.strip()
if s.startswith('+'):
return(s[1:],0)
if s.startswith('-'):
return(s[1:],1)
return (s,0)
reqOptions["sort"] = map(convert,reqOptions['sort'].split(','))

return reqOptions
def decodeJsonRPC2(self, request, reply):
""" In the case of json encoding, we choose jsonrpc2 as the encoding:
Expand Down Expand Up @@ -187,6 +206,17 @@ def write_error_jsonrpc(msg, errcode=400, jsonrpccode=JSONRPC_CODES["internal_er
compact = self._booleanArg(request, 'compact', not as_text)
callback = request.args.get('callback', [None])[0]

if type(data) == list:
total = reqOptions["count"] = len(data)
if "total" in reqOptions:
total = reqOptions["total"]
if reqOptions["count"] != total:
request.setResponseCode(206) # avoid proxy caching!

request.setHeader("Content-Range", 'items %d-%d/%d'%(reqOptions["start"],
reqOptions["start"]+
reqOptions["count"],
total))
# set up the content type
if as_text:
request.setHeader("content-type", 'text/plain')
Expand Down
96 changes: 96 additions & 0 deletions www/src/bb/datastore.js
@@ -0,0 +1,96 @@
// This file is part of Buildbot. Buildbot is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, version 2.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
// FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
// details.
//
// You should have received a copy of the GNU General Public License along with
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Copyright Buildbot Team Members

/* Generic store serving the list style data APIs of buildbot
Serves as a translation of api between rest+websocket, and dojo's store/query/paging/notification
api described in:
dojo/store/api/Store.js
*/
define(["dojo/_base/declare", "dojo/_base/lang",
"bb/jsonapi",
"bb/websocket",
"dojo/store/Observable",
"dojo/store/Memory",
"dojo/store/JsonRest",
"dojo/store/Cache",
"dojo/aspect"

],
function(declare, lang, api, websocket, observable, Memory, JsonRest, Cache, aspect) {
var after = aspect.after;
function createStore(args) {
var memoryStore = new Memory(lang.delegate({},args));
var restStore = new JsonRest(lang.delegate({sortParam:"sort",target: api.APIv2URL(args.path)+"/"},args));
var store = observable(new Cache(restStore, memoryStore));
store._num_listeners = 0;
after(store,"query", function(results) {
/* hook query, in order to register web socket events
if somebody is observing the results */
after(results,"observe", function(handler) {
store._num_listeners +=1;
if (store._num_listeners === 1) {
store.websocket_listener = websocket.on(args.path, function(event) {
var id = event.message[store.idProperty];
var o = event.message;
console.log(event);
if (memoryStore.index.hasOwnProperty(id)) {
/* reconstruct object from the cache */
var no = lang.clone(memoryStore.get(id));
for (var k in o) {
if (o.hasOwnProperty(k)){
no[k] = o[k];
}
}
o = no;
} else if (event.key.length===3 && event.key[2] === "new"){
/* we dont do store access if this is a new item,
the message is already containing full item,
just cache it.
*/
memoryStore.put(o);
} else {
o = store.get(id);
}
dojo.when(o, function(o) {
store.notify(o, id);
});
});
}
after(handler,"remove", function(x) {
store._num_listeners -=1;
if (store._num_listeners === 0) {
store.websocket_listener.remove();
}
return x;
});
return handler;
});
return results;
});
return store;
}
var datastore = {
change: createStore({path:["change"],idProperty:"changeid"}),
master: createStore({path:["master"],idProperty:"masterid"}),
buildsets: createStore({path:["buildsets"]}),
builders: createStore({path:["builders"]})
};
window.bb.datastore = datastore; /* for console debug */
return datastore;
}
);
17 changes: 0 additions & 17 deletions www/src/bb/fakeChangeStore.js

This file was deleted.

38 changes: 0 additions & 38 deletions www/src/bb/fakeStore.js

This file was deleted.

98 changes: 61 additions & 37 deletions www/src/bb/jsonapi.js
@@ -1,41 +1,65 @@
// This file is part of Buildbot. Buildbot is free software: you can
// redistribute it and/or modify it under the terms of the GNU General Public
// License as published by the Free Software Foundation, version 2.
//
// This program is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
// FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
// details.
//
// You should have received a copy of the GNU General Public License along with
// this program; if not, write to the Free Software Foundation, Inc., 51
// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Copyright Buildbot Team Members


/* utilities, and code related to direct access to the api
deferred based!
*/
define(["dojo/_base/declare", "dojo/_base/Deferred", "dojo/request/xhr","dojo/json"],
function(declare, Deferred, xhr, json){
var api_url = dojo.baseUrl + "/../../../../api/";
var jsonrpc_curid = 0;
return {
createAPIPath: function(a) {
var path=[];
for (var i = 0;i<a.length; i+=1) {
path.push(a[i]);
}
path = path.join("/");
console.log(path, a);
return path;
},
getApiV1: function() {
return xhr(api_url+"v1/"+this.createAPIPath(arguments),
{handleAs:"json"});
},
getApiV2: function() {
return xhr(api_url+"v2/"+this.createAPIPath(arguments),
{handleAs:"json"});
},
control: function(path, method, args) {
jsonrpc_curid+=1;
return xhr(api_url+"v2/"+this.createAPIPath(path),
{handleAs:"json",method:"POST",
headers: {
'Content-Type': 'application/json'
},
data:json.stringify({jsonrpc:"2.0",
method:method,
params:args,
id:jsonrpc_curid})
}
);
}
}; // note the singleton..
define(["dojo/_base/declare", "dojo/_base/Deferred", "dojo/request/xhr","dojo/json",
"dojo/_base/lang"],
function(declare, Deferred, xhr, json, lang){
var api_url = dojo.baseUrl + "/../../../../api/";
var jsonrpc_curid = 0;
return {
createAPIPath: function(a) {
var path=[];
for (var i = 0;i<a.length; i+=1) {
path.push(a[i]);
}
path = path.join("/");
return path;
},
APIv2URL: function(path) {
return api_url+"v2/"+this.createAPIPath(path);
},
getApiV1: function() {
return xhr(api_url+"v1/"+this.createAPIPath(arguments),
{handleAs:"json"});
},
getApiV2: function() {
return xhr(this.APIv2URL(arguments),
{handleAs:"json"});
},
get: function(path, args, xhrargs) {
xhrargs = lang.mixin({handleAs:"json"}, xhrargs);
return xhr(this.APIv2URL(path),
xhrargs);
},
control: function(path, method, args) {
jsonrpc_curid+=1;
return xhr(api_url+"v2/"+this.createAPIPath(path),
{handleAs:"json",method:"POST",
headers: {
'Content-Type': 'application/json'
},
data:json.stringify({jsonrpc:"2.0",
method:method,
params:args,
id:jsonrpc_curid})
}
);
}
}; // note the singleton..
});
8 changes: 5 additions & 3 deletions www/src/bb/router.js
Expand Up @@ -36,7 +36,7 @@ define(["dojo/_base/declare", "dojo/_base/connect","dojo/_base/array","dojo/dom"
path = path.substr(1); /* ignore first '/' */
}
path = path.split("?", 2);
array.forEach(dojoConfig.bb.routes, dojo.hitch(this, function(route, index){
array.forEach(dojo.config.bb.routes, dojo.hitch(this, function(route, index){
if (route.enableif && !this.checkEnableIf(route)) {
return;
}
Expand All @@ -50,7 +50,7 @@ define(["dojo/_base/declare", "dojo/_base/connect","dojo/_base/array","dojo/dom"
checkEnableIf: function(route) {
var ok = true;
array.forEach(route.enableif, dojo.hitch(this, function(ei) {
if (ei == 'admin') {
if (ei === 'admin') {
ok = ok && this.isAdmin();
} else {
ok = false;
Expand All @@ -60,7 +60,7 @@ define(["dojo/_base/declare", "dojo/_base/connect","dojo/_base/array","dojo/dom"
},
fill_navbar: function() {
var navlist = dom.byId("navlist");
var baseUrl = dojoConfig.baseUrl;
var baseUrl = dojo.config.baseUrl;
this.forEachRoute( dojo.hitch(this, function(route, match){
if (route.hasOwnProperty("name")){
var klass = "";
Expand Down Expand Up @@ -110,6 +110,7 @@ define(["dojo/_base/declare", "dojo/_base/connect","dojo/_base/array","dojo/dom"
content.innerHTML = "";
loading.style.display = "none";
w.placeAt(content);
w.startup();
window.bb.curWidget = w;
if(test) {
if (window.doh) { /* doh already loaded! We need to cleanup the iframe */
Expand Down Expand Up @@ -177,5 +178,6 @@ define(["dojo/_base/declare", "dojo/_base/connect","dojo/_base/array","dojo/dom"
this.location_changed();
}


});
});

0 comments on commit ca1fe0e

Please sign in to comment.