Permalink
Browse files

islands, heartbeats, other nifty things

  • Loading branch information...
Colin Roache
Colin Roache committed May 10, 2010
1 parent fa0815d commit 913e227278b2337fb400883169d9bb5a1c377400
Showing with 103 additions and 33 deletions.
  1. +60 −16 Server/db.py
  2. +27 −10 Server/dpgp.py
  3. +10 −4 Server/static/gp_brain.js
  4. +2 −0 Server/static/gp_controller.js
  5. +3 −2 Server/templates/gp_worker.js
  6. +1 −1 Server/templates/worker.html
View
@@ -1,19 +1,38 @@
import tornado.database as database
+from time import time
from uuid import uuid4
import json
-def getFreshUUID():
- return uuid4().hex
-
def getConnection():
return database.Connection("/tmp/mysql.sock", user="root", database="dpgpjs")
+def createClientForProblem(problem_id):
+ c=getConnection()
+ num_islands = c.query("SELECT num_islands FROM problems WHERE id = %s LIMIT 1" % problem_id)[0]['num_islands']
+ island_counts = c.query("SELECT island, COUNT(island) as num_on_island FROM clients WHERE problem_id = %s and last_seen > %s GROUP BY island" % (problem_id,int(time())-60)) # checks that the client has checked in within a minute
+ island_list = [[i,0] for i in range(num_islands)]
+
+ for count in island_counts:
+ island_list[count['island']][1] += count['num_on_island']
+
+ island_list.sort(cmp=lambda x,y: cmp(x[1],y[1]))
+
+ target_island = island_list[0][0]
+
+ new_uuid = uuid4().hex
+ insert_sql = "INSERT INTO clients (id,last_seen,island,problem_id) VALUES ('%s', %s, %s, %s)" % (new_uuid, int(time()), target_island, problem_id)
+
+ c.execute(insert_sql)
+
+ print island_list
+ return new_uuid
+
def getProblems():
c=getConnection()
problems = [problem for problem in c.query("SELECT * FROM problems")]
for problem in problems:
- program = c.query("SELECT * FROM programs WHERE problem = %s ORDER BY fitness DESC LIMIT 1" % problem.id)
+ program = c.query("SELECT * FROM programs WHERE problem = %s ORDER BY fitness LIMIT 1" % problem.id)
if(len(program) != 0):
problem.bestFitness = program[0].fitness
problem.bestProgram = program[0].program_string
@@ -24,7 +43,6 @@ def getProblems():
def getGPParams(problem_id):
c=getConnection()
-
problem = c.query("SELECT * FROM problems WHERE id = %s" % problem_id)[0]
return problem
@@ -41,36 +59,62 @@ def getProblem(problem_id):
problem.bestProgram = "none"
return problem
+def getClientInfo(client_id):
+ c=getConnection()
+ sql = "SELECT problem_id, island FROM clients WHERE id = '%s' LIMIT 1" % (client_id)
+ fetch = c.query(sql)[0]
+ return fetch
def updateProblem(problem_id,name,comments,allowed,start_population,max_population,tournament_size,crossover_probability,mutation_probability,clone_probability):
c=getConnection()
return c.execute("UPDATE problems SET name = \"%s\", comments = \"%s\", allowed = \"%s\" start_population = \"%s\", max_population = \"%s\", tournament_size = \"%s\", crossover_probability = \"%s\", mutation_probability = \"%s\", clone_probability = \"%s\" WHERE id = %s LIMIT 1" % (name, comments, allowed, start_population,max_population,tournament_size,crossover_probability,mutation_probability, clone_probability, problem_id))
-def getProgramsForProblem(problem_id,num_programs):
+def getNeighborsForClient(client_id,num_programs):
+ if num_programs == 0: return []
+
c=getConnection()
- sql = "SELECT * FROM programs WHERE problem = %s ORDER BY fitness,RAND() DESC LIMIT 0,%s" % (problem_id,num_programs)
+ fetch = getClientInfo(client_id)
- results = c.query(sql)
+ results = c.query("SELECT * FROM programs WHERE problem = %s AND island = %s ORDER BY fitness,RAND() DESC LIMIT 0,%s" % (fetch['problem_id'],fetch['island'],num_programs))
+
+ return results
- return '[' + ",".join (["\"%s\"" % result['program_string'] for result in results]) + ']'
+def getStrangersForClient(client_id,num_programs):
+ if num_programs == 0: return []
+
+ c=getConnection()
+
+ fetch = getClientInfo(client_id)
+
+ results = c.query("SELECT * FROM programs WHERE problem = %s AND island <> %s ORDER BY fitness,RAND() DESC LIMIT 0,%s" % (fetch['problem_id'],fetch['island'],num_programs))
+
+ if len(results) == 0:
+ return []
+
+ return results
+
+def getProgramArray(dbArray):
+ return '[' + ",".join (["\"%s\"" % result['program_string'] for result in dbArray]) + ']'
def getFitnessCases(problem_id):
c=getConnection()
return [case for case in c.query("SELECT * FROM fitness_cases WHERE problem_id = %s" % problem_id) ]
-def storeWorkerResults(problem_id,program,program_fitness,client_id):
+def storeWorkerResults(client_id,program,program_fitness):
c=getConnection()
+ fetch=getClientInfo(client_id)
- print problem_id
- print program
- print program_fitness
- print client_id
-
- sql = "INSERT INTO programs (`problem`,`program_string`,`fitness`,`client_id`) VALUES (%s,\'%s\',%s,\'%s\')" % (problem_id,program.replace('%','%%'),program_fitness,client_id)
+ sql = "INSERT INTO programs (`problem`,`program_string`,`fitness`,`island`,`client_id`) VALUES (%s,\'%s\',%s,%s,\'%s\')" % (fetch['problem_id'],program.replace('%','%%'),program_fitness,fetch['island'],client_id)
c.execute(sql)
+
+def setLastSeenToNow(client_id):
+ c=getConnection()
+ sql = "UPDATE clients SET last_seen = %s WHERE id = '%s'" % (time(),client_id)
+
+ c.execute(sql)
def importProblemJSON(json_txt):
try:
View
@@ -5,32 +5,48 @@
import tornado.ioloop
import tornado.web
+from random import random
+
import db
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.render("templates/main.html", problems=db.getProblems())
+class HeartbeatHandler(tornado.web.RequestHandler):
+ def get(self,client_id):
+ print "receieved heartbeat from client %s" % client_id
+ db.setLastSeenToNow(client_id)
+
class WorkerHandler(tornado.web.RequestHandler):
- def get(self,worker_id):
- self.render("templates/worker.html", worker_id = worker_id, client_id = db.getFreshUUID())
+ def get(self,problem_id):
+ self.render("templates/worker.html", client_id = db.createClientForProblem(problem_id))
class WorkerJSHandler(tornado.web.RequestHandler):
- def get(self,problem_id):
+ def get(self,client_id):
+ problem_id=db.getClientInfo(client_id)['problem_id']
self.render("templates/gp_worker.js", fitness_cases=db.getFitnessCases(problem_id), gp_params=db.getGPParams(problem_id))
class ResultsUploadHandler(tornado.web.RequestHandler):
def post(self):
uploadedData = json.loads(self.request.body)
# print uploadedData
- db.storeWorkerResults(uploadedData['problem_id'],uploadedData['program']['code'],uploadedData['program']['fitness'],uploadedData['client_id'])
+ db.storeWorkerResults(uploadedData['client_id'],uploadedData['program']['code'],uploadedData['program']['fitness'])
class RequestProgramsHandler(tornado.web.RequestHandler):
- def get(self,problem_id,num_programs):
- self.write(str(db.getProgramsForProblem(problem_id,num_programs)))
- print "gave programs to client"
- # self.write('hi?')
+ def get(self,client_id,num_programs):
+ client_info = db.getClientInfo(client_id)
+
+ problem_info = db.getProblem(client_info['problem_id'])
+ if random() < problem_info['local_fetch_probability']/100.0:
+ program_string = str(db.getProgramArray(db.getNeighborsForClient(client_id,num_programs)))
+ else:
+ strangers = db.getStrangersForClient(client_id,num_programs)
+ programs = strangers + db.getNeighborsForClient(client_id,int(num_programs) - len(strangers))
+ program_string = str(db.getProgramArray(programs))
+ self.write(program_string)
+ print "gave programs to client"
class AdminHandler(tornado.web.RequestHandler):
def get(self):
self.render("templates/admin/index.html",problems=db.getProblems())
@@ -63,10 +79,11 @@ def post(self,problem_id):
application = tornado.web.Application([
(r"/", MainHandler),
- (r"/gp_worker([0-9]+)\.js", WorkerJSHandler),
+ (r"/gp_worker([A-Za-z0-9]+)\.js", WorkerJSHandler),
+ (r'/heartbeat([A-Za-z0-9]+)',HeartbeatHandler),
(r"/worker([0-9]+)", WorkerHandler),
(r"/uploadresults", ResultsUploadHandler),
- (r"/requestprograms([0-9]+)\&num_programs\=([0-9]+)", RequestProgramsHandler),
+ (r"/requestprograms([A-Za-z0-9]+)\&num_programs\=([0-9]+)", RequestProgramsHandler),
(r"/admin", AdminHandler),
(r"/admin/upload/", AdminUploadHandler),
(r"/admin/edit([0-9]+)", AdminEditHandler)
View
@@ -14,7 +14,7 @@ if (!runInWorker) {
function updateStatus() {};
}
-function startGPBrain(problem_id,client_id)
+function startGPBrain(client_id)
{
// testGP();
// start();
@@ -27,7 +27,6 @@ function startGPBrain(problem_id,client_id)
return;
}
else if (a.data['msgtype'] == 'uploadProgram'){
- a.data['msg']['problem_id']=problem_id;
a.data['msg']['client_id']=client_id;
$.ajax({'url':'uploadresults', 'type':'POST', 'data':JSON.stringify(a.data['msg'])});
console.log(JSON.stringify(a.data['msg']));
@@ -52,7 +51,7 @@ function startGPBrain(problem_id,client_id)
worker.postMessage({'msgtype' : "heartbeat"});
}
else if(a.data['msgtype'] == 'downloadPrograms') {
- $.ajax({url:'requestprograms'+problem_id+'&num_programs='+a.data['msg']['numPrograms'],
+ $.ajax({url:'requestprograms'+client_id+'&num_programs='+a.data['msg']['numPrograms'],
type:'GET',
dataType:'json',
success:function(json){
@@ -69,8 +68,15 @@ function startGPBrain(problem_id,client_id)
timeStarted = new Date().getTime();
// var worker = JsWorker.createWorkerFromUrl("gp_worker.js", onMessage);
+ //Send heartbeats every 30 seconds
+ setInterval(function() {
+ console.log("time to heartbeat the server.");
+ $.ajax({url:'heartbeat'+client_id,
+ type: 'GET'});
+ }, 30000);
+
if (runInWorker) {
- worker = new Worker('gp_worker' + problem_id +'.js');
+ worker = new Worker('gp_worker' + client_id +'.js');
worker.onmessage = onMessage;
worker.onerror = function (err) {alert ('Error! ' + err);};
@@ -6,6 +6,8 @@
/*,"BOOLEAN.=","BOOLEAN.AND","BOOLEAN.DEFINE","BOOLEAN.DUP","BOOLEAN.FLUSH","BOOLEAN.FROMFLOAT","BOOLEAN.FROMINTEGER","BOOLEAN.NOT","BOOLEAN.OR","BOOLEAN.POP","BOOLEAN.RAND","BOOLEAN.ROT","BOOLEAN.SHOVE","BOOLEAN.STACKDEPTH","BOOLEAN.SWAP","BOOLEAN.YANK","BOOLEAN.YANKDUP","CODE.=","CODE.ATOM","CODE.CAR","CODE.CDR","CODE.CONS","CODE.CONTAINS","CODE.DEFINE","CODE.DO","CODE.DO*","CODE.DO*COUNT","CODE.DO*RANGE","CODE.DO*TIMES","CODE.DUP","CODE.FLUSH","CODE.IF","CODE.LENGTH","CODE.LIST","CODE.NOOP","CODE.NTH","CODE.NULL","CODE.SHOVE","CODE.POP","CODE.QUOTE","CODE.ROT","CODE.SHOVE","CODE.STACKDEPTH","CODE.SWAP","CODE.YANK","CODE.YANKDUP","EXEC.=","EXEC.DEFINE","EXEC.DO*COUNT","EXEC.DO*RANGE","EXEC.DO*TIMES","EXEC.DUP","EXEC.FLUSH","EXEC.IF","EXEC.K","EXEC.POP","EXEC.ROT","EXEC.S","EXEC.SHOVE","EXEC.STACKDEPTH","EXEC.SWAP","EXEC.Y","EXEC.YANK","EXEC.YANKDUP"/*,"NAME.=","NAME.DUP","NAME.FLUSH","NAME.POP","NAME.RAND","NAME.ROT","NAME.SHOVE","NAME.STACKDEPTH","NAME.SWAP","NAME.YANK","NAME.YANKDUP"];*/
+// "INTEGER.+","INTEGER.-","INTEGER./","INTEGER.*","INTEGER.%","INTEGER.>","INTEGER.<","INTEGER.=","INTEGER.DUP","INTEGER.FLUSH","INTEGER.MAX","INTEGER.MIN","INTEGER.POP","INTEGER.RAND","INTEGER.ROT","INTEGER.SHOVE","INTEGER.STACKDEPTH","INTEGER.SWAP","INTEGER.YANK","INTEGER.YANKDUP",
+
function testGP() {
startGP(function(){});
}
@@ -16,7 +16,8 @@ var gp_params = {
'startPopulation' : {{ gp_params.start_population }},
'tournamentSize' : {{ gp_params.tournament_size }},
'maxPopulation' : {{ gp_params.max_population }},
- 'stagnantGenerations' : 0,
- 'numProgramsToDownload' : 2
+ 'stagnantGenerations' : {{ gp_params.stagnant_generations }},
+ 'numProgramsToDownload' : {{ gp_params.num_programs_to_download }}
};
+
var pushScriptFuncs = {{ gp_params.allowed }};
@@ -18,7 +18,7 @@
function()
{
$('#gpChart').hide();
- startGPBrain({{ worker_id }},"{{ client_id }}");
+ startGPBrain("{{ client_id }}");
});

0 comments on commit 913e227

Please sign in to comment.