From d7d576af8f3c95b996edccc78873ddedfdcdd0a7 Mon Sep 17 00:00:00 2001 From: Joshua Shorenstein Date: Wed, 18 Jun 2014 18:50:04 -0600 Subject: [PATCH 1/9] add jobs to analysis, create wait page --- qiita_pet/handlers/analysis_handlers.py | 48 ++++++++++++++++--- qiita_pet/handlers/websocket_handlers.py | 37 +++++--------- .../{waiting.html => analysis_waiting.html} | 28 +++-------- qiita_pet/templates/select_commands.html | 3 +- qiita_pet/webserver.py | 7 ++- 5 files changed, 66 insertions(+), 57 deletions(-) rename qiita_pet/templates/{waiting.html => analysis_waiting.html} (62%) diff --git a/qiita_pet/handlers/analysis_handlers.py b/qiita_pet/handlers/analysis_handlers.py index b59422d1d..187c25fb3 100644 --- a/qiita_pet/handlers/analysis_handlers.py +++ b/qiita_pet/handlers/analysis_handlers.py @@ -7,6 +7,7 @@ from qiita_db.study import Study from qiita_db.data import ProcessedData from qiita_db.metadata_template import SampleTemplate +from qiita_db.job import Job # login code modified from https://gist.github.com/guillaumevincent/4771570 @@ -45,7 +46,6 @@ def post(self): study_args = self.get_arguments('studies') split = [x.split("#") for x in study_args] - # build dictionary of studies and datatypes selected # as well a set of unique datatypes selected study_dts = defaultdict(list) @@ -59,11 +59,11 @@ def post(self): data_types.sort() # FIXME: Pull out from the database!! - commands = {'16S' : ['Alpha Diversity', 'Beta Diversity', - 'Summarize Taxa'], - '18S' : ['Alpha Diversity', 'Beta Diversity', - 'Summarize Taxa'], - 'Metabolomic' : ['Summarize Taxa']} + commands = {'16S': ['Alpha Diversity', 'Beta Diversity', + 'Summarize Taxa'], + '18S': ['Alpha Diversity', 'Beta Diversity', + 'Summarize Taxa'], + 'Metabolomic': ['Summarize Taxa']} self.render('select_commands.html', user=self.get_current_user(), commands=commands, data_types=data_types, aid=analysis_id) @@ -81,3 +81,39 @@ def post(self): sample_ids] analysis.add_samples(samples) + +class AnalysisWaitHandler(BaseHandler): + def get(self, analysis_id): + analysis = Analysis(analysis_id) + commands = [] + for job in analysis.jobs: + jobject = Job(job) + commands.append("%s:%s" % (jobject.data_type, jobject.command[0])) + + self.render("analysis_waiting.html", user=self.get_current_user(), + aid=analysis_id, aname=analysis.name, + commands=commands) + + def post(self, analysis_id): + command_args = self.get_arguments("commands") + split = [x.split("#") for x in command_args] + analysis = Analysis(analysis_id) + + jobs = [] + commands = [] + for data_type, command in split: + jobs.append(Job.create(data_type, command, {}, analysis)) + commands.append("%s:%s" % (data_type, command)) + + self.render("analysis_waiting.html", user=self.get_current_user(), + aid=analysis_id, aname=analysis.name, + commands=commands) + # fire off analysis run here + + +class AnalysisResultsHandler(BaseHandler): + def get(self, aid): + pass + + def post(self, ignore): + pass diff --git a/qiita_pet/handlers/websocket_handlers.py b/qiita_pet/handlers/websocket_handlers.py index cba1be61e..05c489efb 100644 --- a/qiita_pet/handlers/websocket_handlers.py +++ b/qiita_pet/handlers/websocket_handlers.py @@ -1,25 +1,22 @@ # adapted from # https://github.com/leporo/tornado-redis/blob/master/demos/websockets +from time import sleep -from redis import Redis -from tornadoredis import Client from tornado.websocket import WebSocketHandler -import tornado.gen +from tornado.gen import engine, Task from json import loads # all messages are in json format. They must have the following format: -# 'job': jobname +# 'analysis': analysis_id # 'msg': message to print -# 'analysis': what analysis this is from in format datatype:analysis -# 'results': list of files created if any +# 'command': what command this is from in format datatype#command class MessageHandler(WebSocketHandler): def __init__(self, *args, **kwargs): super(MessageHandler, self).__init__(*args, **kwargs) - self.r_server = Redis() - self.redis = Client() - self.redis.connect() + # self.redis = Client() + # self.redis.connect() def get_current_user(self): user = self.get_secure_cookie("user") @@ -37,28 +34,16 @@ def on_message(self, msg): self.listen() # decorator turns the function into an asynchronous generator object - @tornado.gen.engine + @engine def listen(self): - # runs task given, with the yield required to get returned value - # equivalent of callback/wait pairing from tornado.gen - yield tornado.gen.Task(self.redis.subscribe, self.channel) - if not self.redis.subscribed: - self.write_message('ERROR IN SUBSCRIPTION') - # listen from tornadoredis makes the listen object asynchronous - # if using standard redis lib, it blocks while listening - self.redis.listen(self.callback) - # fight race condition by loading from redis after listen started - # need to use std redis lib because tornadoredis is in subscribed state - oldmessages = self.r_server.lrange(self.channel + ':messages', 0, -1) - if oldmessages is not None: - for message in oldmessages: - self.write_message(message) + sleep(5) + self.write_message({"msg": "allcomplete"}) def callback(self, msg): if msg.kind == 'message': self.write_message(str(msg.body)) - @tornado.gen.engine + @engine def on_close(self): - yield tornado.gen.Task(self.redis.unsubscribe, self.channel) + yield Task(self.redis.unsubscribe, self.channel) self.redis.disconnect() \ No newline at end of file diff --git a/qiita_pet/templates/waiting.html b/qiita_pet/templates/analysis_waiting.html similarity index 62% rename from qiita_pet/templates/waiting.html rename to qiita_pet/templates/analysis_waiting.html index d7fe66910..d4fcb99ee 100644 --- a/qiita_pet/templates/waiting.html +++ b/qiita_pet/templates/analysis_waiting.html @@ -15,9 +15,9 @@ websocket.onmessage = function(evt) { console.log(evt.data); message = JSON.parse(evt.data); - if(message.job == '{{job}}') { + if(message.analysis == '{{aid}}') { if(noerror && message.msg == 'allcomplete') { - window.location.replace("/completed/{{job}}"); + window.location.replace("/completed/{{aid}}"); } list = document.getElementById(message.analysis); if(message.msg.indexOf("ERROR") != -1) { @@ -30,11 +30,7 @@ else { list.style.color="Blue"; } - list.innerHTML = message.msg + '
'; - for(var i=0; iPlot ' + (i+1) + '
'; - } + list.innerHTML = message.msg; } }; websocket.onerror = function(evt) { }; @@ -52,24 +48,12 @@ {% end %} {% block content %} - - - -
- 1) Choose studies
- 2) Select analyses
- 3) Set analysis options
- 4) Review analysis
- 5) Running analysis
- 6) Analysis results
- 		-

Analysis {{job}}

- {% for analysis in analyses %} +

Analysis {{aname}}

+ {% for command in commands%}

- {{analysis}}: Queued + {{command}}: Queued

{% end %} -
{% end %} \ No newline at end of file diff --git a/qiita_pet/templates/select_commands.html b/qiita_pet/templates/select_commands.html index d7150923c..7aa0e727d 100644 --- a/qiita_pet/templates/select_commands.html +++ b/qiita_pet/templates/select_commands.html @@ -19,8 +19,7 @@

Select Commands

{% end %} -
- +
{% for data_type in data_types %}
diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index c77d36015..0b4f785d2 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -17,7 +17,9 @@ from qiita_pet.handlers.auth_handlers import ( AuthCreateHandler, AuthLoginHandler, AuthLogoutHandler, AuthVerifyHandler) from qiita_pet.handlers.analysis_handlers import ( - CreateAnalysisHandler, SelectStudiesHandler, SelectCommandsHandler) + CreateAnalysisHandler, SelectStudiesHandler, SelectCommandsHandler, + AnalysisWaitHandler, AnalysisResultsHandler) +from qiita_pet.handlers.websocket_handlers import MessageHandler define("port", default=8888, help="run on the given port", type=int) @@ -43,6 +45,9 @@ def __init__(self): (r"/analysis/1", CreateAnalysisHandler), (r"/analysis/2", SelectStudiesHandler), (r"/analysis/3", SelectCommandsHandler), + (r"/analysis/wait/(.*)", AnalysisWaitHandler), + (r"/analysis/results/(.*)", AnalysisResultsHandler), + (r"/consumer/", MessageHandler), (r"/mockup/", MockupHandler), # 404 PAGE MUST BE LAST IN THIS LIST! (r".*", NoPageHandler) From 42b36c04e070532cea14bb828a7ec243ce56baa3 Mon Sep 17 00:00:00 2001 From: Joshua Shorenstein Date: Thu, 19 Jun 2014 08:47:13 -0600 Subject: [PATCH 2/9] waiting page with websockeets added --- qiita_db/job.py | 8 ++++---- qiita_pet/handlers/analysis_handlers.py | 8 ++++++-- qiita_pet/handlers/websocket_handlers.py | 5 +++-- qiita_pet/templates/analysis_waiting.html | 6 +++--- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/qiita_db/job.py b/qiita_db/job.py index 974c0616c..358bd0f0d 100644 --- a/qiita_db/job.py +++ b/qiita_db/job.py @@ -97,10 +97,10 @@ def create(cls, datatype, command, options, analysis): Job object The newly created job """ - if cls.exists(datatype, command, options): - raise QiitaDBDuplicateError( - "Job", "datatype: %s, command: %s, options: %s" - % (datatype, command, options)) + # if cls.exists(datatype, command, options): + # raise QiitaDBDuplicateError( + # "Job", "datatype: %s, command: %s, options: %s" + # % (datatype, command, options)) # Get the datatype and command ids from the strings conn_handler = SQLConnectionHandler() diff --git a/qiita_pet/handlers/analysis_handlers.py b/qiita_pet/handlers/analysis_handlers.py index 187c25fb3..94d42f71d 100644 --- a/qiita_pet/handlers/analysis_handlers.py +++ b/qiita_pet/handlers/analysis_handlers.py @@ -83,17 +83,19 @@ def post(self): class AnalysisWaitHandler(BaseHandler): + @authenticated def get(self, analysis_id): analysis = Analysis(analysis_id) commands = [] for job in analysis.jobs: jobject = Job(job) - commands.append("%s:%s" % (jobject.data_type, jobject.command[0])) + commands.append("%s:%s" % (jobject.datatype, jobject.command[0])) self.render("analysis_waiting.html", user=self.get_current_user(), aid=analysis_id, aname=analysis.name, commands=commands) + @authenticated def post(self, analysis_id): command_args = self.get_arguments("commands") split = [x.split("#") for x in command_args] @@ -112,8 +114,10 @@ def post(self, analysis_id): class AnalysisResultsHandler(BaseHandler): + @authenticated def get(self, aid): - pass + self.render("analysis_results.html", user=self.get_current_user()) + @authenticated def post(self, ignore): pass diff --git a/qiita_pet/handlers/websocket_handlers.py b/qiita_pet/handlers/websocket_handlers.py index 05c489efb..b34311406 100644 --- a/qiita_pet/handlers/websocket_handlers.py +++ b/qiita_pet/handlers/websocket_handlers.py @@ -29,7 +29,8 @@ def on_message(self, msg): msginfo = loads(msg) # listens for handshake from page if "user:" in msginfo['msg']: - self.channel = msginfo['msg'].split(':')[1] + self.aid = msginfo['msg'].split()[0] + self.channel = msginfo['msg'].split()[1].split(':')[1] # need to split the rest off to new func so it can be asynchronous self.listen() @@ -37,7 +38,7 @@ def on_message(self, msg): @engine def listen(self): sleep(5) - self.write_message({"msg": "allcomplete"}) + self.write_message({"analysis": self.aid, "msg": "allcomplete"}) def callback(self, msg): if msg.kind == 'message': diff --git a/qiita_pet/templates/analysis_waiting.html b/qiita_pet/templates/analysis_waiting.html index d4fcb99ee..acbdffa9f 100644 --- a/qiita_pet/templates/analysis_waiting.html +++ b/qiita_pet/templates/analysis_waiting.html @@ -9,15 +9,15 @@ noerror = true; websocket.onopen = function() { - websocket.send(JSON.stringify({'msg': 'user:{{user}}'})); - console.log(JSON.stringify({'msg': 'user:{{user}}'})); + websocket.send(JSON.stringify({'msg': '{{aid}} user:{{user}}'})); + console.log(JSON.stringify({'msg':'{{aid}} user:{{user}}'})); }; websocket.onmessage = function(evt) { console.log(evt.data); message = JSON.parse(evt.data); if(message.analysis == '{{aid}}') { if(noerror && message.msg == 'allcomplete') { - window.location.replace("/completed/{{aid}}"); + window.location.replace("/analysis/results/{{aid}}"); } list = document.getElementById(message.analysis); if(message.msg.indexOf("ERROR") != -1) { From f8d32fd6cd60e3d54d8186b7e70a73c608063d36 Mon Sep 17 00:00:00 2001 From: Joshua Shorenstein Date: Thu, 19 Jun 2014 11:27:27 -0600 Subject: [PATCH 3/9] add results page --- qiita_pet/handlers/analysis_handlers.py | 23 +++++++++++++++++------ qiita_pet/templates/analysis_results.html | 18 +++++++++++++++++- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/qiita_pet/handlers/analysis_handlers.py b/qiita_pet/handlers/analysis_handlers.py index 03d8f5ee7..44b0733ff 100644 --- a/qiita_pet/handlers/analysis_handlers.py +++ b/qiita_pet/handlers/analysis_handlers.py @@ -1,3 +1,5 @@ +from os.path import basename + from tornado.web import authenticated from collections import defaultdict @@ -100,10 +102,12 @@ def post(self, analysis_id): split = [x.split("#") for x in command_args] analysis = Analysis(analysis_id) - jobs = [] commands = [] for data_type, command in split: - jobs.append(Job.create(data_type, command, {}, analysis)) + print "INPUT: ", data_type, command + job = Job.create(data_type, command, {}, analysis) + print "INPUT2: ", data_type, command + print "JOB %s %s:%s" % (job.id, job.datatype, job.command[0]) commands.append("%s:%s" % (data_type, command)) self.render("analysis_waiting.html", user=self.get_current_user(), @@ -115,8 +119,15 @@ def post(self, analysis_id): class AnalysisResultsHandler(BaseHandler): @authenticated def get(self, aid): - self.render("analysis_results.html", user=self.get_current_user()) + analysis = Analysis(aid) + jobres = {} + for job in analysis.jobs: + jobject = Job(job) + print "JOB %s %s:%s" % (job, jobject.datatype, jobject.command[0]) + jobres["%s:%s" % (jobject.datatype, + jobject.command[0])] = jobject.results - @authenticated - def post(self, ignore): - pass + print jobres + + self.render("analysis_results.html", user=self.get_current_user(), + jobres=jobres, aname=analysis.name) diff --git a/qiita_pet/templates/analysis_results.html b/qiita_pet/templates/analysis_results.html index 6f1691ee9..b402bf071 100644 --- a/qiita_pet/templates/analysis_results.html +++ b/qiita_pet/templates/analysis_results.html @@ -1,5 +1,21 @@ {% extends sitebase.html%} {%block content%} -RESULTS +
+

Analysis {{aname}}

+
+
+
+{%for job in jobres %} + {{job}}

+ {% for num, res in enumerate(jobres[job]) %} + Result {{num}}
+ {% end %} +

+{% end %} +
+
+ +
+
{%end%} \ No newline at end of file From e190988693cb9c59a7e7ea3ae04369407c9d311d Mon Sep 17 00:00:00 2001 From: Joshua Shorenstein Date: Thu, 19 Jun 2014 12:45:39 -0600 Subject: [PATCH 4/9] add results page --- qiita_pet/handlers/analysis_handlers.py | 9 ++--- qiita_pet/static/css/style.css | 1 + qiita_pet/templates/analysis_results.html | 42 ++++++++++++++++------- qiita_pet/templates/sitebase.html | 2 +- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/qiita_pet/handlers/analysis_handlers.py b/qiita_pet/handlers/analysis_handlers.py index 44b0733ff..27614a574 100644 --- a/qiita_pet/handlers/analysis_handlers.py +++ b/qiita_pet/handlers/analysis_handlers.py @@ -120,14 +120,11 @@ class AnalysisResultsHandler(BaseHandler): @authenticated def get(self, aid): analysis = Analysis(aid) - jobres = {} + jobres = defaultdict(list) for job in analysis.jobs: jobject = Job(job) - print "JOB %s %s:%s" % (job, jobject.datatype, jobject.command[0]) - jobres["%s:%s" % (jobject.datatype, - jobject.command[0])] = jobject.results - - print jobres + jobres[jobject.datatype].append((jobject.command[0], + jobject.results)) self.render("analysis_results.html", user=self.get_current_user(), jobres=jobres, aname=analysis.name) diff --git a/qiita_pet/static/css/style.css b/qiita_pet/static/css/style.css index fda1d72c9..797bcaf77 100644 --- a/qiita_pet/static/css/style.css +++ b/qiita_pet/static/css/style.css @@ -1,4 +1,5 @@ #template-content{ padding: 20px; + height: 100%; } diff --git a/qiita_pet/templates/analysis_results.html b/qiita_pet/templates/analysis_results.html index b402bf071..45aad5578 100644 --- a/qiita_pet/templates/analysis_results.html +++ b/qiita_pet/templates/analysis_results.html @@ -1,21 +1,37 @@ {% extends sitebase.html%} +{%block head%} + + +{%end%} + {%block content%} +

Analysis {{aname}}

-
-
-{%for job in jobres %} - {{job}}

- {% for num, res in enumerate(jobres[job]) %} - Result {{num}}
- {% end %} -

-{% end %} -
-
- -
+
+
+
+ {% for data_type, jobs in jobres.items() %} +
+
+

{{data_type}}

+
+
+ {% for job, result in jobs%} +
+ {{job}} +
+ {% end %} +
+
+ {% end %} +
+
+
+ +
+
{%end%} \ No newline at end of file diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 5ddfd448d..119305eb2 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -13,7 +13,7 @@ {% block head %}{% end %} - +