Permalink
Browse files

separate 1.0 and 2.0

git-svn-id: https://svn.broadinstitute.org/CellProfiler/trunk/CellProfiler@9407 1fc53939-2000-0410-845c-e8453a809027
  • Loading branch information...
1 parent 2684a85 commit 7da533013c6484dac00381d711b59a5f3bce3905 thouis committed Mar 1, 2010
Showing 747 changed files with 2,928 additions and 81,287 deletions.
View
28 AUTHORS
@@ -1,28 +0,0 @@
-CellProfiler was created by Anne Carpenter & Thouis Jones, at the
-Whitehead Institute for Biomedical Research. It is currently hosted
-and developed at the Broad Institute of MIT and Harvard.
-
-CellProfiler Developers:
- Anne Carpenter
- Thouis Jones
- David Logan
- Martha Vokes
-
-Other contributors (in alphabetical order):
- Paolo Arcaini
- Joo Han Chang
- Colin Clarke
- Ola Friman
- Chris Gang
- Rodrigo Ipince
- In Han Kang
- Kyungnam Kim
- Mike Lamprecht
- Vicky Lay
- Jun Liu
- Steve Lowe
- Joaquim Luis
- Susan Ma
- Massimo Manara
- Adam M Papallo
- Peter Swire
File renamed without changes.
@@ -1,51 +0,0 @@
-#!/usr/bin/env /imaging/analysis/People/imageweb/batchprofiler/cgi-bin/python-2.6.sh
-#
-# View a batch from the database, with some options to re-execute it
-#
-import cgitb
-cgitb.enable()
-import RunBatch
-import StyleSheet
-import cgi
-import os
-import os.path
-
-def remove_if_exists(path):
- if os.path.exists(path):
- os.remove(path)
-
-form = cgi.FieldStorage()
-delete_action = form["delete_action"].value
-if form.has_key("run_id"):
- run_id = int(form["run_id"].value)
- my_batch,my_run = RunBatch.LoadRun(run_id)
- if ((delete_action.upper() == "ALL") or (delete_action.upper() == "TEXT")):
- remove_if_exists(RunBatch.RunTextFilePath(my_batch, my_run))
-
- if (delete_action.upper() == "ALL") or (delete_action.upper() == "OUTPUT"):
- remove_if_exists(RunBatch.RunOutFilePath(my_batch, my_run))
-
- if (delete_action.upper() == "ALL") or (delete_action.upper() == "DONE"):
- remove_if_exists(RunBatch.RunDoneFilePath(my_batch, my_run))
-elif form.has_key("batch_id"):
- batch_id = int(form["batch_id"].value)
- my_batch = RunBatch.LoadBatch(batch_id)
- for my_run in my_batch["runs"]:
- if ((delete_action.upper() == "ALL") or (delete_action.upper() == "TEXT")):
- remove_if_exists(RunBatch.RunTextFilePath(my_batch, my_run))
-
- if (delete_action.upper() == "ALL") or (delete_action.upper() == "OUTPUT"):
- remove_if_exists(RunBatch.RunOutFilePath(my_batch, my_run))
-
- if (delete_action.upper() == "ALL") or (delete_action.upper() == "DONE"):
- remove_if_exists(RunBatch.RunDoneFilePath(my_batch, my_run))
-
-
-url = "ViewBatch.py?batch_id=%(batch_id)d"%(my_batch)
-print "Content-Type: text/html"
-print
-print "<html><head>"
-print "<meta http-equiv='refresh' content='0; URL=%(url)s' />"%(globals())
-print "</head>"
-print "<body>This page should be redirected to <a href='%(url)s'/>%(url)s</a></body>"%(globals())
-print "</html>"
View
@@ -1,67 +0,0 @@
-#!/usr/bin/env /imaging/analysis/People/imageweb/batchprofiler/cgi-bin/python-2.6.sh
-#
-# Kill all jobs in a batch
-#
-print "Content-Type: text/html"
-print
-import cgitb
-cgitb.enable()
-import RunBatch
-import cgi
-import subprocess
-import sys
-
-form = cgi.FieldStorage()
-if form.has_key("job_id"):
- import subprocess
- job_id = int(form["job_id"].value)
- run = {"job_id":job_id}
- RunBatch.KillOne(run)
- print"""
- <html><head><title>Job %(job_id)d killed</title></head>
- <body>Job %(job_id)d killed
- </body>
- </html>
-"""%(globals())
-elif form.has_key("batch_id"):
- batch_id = int(form["batch_id"].value)
- my_batch = RunBatch.LoadBatch(batch_id)
- for run in my_batch["runs"]:
- RunBatch.KillOne(run)
-
- url = "ViewBatch.py?batch_id=%(batch_id)d"%(my_batch)
- print "<html><head>"
- print "<meta http-equiv='refresh' content='0; URL=%(url)s' />"%(globals())
- print "</head>"
- print "<body>This page should be redirected to <a href='%(url)s'/>%(url)s</a></body>"%(globals())
- print "</html>"
-else:
- print """<html><head><title>Kill jobs</title></head>
- <body>
- <h1>Kill jobs started by the imageweb webserver</h1>
- <form action='KillJobs.py' method='POST'>
- Job ID:<input type='text' name='job_id' />
- <input type='submit' value='Kill'/>
- </form>
- """
- p = subprocess.Popen(["bash"],stdin = subprocess.PIPE,
- stdout=subprocess.PIPE)
- listing = p.communicate(". /broad/lsf/conf/profile.lsf;bjobs\n")[0]
- listing_lines = listing.split('\n')
- header = listing_lines[0]
- columns = [header.find(x) for x in header.split(' ') if len(x)]
- columns.append(1000)
- body = listing_lines[1:]
- print """
- <h2>Jobs on imageweb</h2>
- <table>
- """
- print "<tr>%s</tr>"%("".join(['<th>%s</th>'%(header[columns[i]:columns[i+1]])
- for i in range(len(columns)-1)]))
- for line in body:
- print "<tr>%s</tr>"%("".join(['<td>%s</td>'%(line[columns[i]:columns[i+1]])
- for i in range(len(columns)-1)]))
- """
- </table>
- </body>
- """
View
@@ -1,208 +0,0 @@
-#!/usr/bin/env /imaging/analysis/People/imageweb/batchprofiler/cgi-bin/python-2.6.sh
-# This is a CGI script that begins to start a batch on the clusters
-# Functionality:
-# Parse the posted fields:
-# email
-# data_dir
-# queue
-# batch_size:
-# write_data: "y" or "n"
-# timeout
-# Create a batch run # and store the details in the DB
-# Send the user an email with the URL of the job
-# Start BatchRunner.py in the background with the supplied info
-import cgitb
-cgitb.enable()
-print "Content-Type: text/html\r"
-print "\r"
-import sys
-import cgi
-import os
-import re
-import socket
-import smtplib
-import email.message
-import email.mime.text
-import subprocess
-import RunBatch
-import traceback
-from scipy.io.matlab.mio import mat_reader_factory, loadmat
-
-# # # # # # # # # #
-#
-# Constants
-#
-# # # # # # # # # #
-
-batch_url = "http://%s/batchprofiler/cgi-bin/ViewBatch.py"%(socket.gethostname())
-SENDMAIL="/usr/sbin/sendmail"
-
-def PartitionBatches(my_batch):
- """Partition the image sets in the batch_info into runs
-
- Returns an array of runs where each run is a dictionary composed of
- start = index of first image set in run
- end = index of last image set in run
- status_file_name = where the status file will be stored
- """
- result=[]
- dd=my_batch["data_dir"]
- for start in range(1, my_batch["num_sets"] + 1, my_batch["batch_size"]):
- end=start+my_batch["batch_size"]-1
- status_file_name = "%(dd)s/status/Batch_%(start)d_to_%(end)d_DONE.mat"%(locals())
- result.append({
- "start":start,
- "end":min(start+my_batch["batch_size"]-1,my_batch["num_sets"]),
- "status_file_name":status_file_name})
-
- my_batch["runs"]=result
- return result
-
-def CheckParameters(my_batch):
- """Check the parameters of the batch for consistency and correctness
-
- Make sure that the data directory is writeable.
- Make sure that txt_output and status are writeable
- Make sure that the cpcluster directory exists
- """
- if not os.path.exists(my_batch["data_dir"]):
- return 'The data directory, %(data_dir)s, must exist'%(my_batch)
- if not os.path.exists(my_batch["batch_file"]):
- return 'The batch file, "%(batch_file)s", must exist'%(my_batch)
- if not os.access(my_batch["batch_file"],os.R_OK):
- return 'The batch file, "%(batch_file)s", must be readable'%(my_batch)
- txt_output = os.sep.join(['%(data_dir)s'%(my_batch),'txt_output'])
- status = os.sep.join(['%(data_dir)s'%(my_batch),'status'])
- writeable= {
- "data directory":my_batch["data_dir"],
- "CPCluster directory":my_batch["cpcluster"]
- }
- if os.path.exists(txt_output):
- writeable["command output"]=txt_output
- text_output_files=os.listdir(txt_output)
- for tof in text_output_files:
- match = re.match("^([0-9]+)_to_([0-9]+)\\.txt$",tof)
- if match:
- (start,last)=match.groups()
- writeable["image set %s to %s command output file"%(start,last)]=os.sep.join([txt_output,tof])
- if os.path.exists(status):
- writeable["Matlab output"]=status
- status_files=os.listdir(status)
- for tof in status_files:
- match = re.match("^Batch_([0-9]+)_to_([0-9]+)_DONE\\.txt$",tof)
- if match:
- (start,last)=match.groups()
- writeable["image set %d to %d Matlab status file"%(start,last)]=os.sep.join([status,tof])
- match = re.match("^Batch_([0-9]+)_to_([0-9]+)_OUT\\.txt$",tof)
- if match:
- (start,last)=match.groups()
- writeable["image set %s to %s Matlab data file"%(start,last)]=os.sep.join([status,tof])
-
- for dir_key in writeable:
- if not os.access(writeable[dir_key],os.W_OK):
- return 'The %s, "%s", must be writeable'%(dir_key,writeable[dir_key])
- if not os.path.exists(txt_output):
- os.mkdir(txt_output)
- os.chmod(txt_output,0777)
- if not os.path.exists(status):
- os.mkdir(status)
- os.chmod(status,0777)
-
-def SendMail(recipient,body):
- if os.name != 'nt':
- pipe=os.popen("%s -t"%(SENDMAIL),"w")
- pipe.write("To: %s\n"%(recipient))
- pipe.write("Subject: Batch %d submitted\n"%(batch_id))
- pipe.write("Content-Type: text/html\n")
- pipe.write("\n")
- pipe.write(body)
- pipe.write("\n")
- pipe.close()
-
- return
-
-# # # # # # # # # # #
-#
-# Main script
-#
-# # # # # # # # # # #
-
-form_data=cgi.FieldStorage()
-if form_data.has_key("data_dir"):
- try:
- if not form_data.has_key("email"):
- raise RuntimeError("The e-mail field isn't filled in.")
- imaging_analysis=(os.name=='nt' and "//iodine/imaging_analysis") or "/imaging/analysis"
- batch_file="%s/Batch_data.mat"%(form_data["data_dir"].value)
- batch_file=batch_file.replace("/imaging/analysis",imaging_analysis)
- batch_file=batch_file.replace("/",os.sep)
- CPCluster='/imaging/analysis/CPCluster/%s'%(form_data["cpcluster"].value)
- my_batch = {
- "email": form_data["email"].value,
- "queue": form_data["queue"].value,
- "project": form_data["project"].value if form_data.has_key("project") else 'imaging',
- "data_dir": form_data["data_dir"].value,
- "write_data": (form_data["write_data"].value.upper()=="Y" and 1) or 0,
- "batch_size": int(form_data["batch_size"].value),
- "memory_limit": float(form_data["memory_limit"].value) if form_data.has_key("memory_limt") else 2000,
- "timeout": float(form_data["timeout"].value),
- "cpcluster": CPCluster,
- "batch_file": batch_file
- }
- error = CheckParameters(my_batch)
- if error:
- exception = RuntimeError()
- exception.message = error
- raise exception
- #
- # This is somewhat dependent on internals, but we
- mrf = mat_reader_factory(batch_file,struct_as_record=True)
- batch_info = mrf.get_variables(['number_of_image_sets'])
- if batch_info.has_key('number_of_image_sets'):
- my_batch["num_sets"] = batch_info['number_of_image_sets'][0,0]
- else:
- batch_info=loadmat(batch_file, struct_as_record=True)
- my_batch["num_sets"] = batch_info['handles']['Current'][0,0]['NumberOfImageSets'][0,0][0,0]
- runs = PartitionBatches(my_batch)
- batch_id = RunBatch.CreateBatchRun(my_batch)
- results = RunBatch.RunAll(batch_id)
- text=[]
- text.append("<html>")
- text.append("<head><title>Batch # %d</title>"%(batch_id))
- text.append("<style type='text/css'>")
- text.append("""
- table {
- border-spacing: 0px;
- border-collapse: collapse;
- }
- td {
- text-align: left;
- vertical-align: baseline;
- padding: 0.1em 0.5em;
- border: 1px solid #666666;
- }
- """)
- text.append("</style></head>")
- text.append("</head>")
- text.append("<body>")
- text.append("<h1>Results for batch # <a href='%s?batch_id=%d'>%d</a></h1>"%(batch_url,batch_id,batch_id))
- text.append("<table>")
- text.append("<th><tr><td>First image set</td><td>Last image set</td><td>job #</td></tr></th>")
- for result in results:
- text.append("<tr><td>%(start)d</td><td>%(end)d</td><td>%(job)d</td></tr>"%(result))
- text.append("</table>")
- text.append("</body>")
- text.append("</html>")
- body= '\n'.join(text)
- SendMail(my_batch["email"],body)
- print body
- except RuntimeError,e:
- print "<html><body>"
- print "<h1 style='color:red'>Unable to process job</h1>"
- print e.message
- print "</body></html>"
-else:
- print "<html>"
- print "<head><link href='/RunNewBatch.html' rel='Alternate' /></head>"
- print "</html>"
-
Oops, something went wrong.

0 comments on commit 7da5330

Please sign in to comment.