Skip to content

Commit

Permalink
Merge pull request #205 from squirrelo/track-analysis-table
Browse files Browse the repository at this point in the history 
Track analysis table
  • Loading branch information
@antgonza
antgonza committed Aug 12, 2014
2 parents 313db0c + 96a2040 commit 95fa935
Show file tree
Hide file tree
Showing 5 changed files with 175 additions and 15 deletions.
48 changes: 46 additions & 2 deletions qiita_db/analysis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ class Analysis(QiitaStatusObject):
samples
data_types
biom_tables
step
shared_with
jobs
pmid
Expand All @@ -52,13 +53,15 @@ class Analysis(QiitaStatusObject):
add_jobs
share
unshare
finish_workflow
"""

_table = "analysis"

def _lock_check(self, conn_handler):
"""Raises QiitaDBStatusError if analysis is public"""
if self.check_status({"public", "completed", "error"}):
"""Raises QiitaDBStatusError if analysis is not in_progress"""
if self.check_status({"public", "completed", "error", "running",
"queued"}):
raise QiitaDBStatusError("Analysis is locked!")

def _status_setter_checks(self, conn_handler):
Expand Down Expand Up @@ -237,6 +240,32 @@ def biom_tables(self):
return None
return [table[0] for table in tables]

@property
def step(self):
conn_handler = SQLConnectionHandler()
self._lock_check(conn_handler)
sql = "SELECT step from qiita.analysis_workflow WHERE analysis_id = %s"
try:
return conn_handler.execute_fetchone(sql, (self._id,))[0]
except TypeError:
raise ValueError("Step not set yet!")

@step.setter
def step(self, value):
conn_handler = SQLConnectionHandler()
self._lock_check(conn_handler)
sql = ("SELECT EXISTS(SELECT analysis_id from qiita.analysis_workflow "
"WHERE analysis_id = %s)")
step_exists = conn_handler.execute_fetchone(sql, (self._id,))[0]

if step_exists:
sql = ("UPDATE qiita.analysis_workflow SET step = %s WHERE "
"analysis_id = %s")
else:
sql = ("INSERT INTO qiita.analysis_workflow (step, analysis_id) "
"VALUES (%s, %s)")
conn_handler.execute(sql, (value, self._id))

@property
def jobs(self):
"""A list of jobs included in the analysis
Expand Down Expand Up @@ -440,3 +469,18 @@ def add_jobs(self, jobs):
sql = ("INSERT INTO qiita.analysis_job (analysis_id, job_id) "
"VALUES (%s, %s)")
conn_handler.executemany(sql, [(self._id, job.id) for job in jobs])

def finish_workflow(self):
"""Do database updates required before running analysis
Notes
-----
Removes analysis from qiita.analysis_workflow table
Set status to queued
"""
conn_handler = SQLConnectionHandler()
self._lock_check(conn_handler)
sql = "DELETE FROM qiita.analysis_workflow WHERE analysis_id = %s"
conn_handler.execute(sql, (self._id,))

self.status = "queued"
13 changes: 13 additions & 0 deletions qiita_db/support_files/qiita-db.dbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,17 @@ If a given analysis is not in child_id, it is the root of the chain. </comment>
<fk_column name="email" pk="email" />
</fk>
</table>
<table name="analysis_workflow" >
<comment>Stores what step in_production analyses are on.</comment>
<column name="analysis_id" type="bigint" jt="-5" mandatory="y" />
<column name="step" type="integer" jt="4" mandatory="y" />
<index name="pk_analysis_workflow" unique="PRIMARY_KEY" >
<column name="analysis_id" />
</index>
<fk name="fk_analysis_workflow" to_schema="qiita" to_table="analysis" >
<fk_column name="analysis_id" pk="analysis_id" />
</fk>
</table>
<table name="annotation" >
<column name="annotation_id" type="bigserial" jt="-5" mandatory="y" />
<column name="term_id" type="bigint" jt="-5" mandatory="y" />
Expand Down Expand Up @@ -1330,6 +1341,7 @@ Controlled Vocabulary]]></comment>
<entity schema="qiita" name="preprocessed_data" color="c0d4f3" x="1200" y="690" />
<entity schema="qiita" name="processed_data" color="d0def5" x="1215" y="930" />
<entity schema="qiita" name="preprocessed_filepath" color="c0d4f3" x="990" y="690" />
<entity schema="qiita" name="analysis_workflow" color="c0d4f3" x="390" y="630" />
<group name="Group_analyses" color="c4e0f9" >
<comment>analysis tables</comment>
<entity schema="qiita" name="analysis" />
Expand All @@ -1344,6 +1356,7 @@ Controlled Vocabulary]]></comment>
<entity schema="qiita" name="analysis_filepath" />
<entity schema="qiita" name="job_results_filepath" />
<entity schema="qiita" name="command_data_type" />
<entity schema="qiita" name="analysis_workflow" />
</group>
<group name="Group_users" color="ffff99" >
<entity schema="qiita" name="user_level" />
Expand Down
64 changes: 55 additions & 9 deletions qiita_db/support_files/qiita-db.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,9 +183,9 @@

<g transform='translate(0,110)'>
<!-- ============= Group 'Group_analyses' ============= -->
<rect class='grp' style='fill:url(#groupGradient3);;' x='29' y='670' width='512' height='590' />
<text x='40' y='685'>Group_analyses</text>
<rect style='fill:url(#groupUnderTitleLine); stroke-width:2;' x='40' y='688' width='490' height='2' />
<rect class='grp' style='fill:url(#groupGradient3);;' x='29' y='595' width='512' height='665' />
<text x='40' y='610'>Group_analyses</text>
<rect style='fill:url(#groupUnderTitleLine); stroke-width:2;' x='40' y='613' width='490' height='2' />

<!-- ============= Group 'Group_users' ============= -->
<rect class='grp' style='fill:url(#groupGradient4);;' x='149' y='25' width='347' height='260' />
Expand Down Expand Up @@ -257,11 +257,11 @@
study_pmid references study ( study_id )</title>
</path>
<text x='1966' y='685' transform='rotate(0 1966,685)' title='Foreign Key fk_study_pmid_study
study_pmid references study ( study_id )' style='fill:#a1a0a0;'>study_id</text><path transform='translate(7,0)' marker-start='url(#foot)' marker-end='url(#arrow)' d='M 345 690 L 345,270' >
study_pmid references study ( study_id )' style='fill:#a1a0a0;'>study_id</text><path transform='translate(7,0)' marker-start='url(#foot)' marker-end='url(#arrow)' d='M 300 690 L 300,232 Q 300,225 307,225 L 315,225' >
<title>Foreign Key fk_analysis_user
analysis references qiita_user ( email )</title>
</path>
<text x='347' y='685' transform='rotate(270 347,685)' title='Foreign Key fk_analysis_user
<text x='302' y='685' transform='rotate(270 302,685)' title='Foreign Key fk_analysis_user
analysis references qiita_user ( email )' style='fill:#a1a0a0;'>email</text><path transform='translate(7,0)' marker-start='url(#foot)' marker-end='url(#arrow)' d='M 210 780 L 172,780 Q 165,780 165,787 L 165,795' >
<title>Foreign Key fk_analysis_analysis_status
analysis references analysis_status ( analysis_status_id )</title>
Expand Down Expand Up @@ -642,7 +642,12 @@
preprocessed_filepath references filepath ( filepath_id )</title>
</path>
<text x='921' y='685' transform='rotate(0 921,685)' title='Foreign Key fk_preprocessed_filepath_0
preprocessed_filepath references filepath ( filepath_id )' style='fill:#a1a0a0;'>filepath_id</text><!-- ============= Table 'term_synonym' ============= -->
preprocessed_filepath references filepath ( filepath_id )' style='fill:#a1a0a0;'>filepath_id</text><path transform='translate(7,0)' marker-start='url(#foot)' marker-end='url(#arrow)' d='M 375 675 L 352,675 Q 345,675 345,682 L 345,690' >
<title>Foreign Key fk_analysis_workflow
analysis_workflow references analysis ( analysis_id )</title>
</path>
<text x='318' y='670' transform='rotate(0 318,670)' title='Foreign Key fk_analysis_workflow
analysis_workflow references analysis ( analysis_id )' style='fill:#a1a0a0;'>analysis_id</text><!-- ============= Table 'term_synonym' ============= -->
<rect class='table' x='525' y='1868' width='135' height='105' rx='7' ry='7' />
<path d='M 525.50 1894.50 L 525.50 1875.50 Q 525.50 1868.50 532.50 1868.50 L 652.50 1868.50 Q 659.50 1868.50 659.50 1875.50 L 659.50 1894.50 L525.50 1894.50 ' style='fill:url(#tableHeaderGradient0); stroke:none;' />
<a xlink:href='#term_synonym'><text x='552' y='1882' class='tableTitle'>term_synonym</text><title>Table qiita.term_synonym</title></a>
Expand Down Expand Up @@ -783,7 +788,9 @@
Referred by analysis_filepath ( analysis_id )
Referred by analysis_job ( analysis_id )
Referred by analysis_sample ( analysis_id )
Referred by analysis_users ( analysis_id ) </title></a>
Referred by analysis_users ( analysis_id )
Referred by analysis_workflow ( analysis_id )
Referred by analysis_workflow ( analysis_id ) </title></a>
<use id='nn' x='227' y='747' xlink:href='#nn'/><a xlink:href='#email'><use id='idx' x='227' y='746' xlink:href='#idx'/><title>Index ( email ) </title></a>
<a xlink:href='#email'><text x='243' y='757'>email</text><title>email varchar not null
Email for user who owns the analysis</title></a>
Expand Down Expand Up @@ -1138,8 +1145,6 @@
Referred by common_prep_info ( data_type_id )
Referred by job ( data_type_id )
Referred by preprocessed_data ( data_type_id )
Referred by preprocessed_data ( data_type_id )
Referred by processed_data ( data_type_id )
Referred by processed_data ( data_type_id ) </title></a>
<use id='nn' x='692' y='1047' xlink:href='#nn'/><a xlink:href='#data_type'><text x='708' y='1057'>data_type</text><title>data_type varchar not null
Data type &#040;16S&#044; metabolome&#044; etc&#041; the job will use</title></a>
Expand Down Expand Up @@ -1622,6 +1627,16 @@
<a xlink:href='#filepath_id'><text x='1008' y='742'>filepath_id</text><title>filepath_id bigint not null</title></a>
<a xlink:href='#filepath_id'><use id='fk' x='1128' y='731' xlink:href='#fk'/><title>References filepath ( filepath_id ) </title></a>

<!-- ============= Table 'analysis_workflow' ============= -->
<rect class='table' x='390' y='623' width='120' height='75' rx='7' ry='7' />
<path d='M 390.50 649.50 L 390.50 630.50 Q 390.50 623.50 397.50 623.50 L 502.50 623.50 Q 509.50 623.50 509.50 630.50 L 509.50 649.50 L390.50 649.50 ' style='fill:url(#tableHeaderGradient1); stroke:none;' />
<a xlink:href='#analysis_workflow'><text x='401' y='637' class='tableTitle'>analysis_workflow</text><title>Table qiita.analysis_workflow
Stores what step in&#095;production analyses are on&#046;</title></a>
<use id='nn' x='392' y='657' xlink:href='#nn'/><a xlink:href='#analysis_id'><use id='pk' x='392' y='656' xlink:href='#pk'/><title>Primary Key ( analysis_id ) </title></a>
<a xlink:href='#analysis_id'><text x='408' y='667'>analysis_id</text><title>analysis_id bigint not null</title></a>
<a xlink:href='#analysis_id'><use id='fk' x='498' y='656' xlink:href='#fk'/><title>References analysis ( analysis_id ) </title></a>
<use id='nn' x='392' y='672' xlink:href='#nn'/><a xlink:href='#step'><text x='408' y='682'>step</text><title>step integer not null</title></a>

</g></svg>

<br/><br/>
Expand Down Expand Up @@ -4725,4 +4740,35 @@
</tbody>
</table>

<br/><br/>
<table id='dbs' >
<thead>
<tr><th colspan='3'><a name='analysis_workflow'>analysis_workflow</a></th></tr>
<tr><td colspan='3'>Stores what step in&#095;production analyses are on&#046; </td></tr>
</thead>
<tbody>
<tr>
<td><a name='analysis_id'>analysis&#095;id</a></td>
<td width='40%'> bigint NOT NULL </td>
<td width='99%'> </td>
</tr>
<tr>
<td><a name='step'>step</a></td>
<td width='40%'> integer NOT NULL </td>
<td width='99%'> </td>
</tr>
<tr><td colspan='3' class='subpart'><b>Indexes</b></td></tr>
<tr> <td>pk&#095;analysis&#095;workflow primary key</td>
<td> ON analysis&#095;id</td>
<td> </td>
</tr>
<tr><td colspan='3' class='subpart'><b>Foreign Keys</b></td></tr>
<tr>
<td>fk_analysis_workflow</td>
<td > ( analysis&#095;id ) ref <a href='#analysis'>analysis</a> (analysis&#095;id) </td>
<td> </td>
</tr>
</tbody>
</table>

</body></html>
9 changes: 9 additions & 0 deletions qiita_db/support_files/qiita-db.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -754,6 +754,15 @@ CREATE INDEX idx_analysis_users_email ON qiita.analysis_users ( email );

COMMENT ON TABLE qiita.analysis_users IS 'Links analyses to the users they are shared with';

CREATE TABLE qiita.analysis_workflow (
analysis_id bigint NOT NULL,
step integer NOT NULL,
CONSTRAINT pk_analysis_workflow PRIMARY KEY ( analysis_id ),
CONSTRAINT fk_analysis_workflow FOREIGN KEY ( analysis_id ) REFERENCES qiita.analysis( analysis_id )
);

COMMENT ON TABLE qiita.analysis_workflow IS 'Stores what step in_production analyses are on.';

CREATE TABLE qiita.investigation_study (
investigation_id bigint NOT NULL,
study_id bigint NOT NULL,
Expand Down
56 changes: 52 additions & 4 deletions qiita_db/test/test_analysis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,15 @@ def setUp(self):
self.analysis = Analysis(1)

def test_lock_check(self):
self.analysis.status = "public"
with self.assertRaises(QiitaDBStatusError):
self.analysis._lock_check(self.conn_handler)
for status in ["queued", "running", "completed", "public"]:
new = Analysis.create(User("admin@foo.bar"), "newAnalysis",
"A New Analysis")
new.status = status
with self.assertRaises(QiitaDBStatusError):
new._lock_check(self.conn_handler)

def test_lock_check_ok(self):
self.analysis.status = "queued"
self.analysis.status = "in_construction"
self.analysis._lock_check(self.conn_handler)

def test_get_public(self):
Expand Down Expand Up @@ -91,6 +94,40 @@ def test_retrieve_biom_tables_none(self):
"A New Analysis", Analysis(1))
self.assertEqual(new.biom_tables, None)

def test_set_step(self):
new = Analysis.create(User("admin@foo.bar"), "newAnalysis",
"A New Analysis", Analysis(1))
new.step = 2
sql = "SELECT * FROM qiita.analysis_workflow WHERE analysis_id = 3"
obs = self.conn_handler.execute_fetchall(sql)
self.assertEqual(obs, [[3, 2]])

def test_set_step_twice(self):
new = Analysis.create(User("admin@foo.bar"), "newAnalysis",
"A New Analysis", Analysis(1))
new.step = 2
new.step = 4
sql = "SELECT * FROM qiita.analysis_workflow WHERE analysis_id = 3"
obs = self.conn_handler.execute_fetchall(sql)
self.assertEqual(obs, [[3, 4]])

def test_retrive_step(self):
new = Analysis.create(User("admin@foo.bar"), "newAnalysis",
"A New Analysis", Analysis(1))
new.step = 2
self.assertEqual(new.step, 2)

def test_retrieve_step_new(self):
new = Analysis.create(User("admin@foo.bar"), "newAnalysis",
"A New Analysis", Analysis(1))
with self.assertRaises(ValueError):
new.step

def test_retrieve_step_locked(self):
self.analysis.status = "queued"
with self.assertRaises(QiitaDBStatusError):
self.analysis.step = 3

def test_retrieve_jobs(self):
self.assertEqual(self.analysis.jobs, [1, 2])

Expand Down Expand Up @@ -165,6 +202,17 @@ def test_unshare(self):
self.analysis.unshare(User("shared@foo.bar"))
self.assertEqual(self.analysis.shared_with, [])

def test_finish_workflow(self):
new = Analysis.create(User("admin@foo.bar"), "newAnalysis",
"A New Analysis", Analysis(1))
new.step = 2
new.finish_workflow()

obs = self.conn_handler.execute_fetchall(
"SELECT * FROM qiita.analysis_workflow WHERE analysis_id = 3")
self.assertEqual(obs, [])
self.assertEqual(new.status, "queued")


if __name__ == "__main__":
main()

0 comments on commit 95fa935

Please sign in to comment.