Skip to content

Commit 5489236

Browse files
antgonzaantgonza
committed
Merge pull request #1201 from josenavas/1084-db-changes
1084 db changes
2 parents 3a0976b + 074147e commit 5489236

File tree

4 files changed

+728
-600
lines changed

4 files changed

+728
-600
lines changed

‎qiita_db/support_files/patches/25.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
-- May 19, 2015
2+
3+
SELECT 42;

‎qiita_db/support_files/patches/python_patches/25.py‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
# May 19, 2015
2+
# We attach the prep template directly to the study. The raw data is no longer
3+
# attached to the study directly, the prep template points to them. This will
4+
# make the RawData to be effectively just a container for the raw files,
5+
# which is how it was acting previously.
6+
7+
from qiita_db.sql_connection import SQLConnectionHandler
8+
from qiita_db.data import RawData
9+
from qiita_db.util import move_filepaths_to_upload_folder
10+
11+
conn_handler = SQLConnectionHandler()
12+
queue = "PATCH_25"
13+
conn_handler.create_queue(queue)
14+
15+
# the system may contain raw data with no prep template associated to it.
16+
# Retrieve all those raw data ids
17+
sql = """SELECT raw_data_id
18+
FROM qiita.raw_data
19+
WHERE raw_data_id NOT IN (
20+
SELECT DISTINCT raw_data_id FROM qiita.prep_template);"""
21+
rd_ids = [x[0] for x in conn_handler.execute_fetchall(sql)]
22+
23+
# We will delete those RawData. However, if they have files attached, we should
24+
# move them to the uploads folder of the study
25+
sql_detach = """DELETE FROM qiita.study_raw_data
26+
WHERE raw_data_id = %s AND study_id = %s"""
27+
sql_unlink = "DELETE FROM qiita.raw_filepath WHERE raw_data_id = %s"
28+
sql_delete = "DELETE FROM qiita.raw_data WHERE raw_data_id = %s"
29+
move_files = []
30+
for rd_id in rd_ids:
31+
rd = RawData(rd_id)
32+
filepaths = rd.get_filepaths()
33+
studies = sorted(rd.studies)
34+
if filepaths:
35+
# we need to move the files to a study. We chose the one with lower
36+
# study id. Currently there is no case in the live database in which a
37+
# RawData with no prep templates is attached to more than one study,
38+
# but I think it is better to normalize this just in case
39+
move_files.append((min(studies), filepaths))
40+
41+
# To delete the RawData we first need to unlink all the files
42+
conn_handler.add_to_queue(queue, sql_unlink, (rd_id,))
43+
44+
# Then, remove the raw data from all the studies
45+
for st_id in studies:
46+
conn_handler.add_to_queue(queue, sql_detach, (rd_id, st_id))
47+
48+
conn_handler.add_to_queue(queue, sql_delete, (rd_id,))
49+
50+
# We can now perform all changes in the DB. Although these changes can be
51+
# done in an SQL patch, they are done here because we need to execute the
52+
# previous clean up in the database before we can actually execute the SQL
53+
# patch.
54+
sql = """CREATE TABLE qiita.study_prep_template (
55+
study_id bigint NOT NULL,
56+
prep_template_id bigint NOT NULL,
57+
CONSTRAINT idx_study_prep_template
58+
PRIMARY KEY ( study_id, prep_template_id )
59+
);
60+
61+
CREATE INDEX idx_study_prep_template_0
62+
ON qiita.study_prep_template ( study_id );
63+
64+
CREATE INDEX idx_study_prep_template_1
65+
ON qiita.study_prep_template ( prep_template_id );
66+
67+
COMMENT ON TABLE qiita.study_prep_template IS
68+
'links study to its prep templates';
69+
70+
ALTER TABLE qiita.study_prep_template
71+
ADD CONSTRAINT fk_study_prep_template_study
72+
FOREIGN KEY ( study_id ) REFERENCES qiita.study( study_id );
73+
74+
ALTER TABLE qiita.study_prep_template
75+
ADD CONSTRAINT fk_study_prep_template_pt
76+
FOREIGN KEY ( prep_template_id )
77+
REFERENCES qiita.prep_template( prep_template_id );
78+
79+
-- Connect the existing prep templates in the system with their studies
80+
DO $do$
81+
DECLARE
82+
vals RECORD;
83+
BEGIN
84+
FOR vals IN
85+
SELECT prep_template_id, study_id
86+
FROM qiita.prep_template
87+
JOIN qiita.study_raw_data USING (raw_data_id)
88+
LOOP
89+
INSERT INTO qiita.study_prep_template (study_id, prep_template_id)
90+
VALUES (vals.study_id, vals.prep_template_id);
91+
END LOOP;
92+
END $do$;
93+
94+
--- Drop the study_raw__data table as it's not longer used
95+
DROP TABLE qiita.study_raw_data;
96+
97+
-- The raw_data_id column now can be nullable
98+
ALTER TABLE qiita.prep_template
99+
ALTER COLUMN raw_data_id DROP NOT NULL;
100+
"""
101+
conn_handler.add_to_queue(queue, sql)
102+
conn_handler.execute_queue(queue)
103+
104+
# After the changes in the database have been performed, move the files
105+
# to the uploads folder
106+
errors = []
107+
for st_id, fps in move_files:
108+
try:
109+
move_filepaths_to_upload_folder(st_id, fps)
110+
except Exception, e:
111+
# An error here is unlikely. However, it's possible and there is no
112+
# clean way that we can unroll all the previous changes in the DB.
113+
errors.append((st_id, fps, str(e)))
114+
115+
# Show the user any error that could have been generated during the files
116+
# movement
117+
if errors:
118+
print ("The following errors where generated when trying to move files "
119+
"to the upload folder")
120+
for st_id, fps, e in errors:
121+
print "Study: %d, Filepaths: %s, Error: %s" % (st_id, fps, e)

‎qiita_db/support_files/qiita-db.dbs‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -747,9 +747,6 @@
747747
<index name="idx_common_prep_info_1" unique="NORMAL" >
748748
<column name="prep_template_id" />
749749
</index>
750-
<fk name="fk_common_prep_info" to_schema="qiita" to_table="study_sample" >
751-
<fk_column name="sample_id" pk="sample_id" />
752-
</fk>
753750
<fk name="fk_prep_template" to_schema="qiita" to_table="prep_template" >
754751
<fk_column name="prep_template_id" pk="prep_template_id" />
755752
</fk>
@@ -1340,6 +1337,27 @@ Controlled Vocabulary]]></comment>
13401337
<fk_column name="study_id" pk="study_id" />
13411338
</fk>
13421339
</table>
1340+
<table name="study_prep_template" >
1341+
<comment>links study to its prep templates</comment>
1342+
<column name="study_id" type="bigint" jt="-5" mandatory="y" />
1343+
<column name="prep_template_id" type="bigint" jt="-5" mandatory="y" />
1344+
<index name="idx_study_raw_data" unique="PRIMARY_KEY" >
1345+
<column name="study_id" />
1346+
<column name="prep_template_id" />
1347+
</index>
1348+
<index name="idx_study_raw_data_0" unique="NORMAL" >
1349+
<column name="study_id" />
1350+
</index>
1351+
<index name="idx_study_raw_data_1" unique="NORMAL" >
1352+
<column name="prep_template_id" />
1353+
</index>
1354+
<fk name="fk_study_prep_template_study" to_schema="qiita" to_table="study" >
1355+
<fk_column name="study_id" pk="study_id" />
1356+
</fk>
1357+
<fk name="fk_study_prep_template_pt" to_schema="qiita" to_table="prep_template" >
1358+
<fk_column name="prep_template_id" pk="prep_template_id" />
1359+
</fk>
1360+
</table>
13431361
<table name="study_preprocessed_data" >
13441362
<column name="study_id" type="bigint" jt="-5" mandatory="y" />
13451363
<column name="preprocessed_data_id" type="bigint" jt="-5" mandatory="y" />
@@ -1380,24 +1398,6 @@ Controlled Vocabulary]]></comment>
13801398
<fk_column name="processed_data_id" pk="processed_data_id" />
13811399
</fk>
13821400
</table>
1383-
<table name="study_raw_data" >
1384-
<comment>links study to its raw data</comment>
1385-
<column name="study_id" type="bigint" jt="-5" mandatory="y" />
1386-
<column name="raw_data_id" type="bigint" jt="-5" mandatory="y" />
1387-
<index name="idx_study_raw_data" unique="NORMAL" >
1388-
<column name="study_id" />
1389-
</index>
1390-
<index name="idx_study_raw_data_0" unique="PRIMARY_KEY" >
1391-
<column name="study_id" />
1392-
<column name="raw_data_id" />
1393-
</index>
1394-
<fk name="fk_study_raw_data_study" to_schema="qiita" to_table="study" >
1395-
<fk_column name="study_id" pk="study_id" />
1396-
</fk>
1397-
<fk name="fk_study_raw_data_raw_data" to_schema="qiita" to_table="raw_data" >
1398-
<fk_column name="raw_data_id" pk="raw_data_id" />
1399-
</fk>
1400-
</table>
14011401
<table name="study_sample" >
14021402
<comment>Required info for each sample. One row is one sample.</comment>
14031403
<column name="sample_id" type="varchar" jt="12" mandatory="y" />
@@ -1522,7 +1522,6 @@ Controlled Vocabulary]]></comment>
15221522
<entity schema="qiita" name="study_preprocessed_data" color="c0d4f3" x="1545" y="720" />
15231523
<entity schema="qiita" name="study_users" color="d0def5" x="1065" y="60" />
15241524
<entity schema="qiita" name="sample_x" color="d0def5" x="1635" y="165" />
1525-
<entity schema="qiita" name="study_raw_data" color="d0def5" x="1575" y="510" />
15261525
<entity schema="qiita" name="processed_filepath" color="c0d4f3" x="1065" y="945" />
15271526
<entity schema="qiita" name="command" color="d0def5" x="210" y="1110" />
15281527
<entity schema="qiita" name="logging" color="c0d4f3" x="1335" y="1290" />
@@ -1540,27 +1539,20 @@ Controlled Vocabulary]]></comment>
15401539
<entity schema="qiita" name="preprocessed_processed_data" color="b2cdf7" x="1275" y="870" />
15411540
<entity schema="qiita" name="qiita_user" color="d0def5" x="330" y="90" />
15421541
<entity schema="qiita" name="prep_y" color="d0def5" x="1230" y="195" />
1543-
<entity schema="qiita" name="prep_columns" color="b2cdf7" x="1275" y="345" />
1544-
<entity schema="qiita" name="raw_filepath" color="c0d4f3" x="1080" y="510" />
1545-
<entity schema="qiita" name="filetype" color="d0def5" x="1560" y="600" />
15461542
<entity schema="qiita" name="filepath_type" color="c0d4f3" x="585" y="885" />
15471543
<entity schema="qiita" name="checksum_algorithm" color="b2cdf7" x="735" y="885" />
15481544
<entity schema="qiita" name="data_type" color="d0def5" x="690" y="1020" />
15491545
<entity schema="qiita" name="user_level" color="d0def5" x="165" y="75" />
15501546
<entity schema="qiita" name="job_status" color="d0def5" x="210" y="1020" />
15511547
<entity schema="qiita" name="severity" color="c0d4f3" x="1470" y="1290" />
1552-
<entity schema="qiita" name="prep_template" color="b2cdf7" x="1065" y="360" />
1553-
<entity schema="qiita" name="raw_data" color="d0def5" x="1275" y="495" />
15541548
<entity schema="qiita" name="job" color="d0def5" x="405" y="1005" />
15551549
<entity schema="qiita" name="filepath" color="c0d4f3" x="645" y="675" />
15561550
<entity schema="qiita" name="data_directory" color="b2cdf7" x="840" y="585" />
15571551
<entity schema="qiita" name="term" color="d0def5" x="810" y="1650" />
15581552
<entity schema="qiita" name="environmental_package" color="b2cdf7" x="2250" y="150" />
15591553
<entity schema="qiita" name="study_environmental_package" color="b2cdf7" x="2250" y="45" />
15601554
<entity schema="qiita" name="timeseries_type" color="c0d4f3" x="1680" y="615" />
1561-
<entity schema="qiita" name="prep_template_filepath" color="b2cdf7" x="1035" y="600" />
15621555
<entity schema="qiita" name="sample_template_filepath" color="b2cdf7" x="1050" y="795" />
1563-
<entity schema="qiita" name="prep_template_preprocessed_data" color="b2cdf7" x="750" y="450" />
15641556
<entity schema="qiita" name="preprocessed_data" color="c0d4f3" x="1260" y="675" />
15651557
<entity schema="qiita" name="reference" color="c0d4f3" x="2280" y="960" />
15661558
<entity schema="qiita" name="preprocessed_sequence_454_params" color="c0d4f3" x="1740" y="960" />
@@ -1582,8 +1574,16 @@ Controlled Vocabulary]]></comment>
15821574
<entity schema="qiita" name="processed_data_status" color="c0d4f3" x="1500" y="1050" />
15831575
<entity schema="qiita" name="portal_type" color="c0d4f3" x="1995" y="660" />
15841576
<entity schema="qiita" name="analysis_sample" color="d0def5" x="45" y="1170" />
1585-
<entity schema="qiita" name="study_sample" color="d0def5" x="1410" y="120" />
1586-
<entity schema="qiita" name="prep_template_sample" color="d0def5" x="1050" y="165" />
1577+
<entity schema="qiita" name="study_sample" color="d0def5" x="1515" y="105" />
1578+
<entity schema="qiita" name="raw_data" color="d0def5" x="1020" y="300" />
1579+
<entity schema="qiita" name="filetype" color="d0def5" x="1035" y="180" />
1580+
<entity schema="qiita" name="prep_template_preprocessed_data" color="b2cdf7" x="1275" y="555" />
1581+
<entity schema="qiita" name="prep_template" color="b2cdf7" x="1305" y="375" />
1582+
<entity schema="qiita" name="raw_filepath" color="c0d4f3" x="1035" y="435" />
1583+
<entity schema="qiita" name="prep_template_filepath" color="b2cdf7" x="1035" y="525" />
1584+
<entity schema="qiita" name="study_prep_template" color="d0def5" x="1590" y="420" />
1585+
<entity schema="qiita" name="prep_columns" color="b2cdf7" x="1470" y="285" />
1586+
<entity schema="qiita" name="prep_template_sample" color="d0def5" x="1335" y="195" />
15871587
<group name="Group_analyses" color="c4e0f9" >
15881588
<comment>analysis tables</comment>
15891589
<entity schema="qiita" name="analysis" />
@@ -1610,7 +1610,7 @@ Controlled Vocabulary]]></comment>
16101610
<entity schema="qiita" name="study_sample_columns" />
16111611
<entity schema="qiita" name="raw_data" />
16121612
<entity schema="qiita" name="filetype" />
1613-
<entity schema="qiita" name="study_raw_data" />
1613+
<entity schema="qiita" name="study_prep_template" />
16141614
<entity schema="qiita" name="sample_x" />
16151615
<entity schema="qiita" name="preprocessed_spectra_params" />
16161616
<entity schema="qiita" name="preprocessed_sequence_illumina_params" />

0 commit comments

Comments
 (0)