diff --git a/.travis.yml b/.travis.yml index 8e4c86827..5a191929f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,9 +40,10 @@ script: - qiita-env start_cluster qiita-general - qiita-env make --no-load-ontologies - if [ ${TEST_ADD_STUDIES} == "True" ]; then test_data_studies/commands.sh ; fi + - if [ ${TEST_ADD_STUDIES} == "True" ]; then qiita-cron-job ; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then qiita-test-install ; fi - if [ ${TEST_ADD_STUDIES} == "False" ]; then nosetests --with-doctest --with-coverage -v --cover-package=qiita_db,qiita_pet,qiita_core,qiita_ware; fi - - flake8 qiita_* setup.py scripts/qiita scripts/qiita-env scripts/qiita-test-install + - flake8 qiita_* setup.py scripts/* - ls -R /home/travis/miniconda3/envs/qiita/lib/python2.7/site-packages/qiita_pet/support_files/doc/ - qiita pet webserver addons: diff --git a/qiita_core/support_files/server.crt b/qiita_core/support_files/server.crt index 0a56752a6..361fb497e 100644 --- a/qiita_core/support_files/server.crt +++ b/qiita_core/support_files/server.crt @@ -1,15 +1,15 @@ -----BEGIN CERTIFICATE----- -MIICQzCCAawCCQDD7K/frIbu8DANBgkqhkiG9w0BAQUFADBmMQswCQYDVQQGEwJV +MIICRTCCAa4CCQDPGmrQ4bra7TANBgkqhkiG9w0BAQUFADBmMQswCQYDVQQGEwJV UzELMAkGA1UECBMCQ0ExEjAQBgNVBAcTCVNhbiBEaWVnbzENMAsGA1UEChMEVUNT -RDETMBEGA1UECxMKS25pZ2h0IExhYjESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTE2 -MTIxOTE2MDA1NFoXDTE3MDExODE2MDA1NFowZjELMAkGA1UEBhMCVVMxCzAJBgNV -BAgTAkNBMRIwEAYDVQQHEwlTYW4gRGllZ28xDTALBgNVBAoTBFVDU0QxEzARBgNV -BAsTCktuaWdodCBMYWIxEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0B -AQEFAAOBjQAwgYkCgYEAq6ChN/5vk1fn45Ys5inttHe8IntBQtU31oKy+2IR+znT -GBvG/iht0veG5sbjlkm+Hn4auk5lR9EOmnTy+fl44LJ81rZuYmy3mjLSAHwmx7ee -ZTJ2lNjH/Blq5vC4VmPQ3Ka7zMusOTZSBDw6k8r6bxbMgarXc+rQtDvQfv2QITsC -AwEAATANBgkqhkiG9w0BAQUFAAOBgQBBir71K7HdTbU7129ZYLDyeXJfAjzCsSxj -evSqa6PJuh5PODdPyO01Hyxb5J/aHzmE5FRZKMLdgOTlqCpQjyMMvVc6UJzX5bZo -x6Y5gvoTNeCfaD0N6eZxxd7BqFGq+gmqk5U1cyKf+QjIhu/Q4p/Ga+Cx9b3t/Sk+ -/iUPu/otBw== +RDETMBEGA1UECxMKS25pZ2h0IExhYjESMBAGA1UEAxMJbG9jYWxob3N0MCAXDTE3 +MDExOTA4MTQ1NloYDzIxMTYxMjI2MDgxNDU2WjBmMQswCQYDVQQGEwJVUzELMAkG +A1UECBMCQ0ExEjAQBgNVBAcTCVNhbiBEaWVnbzENMAsGA1UEChMEVUNTRDETMBEG +A1UECxMKS25pZ2h0IExhYjESMBAGA1UEAxMJbG9jYWxob3N0MIGfMA0GCSqGSIb3 +DQEBAQUAA4GNADCBiQKBgQCroKE3/m+TV+fjlizmKe20d7wie0FC1TfWgrL7YhH7 +OdMYG8b+KG3S94bmxuOWSb4efhq6TmVH0Q6adPL5+XjgsnzWtm5ibLeaMtIAfCbH +t55lMnaU2Mf8GWrm8LhWY9DcprvMy6w5NlIEPDqTyvpvFsyBqtdz6tC0O9B+/ZAh +OwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAGLau2DhrdnR5P2C2rGZuSaHLYCsVPJO +nj3Q+v5md1UzTDitlzHwM3pX1QBLxfiTJ6e7/0QLkrDceYKOfU/eucLGM1KG1YjS +nB39W2BNLKXu4QXWJUx4WC1Qxib9wbxxm4NyMb0ir2/PZTs+gKMtguBUyVHqETvs +n1b0mapYTJ/Q -----END CERTIFICATE----- diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index fbd5129da..3acbe2d11 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -420,6 +420,15 @@ def _associate_with_analysis(instance, analysis_id): sql_args = [(instance.id, p.id) for p in parents] qdb.sql_connection.TRN.add(sql, sql_args, many=True) + # inheriting visibility + visibilities = {a.visibility for a in instance.parents} + # set based on the "lowest" visibility + if 'sandbox' in visibilities: + instance.visibility = 'sandbox' + elif 'private' in visibilities: + instance.visibility = 'private' + else: + instance.visibility = 'public' elif prep_template: # This artifact is uploaded by the user in the # processing pipeline diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 1f8956db5..33a84de37 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -13,7 +13,6 @@ ..autosummary:: :toctree: generated/ - get_accessible_filepath_ids get_lat_longs """ # ----------------------------------------------------------------------------- @@ -25,7 +24,16 @@ # ----------------------------------------------------------------------------- from __future__ import division -from itertools import chain +from moi import r_client +from os import stat +from time import strftime, localtime +import matplotlib.pyplot as plt +import matplotlib as mpl +from base64 import b64encode +from urllib import quote +from StringIO import StringIO +from future.utils import viewitems +from datetime import datetime from qiita_core.qiita_settings import qiita_config import qiita_db as qdb @@ -50,85 +58,232 @@ def _get_data_fpids(constructor, object_id): return {fpid for fpid, _, _ in obj.get_filepaths()} -def get_accessible_filepath_ids(user): - """Gets all filepaths that this user should have access to - - This gets all raw, preprocessed, and processed filepaths, for studies - that the user has access to, as well as all the mapping files and biom - tables associated with the analyses that the user has access to. +def validate_filepath_access_by_user(user, filepath_id): + """Validates if the user has access to the filepath_id Parameters ---------- user : User object The user we are interested in - + filepath_id : int + The filepath id Returns ------- - set - A set of filepath ids + bool + If the user has access or not to the filepath_id Notes ----- - Admins have access to all files, so all filepath ids are returned for - admins + Admins have access to all files so True is always returned """ - with qdb.sql_connection.TRN: + TRN = qdb.sql_connection.TRN + with TRN: if user.level == "admin": # admins have access all files - qdb.sql_connection.TRN.add( - "SELECT filepath_id FROM qiita.filepath") - return set(qdb.sql_connection.TRN.execute_fetchflatten()) - - # First, the studies - # There are private and shared studies - studies = user.user_studies | user.shared_studies - - filepath_ids = set() - for study in studies: - # Add the sample template files - if study.sample_template: - filepath_ids.update( - {fid for fid, _ in study.sample_template.get_filepaths()}) - - # Add the prep template filepaths - for pt in study.prep_templates(): - filepath_ids.update({fid for fid, _ in pt.get_filepaths()}) - - # Add the artifact filepaths - for artifact in study.artifacts(): - filepath_ids.update({fid for fid, _, _ in artifact.filepaths}) - - # Next, the public artifacts - for artifact in qdb.artifact.Artifact.iter_public(): - # Add the filepaths of the artifact - filepath_ids.update({fid for fid, _, _ in artifact.filepaths}) - - # Then add the filepaths of the prep templates - for pt in artifact.prep_templates: - filepath_ids.update({fid for fid, _ in pt.get_filepaths()}) - - # Then add the filepaths of the sample template - study = artifact.study - if study: - filepath_ids.update( - {fid - for fid, _ in study.sample_template.get_filepaths()}) - - # Next, analyses - # Same as before, there are public, private, and shared - analyses = qdb.analysis.Analysis.get_by_status('public') | \ - user.private_analyses | user.shared_analyses - - if analyses: - sql = """SELECT filepath_id - FROM qiita.analysis_filepath - WHERE analysis_id IN %s""" - sql_args = tuple([a.id for a in analyses]) - qdb.sql_connection.TRN.add(sql, [sql_args]) - filepath_ids.update(qdb.sql_connection.TRN.execute_fetchflatten()) - - return filepath_ids + return True + + sql = """SELECT + (SELECT array_agg(artifact_id) + FROM qiita.artifact_filepath + WHERE filepath_id = {0}) AS artifact, + (SELECT array_agg(study_id) + FROM qiita.sample_template_filepath + WHERE filepath_id = {0}) AS sample_info, + (SELECT array_agg(prep_template_id) + FROM qiita.prep_template_filepath + WHERE filepath_id = {0}) AS prep_info, + (SELECT array_agg(analysis_id) + FROM qiita.analysis_filepath + WHERE filepath_id = {0}) AS analysis""".format(filepath_id) + TRN.add(sql) + + arid, sid, pid, anid = TRN.execute_fetchflatten() + + # artifacts + if arid: + # [0] cause we should only have 1 + artifact = qdb.artifact.Artifact(arid[0]) + if artifact.visibility == 'public': + return True + else: + study = artifact.study + if study: + # let's take the visibility via the Study + return artifact.study.has_access(user) + else: + analysis = artifact.analysis + return analysis in ( + user.private_analyses | user.shared_analyses) + # sample info files + elif sid: + # the visibility of the sample info file is given by the + # study visibility + # [0] cause we should only have 1 + return qdb.study.Study(sid[0]).has_access(user) + # prep info files + elif pid: + # the prep access is given by it's artifacts, if the user has + # access to any artifact, it should have access to the prep + # [0] cause we should only have 1 + a = qdb.metadata_template.prep_template.PrepTemplate( + pid[0]).artifact + if (a.visibility == 'public' or a.study.has_access(user)): + return True + else: + for c in a.descendants.nodes(): + if (c.visibility == 'public' or c.study.has_access(user)): + return True + return False + # analyses + elif anid: + # [0] cause we should only have 1 + aid = anid[0] + analysis = qdb.analysis.Analysis(aid) + return analysis in ( + user.private_analyses | user.shared_analyses) + return False + + +def update_redis_stats(): + """Generate the system stats and save them in redis + + Returns + ------- + list of str + artifact filepaths that are not present in the file system + """ + STUDY = qdb.study.Study + studies = {'public': STUDY.get_by_status('private'), + 'private': STUDY.get_by_status('public'), + 'sanbox': STUDY.get_by_status('sandbox')} + number_studies = {k: len(v) for k, v in viewitems(studies)} + + number_of_samples = {} + ebi_samples_prep = {} + num_samples_ebi = 0 + for k, sts in viewitems(studies): + number_of_samples[k] = 0 + for s in sts: + st = s.sample_template + if st is not None: + number_of_samples[k] += len(list(st.keys())) + + ebi_samples_prep_count = 0 + for pt in s.prep_templates(): + ebi_samples_prep_count += len([ + 1 for _, v in viewitems(pt.ebi_experiment_accessions) + if v is not None and v != '']) + ebi_samples_prep[s.id] = ebi_samples_prep_count + + if s.sample_template is not None: + num_samples_ebi += len([ + 1 for _, v in viewitems( + s.sample_template.ebi_sample_accessions) + if v is not None and v != '']) + + num_users = qdb.util.get_count('qiita.qiita_user') + + lat_longs = get_lat_longs() + + num_studies_ebi = len(ebi_samples_prep) + number_samples_ebi_prep = sum([v for _, v in viewitems(ebi_samples_prep)]) + + # generating file size stats + stats = [] + missing_files = [] + for k, sts in viewitems(studies): + for s in sts: + for a in s.artifacts(): + for _, fp, dt in a.filepaths: + try: + s = stat(fp) + stats.append((dt, s.st_size, strftime('%Y-%m', + localtime(s.st_ctime)))) + except OSError: + missing_files.append(fp) + + summary = {} + all_dates = [] + for ft, size, ym in stats: + if ft not in summary: + summary[ft] = {} + if ym not in summary[ft]: + summary[ft][ym] = 0 + all_dates.append(ym) + summary[ft][ym] += size + all_dates = sorted(set(all_dates)) + + # sorting summaries + rm_from_data = ['html_summary', 'tgz', 'directory', 'raw_fasta', 'log', + 'biom', 'raw_sff', 'raw_qual'] + ordered_summary = {} + for dt in summary: + if dt in rm_from_data: + continue + new_list = [] + current_value = 0 + for ad in all_dates: + if ad in summary[dt]: + current_value += summary[dt][ad] + new_list.append(current_value) + ordered_summary[dt] = new_list + + plot_order = sorted([(k, ordered_summary[k][-1]) for k in ordered_summary], + key=lambda x: x[1]) + + # helper function to generate y axis, modified from: + # http://stackoverflow.com/a/1094933 + def sizeof_fmt(value, position): + number = None + for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: + if abs(value) < 1024.0: + number = "%3.1f%s" % (value, unit) + break + value /= 1024.0 + if number is None: + number = "%.1f%s" % (value, 'Yi') + return number + + all_dates_axis = range(len(all_dates)) + plt.locator_params(axis='y', nbins=10) + plt.figure(figsize=(20, 10)) + for k, v in plot_order: + plt.plot(all_dates_axis, ordered_summary[k], linewidth=2, label=k) + + plt.xticks(all_dates_axis, all_dates) + plt.legend() + plt.grid() + ax = plt.gca() + ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(sizeof_fmt)) + plt.xlabel('Date') + plt.ylabel('Storage space per data type') + + plot = StringIO() + plt.savefig(plot, format='png') + plot.seek(0) + img = 'data:image/png;base64,' + quote(b64encode(plot.buf)) + + time = datetime.now().strftime('%m-%d-%y %H:%M:%S') + + portal = qiita_config.portal + vals = [ + ('number_studies', number_studies, r_client.hmset), + ('number_of_samples', number_of_samples, r_client.hmset), + ('num_users', num_users, r_client.set), + ('lat_longs', lat_longs, r_client.set), + ('num_studies_ebi', num_studies_ebi, r_client.set), + ('num_samples_ebi', num_samples_ebi, r_client.set), + ('number_samples_ebi_prep', number_samples_ebi_prep, r_client.set), + ('img', img, r_client.set), + ('time', time, r_client.set)] + for k, v, f in vals: + redis_key = '%s:stats:%s' % (portal, k) + # important to "flush" variables to avoid errors + r_client.delete(redis_key) + f(redis_key, v) + + return missing_files def get_lat_longs(): @@ -143,6 +298,7 @@ def get_lat_longs(): s.id for s in qdb.portal.Portal(qiita_config.portal).get_studies()] with qdb.sql_connection.TRN: + # getting all tables in the portal sql = """SELECT DISTINCT table_name FROM information_schema.columns WHERE table_name SIMILAR TO 'sample_[0-9]+' @@ -151,16 +307,14 @@ def get_lat_longs(): AND SPLIT_PART(table_name, '_', 2)::int IN %s;""" qdb.sql_connection.TRN.add(sql, [tuple(portal_table_ids)]) - sql = """SELECT CAST(latitude AS FLOAT), CAST(longitude AS FLOAT) - FROM qiita.{0} - WHERE isnumeric(latitude) AND isnumeric(latitude)""" - idx = qdb.sql_connection.TRN.index - - portal_tables = qdb.sql_connection.TRN.execute_fetchflatten() - - ebi_null = tuple(qdb.metadata_template.constants.EBI_NULL_VALUES) - for table in portal_tables: - qdb.sql_connection.TRN.add(sql.format(table), [ebi_null, ebi_null]) - - return list( - chain.from_iterable(qdb.sql_connection.TRN.execute()[idx:])) + sql = [('SELECT CAST(latitude AS FLOAT), ' + ' CAST(longitude AS FLOAT) ' + 'FROM qiita.%s ' + 'WHERE isnumeric(latitude) AND isnumeric(longitude) ' + "AND latitude <> 'NaN' " + "AND longitude <> 'NaN' " % s) + for s in qdb.sql_connection.TRN.execute_fetchflatten()] + sql = ' UNION '.join(sql) + qdb.sql_connection.TRN.add(sql) + + return qdb.sql_connection.TRN.execute_fetchindex() diff --git a/qiita_db/metadata_template/test/test_util.py b/qiita_db/metadata_template/test/test_util.py index fc9f564de..ad43be487 100644 --- a/qiita_db/metadata_template/test/test_util.py +++ b/qiita_db/metadata_template/test/test_util.py @@ -359,7 +359,8 @@ def test_get_pgsql_reserved_words(self): "has_physical_specimen\thost_subject_id\tint_column\tlatitude\tlongitude\t" "physical_location\trequired_sample_info_status\tsample_type\t" "str_column\n" - "2.Sample1 \t05/29/2014 12:24:51\tTest Sample 1\tTrue\tTrue\t" + "2.Sample1 \t05/29/2014 12:24:51\tTest Sample 1\t" + '"True\t"\t"\nTrue"\t' "NotIdentified\t1\t42.42\t41.41\tlocation1\treceived\ttype1\t" "Value for sample 1\n" "2.Sample2 \t05/29/2014 12:24:51\t" diff --git a/qiita_db/metadata_template/util.py b/qiita_db/metadata_template/util.py index dddcdcf45..ce2b520da 100644 --- a/qiita_db/metadata_template/util.py +++ b/qiita_db/metadata_template/util.py @@ -60,16 +60,13 @@ def prefix_sample_names_with_id(md_template, study_id): md_template.index.name = None -def load_template_to_dataframe(fn, strip_whitespace=True, index='sample_name'): +def load_template_to_dataframe(fn, index='sample_name'): """Load a sample/prep template or a QIIME mapping file into a data frame Parameters ---------- fn : str or file-like object filename of the template to load, or an already open template file - strip_whitespace : bool, optional - Defaults to True. Whether or not to strip whitespace from values in the - input file index : str, optional Defaults to 'sample_name'. The index to use in the loaded information @@ -110,19 +107,6 @@ def load_template_to_dataframe(fn, strip_whitespace=True, index='sample_name'): if not holdfile: raise ValueError('Empty file passed!') - # Strip all values in the cells in the input file, if requested - if strip_whitespace: - for pos, line in enumerate(holdfile): - holdfile[pos] = '\t'.join(d.strip(" \r\x0b\x0c") - for d in line.split('\t')) - - # get and clean the controlled columns - cols = holdfile[0].split('\t') - controlled_cols = {'sample_name'} - controlled_cols.update(qdb.metadata_template.constants.CONTROLLED_COLS) - holdfile[0] = '\t'.join(c.lower() if c.lower() in controlled_cols else c - for c in cols) - if index == "#SampleID": # We're going to parse a QIIME mapping file. We are going to first # parse it with the QIIME function so we can remove the comments @@ -133,11 +117,29 @@ def load_template_to_dataframe(fn, strip_whitespace=True, index='sample_name'): # The QIIME parser fixes the index and removes the # index = 'SampleID' - # Check that we don't have duplicate columns - col_names = [c.lower() for c in holdfile[0].strip().split('\t')] - if len(set(col_names)) != len(col_names): - raise qdb.exceptions.QiitaDBDuplicateHeaderError( - find_duplicates(col_names)) + # Strip all values in the cells in the input file + for pos, line in enumerate(holdfile): + cols = line.split('\t') + if pos == 0 and index != 'SampleID': + # get and clean the controlled columns + ccols = {'sample_name'} + ccols.update(qdb.metadata_template.constants.CONTROLLED_COLS) + newcols = [ + c.lower().strip() if c.lower().strip() in ccols + else c.strip() + for c in cols] + + # while we are here, let's check for duplicate columns headers + if len(set(newcols)) != len(newcols): + raise qdb.exceptions.QiitaDBDuplicateHeaderError( + find_duplicates(newcols)) + else: + # .strip will remove odd chars, newlines, tabs and multiple + # spaces but we need to read a new line at the end of the + # line(+'\n') + newcols = [d.strip(" \r\x0b\x0c\n") for d in cols] + + holdfile[pos] = '\t'.join(newcols) + '\n' # index_col: # is set as False, otherwise it is cast as a float and we want a string @@ -158,6 +160,9 @@ def load_template_to_dataframe(fn, strip_whitespace=True, index='sample_name'): index_col=False, comment='\t', converters={index: lambda x: str(x).strip()}) + # remove newlines and tabs from fields + template.replace(to_replace='[\t\n\r\x0b\x0c]+', value='', + regex=True, inplace=True) except UnicodeDecodeError: # Find row number and col number for utf-8 encoding errors headers = holdfile[0].strip().split('\t') diff --git a/qiita_db/study.py b/qiita_db/study.py index 03e6ef607..ef21feddc 100644 --- a/qiita_db/study.py +++ b/qiita_db/study.py @@ -464,6 +464,42 @@ def delete(cls, id_): qdb.sql_connection.TRN.execute() + @classmethod + def get_tags(cls): + """Returns the available study tags + + Returns + ------- + list of DictCursor + Table-like structure of metadata, one tag per row. Can be + accessed as a list of dictionaries, keyed on column name. + """ + with qdb.sql_connection.TRN: + sql = """SELECT study_tag_id, study_tag + FROM qiita.study_tags""" + + qdb.sql_connection.TRN.add(sql) + return qdb.sql_connection.TRN.execute_fetchindex() + + @classmethod + def insert_tags(cls, user, tags): + """Insert available study tags + + Parameters + ---------- + user : qiita_db.user.User + The user adding the tags + tags : list of str + The list of tags to add + """ + with qdb.sql_connection.TRN: + sql = """INSERT INTO qiita.study_tags (email, study_tag) + VALUES (%s, %s)""" + sql_args = [[user.email, tag] for tag in tags] + + qdb.sql_connection.TRN.add(sql, sql_args, many=True) + qdb.sql_connection.TRN.execute() + # --- Attributes --- @property @@ -921,7 +957,52 @@ def ebi_submission_status(self, value): ebi_submission_status.__doc__.format(', '.join(_VALID_EBI_STATUS)) - # --- methods --- + @property + def tags(self): + """Returns the tags of the study + + Returns + ------- + list of str + The study tags + """ + with qdb.sql_connection.TRN: + sql = """SELECT study_tag_id, study_tag + FROM qiita.study_tags + LEFT JOIN qiita.per_study_tags USING (study_tag_id) + WHERE study_id = {0}""".format(self._id) + qdb.sql_connection.TRN.add(sql) + return qdb.sql_connection.TRN.execute_fetchindex() + + @tags.setter + def tags(self, tag_ids): + """Sets the tags of the study + + Parameters + ---------- + tag_ids : list of int + The tag ids of the study + """ + with qdb.sql_connection.TRN: + sql = """DELETE FROM qiita.per_study_tags WHERE study_id = %s""" + qdb.sql_connection.TRN.add(sql, [self._id]) + + if tag_ids: + sql = """INSERT INTO qiita.per_study_tags + (study_tag_id, study_id) + SELECT %s, %s + WHERE + NOT EXISTS ( + SELECT study_tag_id, study_id + FROM qiita.per_study_tags + WHERE study_tag_id = %s AND study_id = %s + )""" + sql_args = [[tid, self._id, tid, self._id] for tid in tag_ids] + qdb.sql_connection.TRN.add(sql, sql_args, many=True) + + qdb.sql_connection.TRN.execute() + +# --- methods --- def artifacts(self, dtype=None, artifact_type=None): """Returns the list of artifacts associated with the study diff --git a/qiita_db/support_files/patches/47.sql b/qiita_db/support_files/patches/47.sql index 13b1fbccb..077bb6690 100644 --- a/qiita_db/support_files/patches/47.sql +++ b/qiita_db/support_files/patches/47.sql @@ -1,115 +1,5 @@ --- Jan 5, 2017 --- Move the analysis to the plugin system. This is a major rewrite of the --- database backend that supports the analysis pipeline. --- After exploring the data on the database, we realized that --- there are a lot of inconsistencies in the data. Unfortunately, this --- makes the process of transferring the data from the old structure --- to the new one a bit more challenging, as we will need to handle --- different special cases. Furthermore, all the information needed is not --- present in the database, since it requires checking BIOM files. Due to these --- reason, the vast majority of the data transfer is done in the python patch --- 47.py +-- Jan 15, 2017 +-- Inherit the status of the study to all it's artifacts. +-- This code is much easier using python so check that patch --- In this file we are just creating the new data structures. The old --- datastructure will be dropped in the python patch once all data has been --- transferred. - --- Create the new data structures - --- Table that links the analysis with the initial set of artifacts -CREATE TABLE qiita.analysis_artifact ( - analysis_id bigint NOT NULL, - artifact_id bigint NOT NULL, - CONSTRAINT idx_analysis_artifact_0 PRIMARY KEY (analysis_id, artifact_id) -); -CREATE INDEX idx_analysis_artifact_analysis ON qiita.analysis_artifact (analysis_id); -CREATE INDEX idx_analysis_artifact_artifact ON qiita.analysis_artifact (artifact_id); -ALTER TABLE qiita.analysis_artifact ADD CONSTRAINT fk_analysis_artifact_analysis FOREIGN KEY ( analysis_id ) REFERENCES qiita.analysis( analysis_id ); -ALTER TABLE qiita.analysis_artifact ADD CONSTRAINT fk_analysis_artifact_artifact FOREIGN KEY ( artifact_id ) REFERENCES qiita.artifact( artifact_id ); - --- Droping the analysis status column cause now it depends on the artifacts --- status, like the study does. -ALTER TABLE qiita.analysis DROP COLUMN analysis_status_id; - --- Create a table to link the analysis with the jobs that create the initial --- artifacts -CREATE TABLE qiita.analysis_processing_job ( - analysis_id bigint NOT NULL, - processing_job_id uuid NOT NULL, - CONSTRAINT idx_analysis_processing_job PRIMARY KEY ( analysis_id, processing_job_id ) - ) ; - -CREATE INDEX idx_analysis_processing_job_analysis ON qiita.analysis_processing_job ( analysis_id ) ; -CREATE INDEX idx_analysis_processing_job_pj ON qiita.analysis_processing_job ( processing_job_id ) ; -ALTER TABLE qiita.analysis_processing_job ADD CONSTRAINT fk_analysis_processing_job FOREIGN KEY ( analysis_id ) REFERENCES qiita.analysis( analysis_id ) ; -ALTER TABLE qiita.analysis_processing_job ADD CONSTRAINT fk_analysis_processing_job_pj FOREIGN KEY ( processing_job_id ) REFERENCES qiita.processing_job( processing_job_id ) ; - --- Add a logging column in the analysis -ALTER TABLE qiita.analysis ADD logging_id bigint ; -CREATE INDEX idx_analysis_0 ON qiita.analysis ( logging_id ) ; -ALTER TABLE qiita.analysis ADD CONSTRAINT fk_analysis_logging FOREIGN KEY ( logging_id ) REFERENCES qiita.logging( logging_id ) ; - --- We can handle some of the special cases here, so we simplify the work in the --- python patch - --- Special case 1: there are jobs in the database that do not contain --- any information about the options used to process those parameters. --- However, these jobs do not have any results and all are marked either --- as queued or error, although no error log has been saved. Since these --- jobs are mainly useleess, we are going to remove them from the system -DELETE FROM qiita.analysis_job - WHERE job_id IN (SELECT job_id FROM qiita.job WHERE options = '{}'); -DELETE FROM qiita.job WHERE options = '{}'; - --- Special case 2: there are a fair amount of jobs (719 last time I --- checked) that are not attached to any analysis. Not sure how this --- can happen, but these orphan jobs can't be accessed from anywhere --- in the interface. Remove them from the system. Note that we are --- unlinking the files but we are not removing them from the filepath --- table. We will do that on the patch 47.py using the --- purge_filepaths function, as it will make sure that those files are --- not used anywhere else -DELETE FROM qiita.job_results_filepath WHERE job_id IN ( - SELECT job_id FROM qiita.job J WHERE NOT EXISTS ( - SELECT * FROM qiita.analysis_job AJ WHERE J.job_id = AJ.job_id)); -DELETE FROM qiita.job J WHERE NOT EXISTS ( - SELECT * FROM qiita.analysis_job AJ WHERE J.job_id = AJ.job_id); - --- In the analysis pipeline, an artifact can have mutliple datatypes --- (e.g. procrustes). Allow this by creating a new data_type being "multiomic" -INSERT INTO qiita.data_type (data_type) VALUES ('Multiomic'); - - --- The valdiate command from BIOM will have an extra parameter, analysis --- Magic number -> 4 BIOM command_id -> known for sure since it was added in --- patch 36.sql -INSERT INTO qiita.command_parameter (command_id, parameter_name, parameter_type, required) - VALUES (4, 'analysis', 'analysis', FALSE); --- The template comand now becomes optional, since it can be added either to --- an analysis or to a prep template. command_parameter_id known from patch --- 36.sql -UPDATE qiita.command_parameter SET required = FALSE WHERE command_parameter_id = 34; - --- We are going to add a new special software type, and a new software. --- This is going to be used internally by Qiita, so submit the private jobs. --- This is needed for the analysis. -INSERT INTO qiita.software_type (software_type, description) - VALUES ('private', 'Internal Qiita jobs'); - -DO $do$ -DECLARE - qiita_sw_id bigint; - baf_cmd_id bigint; -BEGIN - INSERT INTO qiita.software (name, version, description, environment_script, start_script, software_type_id, active) - VALUES ('Qiita', 'alpha', 'Internal Qiita jobs', 'source activate qiita', 'qiita-private-2', 3, True) - RETURNING software_id INTO qiita_sw_id; - - INSERT INTO qiita.software_command (software_id, name, description) - VALUES (qiita_sw_id, 'build_analysis_files', 'Builds the files needed for the analysis') - RETURNING command_id INTO baf_cmd_id; - - INSERT INTO qiita.command_parameter (command_id, parameter_name, parameter_type, required, default_value) - VALUES (baf_cmd_id, 'analysis', 'analysis', True, NULL), - (baf_cmd_id, 'merge_dup_sample_ids', 'bool', False, 'False'); -END $do$ +SELECT 1; diff --git a/qiita_db/support_files/patches/48.sql b/qiita_db/support_files/patches/48.sql new file mode 100644 index 000000000..f18e28868 --- /dev/null +++ b/qiita_db/support_files/patches/48.sql @@ -0,0 +1,4 @@ +-- Jan 20, 2017 +-- see py file + +SELECT 1; diff --git a/qiita_db/support_files/patches/49.sql b/qiita_db/support_files/patches/49.sql new file mode 100644 index 000000000..4b2b3c42a --- /dev/null +++ b/qiita_db/support_files/patches/49.sql @@ -0,0 +1,6 @@ +-- Jan 27, 2017 +-- sequeneces -> sequences + +UPDATE qiita.artifact_type SET description = 'Demultiplexed and QC sequences' + WHERE artifact_type = 'Demultiplexed' + AND description = 'Demultiplexed and QC sequeneces'; diff --git a/qiita_db/support_files/patches/50.sql b/qiita_db/support_files/patches/50.sql new file mode 100644 index 000000000..f732ef7b5 --- /dev/null +++ b/qiita_db/support_files/patches/50.sql @@ -0,0 +1,19 @@ +-- Feb 3, 2017 +-- adding study tagging system + +CREATE TABLE qiita.study_tags ( + study_tag_id bigserial NOT NULL, + email varchar NOT NULL, + study_tag varchar NOT NULL, + CONSTRAINT pk_study_tag UNIQUE ( study_tag ), + CONSTRAINT pk_study_tag_id PRIMARY KEY ( study_tag_id ) +) ; + +CREATE INDEX idx_study_tag_id ON qiita.study_tags ( study_tag_id ) ; +ALTER TABLE qiita.study_tags ADD CONSTRAINT fk_study_tags FOREIGN KEY ( email ) REFERENCES qiita.qiita_user( email ); + +CREATE TABLE qiita.per_study_tags ( + study_tag_id bigint NOT NULL, + study_id bigint NOT NULL, + CONSTRAINT pk_per_study_tags PRIMARY KEY ( study_tag_id, study_id ) +) ; diff --git a/qiita_db/support_files/patches/51.sql b/qiita_db/support_files/patches/51.sql new file mode 100644 index 000000000..a484d5c24 --- /dev/null +++ b/qiita_db/support_files/patches/51.sql @@ -0,0 +1,115 @@ +-- Jan 5, 2017 +-- Move the analysis to the plugin system. This is a major rewrite of the +-- database backend that supports the analysis pipeline. +-- After exploring the data on the database, we realized that +-- there are a lot of inconsistencies in the data. Unfortunately, this +-- makes the process of transferring the data from the old structure +-- to the new one a bit more challenging, as we will need to handle +-- different special cases. Furthermore, all the information needed is not +-- present in the database, since it requires checking BIOM files. Due to these +-- reason, the vast majority of the data transfer is done in the python patch +-- 51.py + +-- In this file we are just creating the new data structures. The old +-- datastructure will be dropped in the python patch once all data has been +-- transferred. + +-- Create the new data structures + +-- Table that links the analysis with the initial set of artifacts +CREATE TABLE qiita.analysis_artifact ( + analysis_id bigint NOT NULL, + artifact_id bigint NOT NULL, + CONSTRAINT idx_analysis_artifact_0 PRIMARY KEY (analysis_id, artifact_id) +); +CREATE INDEX idx_analysis_artifact_analysis ON qiita.analysis_artifact (analysis_id); +CREATE INDEX idx_analysis_artifact_artifact ON qiita.analysis_artifact (artifact_id); +ALTER TABLE qiita.analysis_artifact ADD CONSTRAINT fk_analysis_artifact_analysis FOREIGN KEY ( analysis_id ) REFERENCES qiita.analysis( analysis_id ); +ALTER TABLE qiita.analysis_artifact ADD CONSTRAINT fk_analysis_artifact_artifact FOREIGN KEY ( artifact_id ) REFERENCES qiita.artifact( artifact_id ); + +-- Droping the analysis status column cause now it depends on the artifacts +-- status, like the study does. +ALTER TABLE qiita.analysis DROP COLUMN analysis_status_id; + +-- Create a table to link the analysis with the jobs that create the initial +-- artifacts +CREATE TABLE qiita.analysis_processing_job ( + analysis_id bigint NOT NULL, + processing_job_id uuid NOT NULL, + CONSTRAINT idx_analysis_processing_job PRIMARY KEY ( analysis_id, processing_job_id ) + ) ; + +CREATE INDEX idx_analysis_processing_job_analysis ON qiita.analysis_processing_job ( analysis_id ) ; +CREATE INDEX idx_analysis_processing_job_pj ON qiita.analysis_processing_job ( processing_job_id ) ; +ALTER TABLE qiita.analysis_processing_job ADD CONSTRAINT fk_analysis_processing_job FOREIGN KEY ( analysis_id ) REFERENCES qiita.analysis( analysis_id ) ; +ALTER TABLE qiita.analysis_processing_job ADD CONSTRAINT fk_analysis_processing_job_pj FOREIGN KEY ( processing_job_id ) REFERENCES qiita.processing_job( processing_job_id ) ; + +-- Add a logging column in the analysis +ALTER TABLE qiita.analysis ADD logging_id bigint ; +CREATE INDEX idx_analysis_0 ON qiita.analysis ( logging_id ) ; +ALTER TABLE qiita.analysis ADD CONSTRAINT fk_analysis_logging FOREIGN KEY ( logging_id ) REFERENCES qiita.logging( logging_id ) ; + +-- We can handle some of the special cases here, so we simplify the work in the +-- python patch + +-- Special case 1: there are jobs in the database that do not contain +-- any information about the options used to process those parameters. +-- However, these jobs do not have any results and all are marked either +-- as queued or error, although no error log has been saved. Since these +-- jobs are mainly useleess, we are going to remove them from the system +DELETE FROM qiita.analysis_job + WHERE job_id IN (SELECT job_id FROM qiita.job WHERE options = '{}'); +DELETE FROM qiita.job WHERE options = '{}'; + +-- Special case 2: there are a fair amount of jobs (719 last time I +-- checked) that are not attached to any analysis. Not sure how this +-- can happen, but these orphan jobs can't be accessed from anywhere +-- in the interface. Remove them from the system. Note that we are +-- unlinking the files but we are not removing them from the filepath +-- table. We will do that on the patch 47.py using the +-- purge_filepaths function, as it will make sure that those files are +-- not used anywhere else +DELETE FROM qiita.job_results_filepath WHERE job_id IN ( + SELECT job_id FROM qiita.job J WHERE NOT EXISTS ( + SELECT * FROM qiita.analysis_job AJ WHERE J.job_id = AJ.job_id)); +DELETE FROM qiita.job J WHERE NOT EXISTS ( + SELECT * FROM qiita.analysis_job AJ WHERE J.job_id = AJ.job_id); + +-- In the analysis pipeline, an artifact can have mutliple datatypes +-- (e.g. procrustes). Allow this by creating a new data_type being "multiomic" +INSERT INTO qiita.data_type (data_type) VALUES ('Multiomic'); + + +-- The valdiate command from BIOM will have an extra parameter, analysis +-- Magic number -> 4 BIOM command_id -> known for sure since it was added in +-- patch 36.sql +INSERT INTO qiita.command_parameter (command_id, parameter_name, parameter_type, required) + VALUES (4, 'analysis', 'analysis', FALSE); +-- The template comand now becomes optional, since it can be added either to +-- an analysis or to a prep template. command_parameter_id known from patch +-- 36.sql +UPDATE qiita.command_parameter SET required = FALSE WHERE command_parameter_id = 34; + +-- We are going to add a new special software type, and a new software. +-- This is going to be used internally by Qiita, so submit the private jobs. +-- This is needed for the analysis. +INSERT INTO qiita.software_type (software_type, description) + VALUES ('private', 'Internal Qiita jobs'); + +DO $do$ +DECLARE + qiita_sw_id bigint; + baf_cmd_id bigint; +BEGIN + INSERT INTO qiita.software (name, version, description, environment_script, start_script, software_type_id, active) + VALUES ('Qiita', 'alpha', 'Internal Qiita jobs', 'source activate qiita', 'qiita-private-2', 3, True) + RETURNING software_id INTO qiita_sw_id; + + INSERT INTO qiita.software_command (software_id, name, description) + VALUES (qiita_sw_id, 'build_analysis_files', 'Builds the files needed for the analysis') + RETURNING command_id INTO baf_cmd_id; + + INSERT INTO qiita.command_parameter (command_id, parameter_name, parameter_type, required, default_value) + VALUES (baf_cmd_id, 'analysis', 'analysis', True, NULL), + (baf_cmd_id, 'merge_dup_sample_ids', 'bool', False, 'False'); +END $do$ diff --git a/qiita_db/support_files/patches/python_patches/47.py b/qiita_db/support_files/patches/python_patches/47.py index 43f1b65a9..a325a7a2e 100644 --- a/qiita_db/support_files/patches/python_patches/47.py +++ b/qiita_db/support_files/patches/python_patches/47.py @@ -1,688 +1,30 @@ -# The code is commented with details on the changes implemented here, -# but here is an overview of the changes needed to transfer the analysis -# data to the plugins structure: -# 1) Create a new type plugin to define the diversity types -# 2) Create the new commands on the existing QIIME plugin to execute the -# existing analyses (beta div, taxa summaries and alpha rarefaction) -# 3) Transfer all the data in the old structures to the plugin structures -# 4) Delete old structures - -from os.path import join, exists, basename -from os import makedirs -from json import loads - -from biom import load_table, Table -from biom.util import biom_open - -from qiita_db.sql_connection import TRN -from qiita_db.util import (get_db_files_base_dir, purge_filepaths, - get_mountpoint, compute_checksum) -from qiita_db.artifact import Artifact - -# Create some aux functions that are going to make the code more modular -# and easier to understand, since there is a fair amount of work to do to -# trasnfer the data from the old structure to the new one - - -def create_non_rarefied_biom_artifact(analysis, biom_data, rarefied_table): - """Creates the initial non-rarefied BIOM artifact of the analysis - - Parameters - ---------- - analysis : dict - Dictionary with the analysis information - biom_data : dict - Dictionary with the biom file information - rarefied_table : biom.Table - The rarefied BIOM table - - Returns - ------- - int - The id of the new artifact - """ - # The non rarefied biom artifact is the initial biom table of the analysis. - # This table does not currently exist anywhere, so we need to actually - # create the BIOM file. To create this BIOM file we need: (1) the samples - # and artifacts they come from and (2) whether the samples where - # renamed or not. (1) is on the database, but we need to inferr (2) from - # the existing rarefied BIOM table. Fun, fun... - - with TRN: - # Get the samples included in the BIOM table grouped by artifact id - # Note that the analysis contains a BIOM table per data type included - # in it, and the table analysis_sample does not differentiate between - # datatypes, so we need to check the data type in the artifact table - sql = """SELECT artifact_id, array_agg(sample_id) - FROM qiita.analysis_sample - JOIN qiita.artifact USING (artifact_id) - WHERE analysis_id = %s AND data_type_id = %s - GROUP BY artifact_id""" - TRN.add(sql, [analysis['analysis_id'], biom_data['data_type_id']]) - samples_by_artifact = TRN.execute_fetchindex() - - # Create an empty BIOM table to be the new master table - new_table = Table([], [], []) - ids_map = {} - for a_id, samples in samples_by_artifact: - # Get the filepath of the BIOM table from the artifact - artifact = Artifact(a_id) - biom_fp = None - for _, fp, fp_type in artifact.filepaths: - if fp_type == 'biom': - biom_fp = fp - # Note that we are sure that the biom table exists for sure, so - # no need to check if biom_fp is undefined - biom_table = load_table(biom_fp) - biom_table.filter(samples, axis='sample', inplace=True) - new_table = new_table.merge(biom_table) - ids_map.update({sid: "%d.%s" % (a_id, sid) - for sid in biom_table.ids()}) - - # Check if we need to rename the sample ids in the biom table - new_table_ids = set(new_table.ids()) - if not new_table_ids.issuperset(rarefied_table.ids()): - # We need to rename the sample ids - new_table.update_ids(ids_map, 'sample', True, True) - - sql = """INSERT INTO qiita.artifact - (generated_timestamp, data_type_id, visibility_id, - artifact_type_id, submitted_to_vamps) - VALUES (%s, %s, %s, %s, %s) - RETURNING artifact_id""" - # Magic number 4 -> visibility sandbox - # Magix number 7 -> biom artifact type - TRN.add(sql, [analysis['timestamp'], biom_data['data_type_id'], - 4, 7, False]) - artifact_id = TRN.execute_fetchlast() - # Associate the artifact with the analysis - sql = """INSERT INTO qiita.analysis_artifact - (analysis_id, artifact_id) - VALUES (%s, %s)""" - TRN.add(sql, [analysis['analysis_id'], artifact_id]) - # Link the artifact with its file - dd_id, mp = get_mountpoint('BIOM')[0] - dir_fp = join(get_db_files_base_dir(), mp, str(artifact_id)) - if not exists(dir_fp): - makedirs(dir_fp) - new_table_fp = join(dir_fp, "biom_table.biom") - with biom_open(new_table_fp, 'w') as f: - new_table.to_hdf5(f, "Generated by Qiita") - - sql = """INSERT INTO qiita.filepath - (filepath, filepath_type_id, checksum, - checksum_algorithm_id, data_directory_id) - VALUES (%s, %s, %s, %s, %s) - RETURNING filepath_id""" - # Magic number 7 -> filepath_type_id = 'biom' - # Magic number 1 -> the checksum algorithm id - TRN.add(sql, [basename(new_table_fp), 7, - compute_checksum(new_table_fp), 1, dd_id]) - fp_id = TRN.execute_fetchlast() - sql = """INSERT INTO qiita.artifact_filepath - (artifact_id, filepath_id) - VALUES (%s, %s)""" - TRN.add(sql, [artifact_id, fp_id]) - TRN.execute() - - return artifact_id - - -def create_rarefaction_job(depth, biom_artifact_id, analysis, srare_cmd_id): - """Create a new rarefaction job - - Parameters - ---------- - depth : int - The rarefaction depth - biom_artifact_id : int - The artifact id of the input rarefaction biom table - analysis : dict - Dictionary with the analysis information - srare_cmd_id : int - The command id of the single rarefaction command - - Returns - ------- - job_id : str - The job id - params : str - The job parameters - """ - # Add the row in the procesisng job table - params = ('{"depth":%d,"subsample_multinomial":false,"biom_table":%s}' - % (depth, biom_artifact_id)) - with TRN: - # magic number 3: status -> success - sql = """INSERT INTO qiita.processing_job - (email, command_id, command_parameters, - processing_job_status_id) - VALUES (%s, %s, %s, %s) - RETURNING processing_job_id""" - TRN.add(sql, [analysis['email'], srare_cmd_id, params, 3]) - job_id = TRN.execute_fetchlast() - # Step 1.2.b: Link the job with the input artifact - sql = """INSERT INTO qiita.artifact_processing_job - (artifact_id, processing_job_id) - VALUES (%s, %s)""" - TRN.add(sql, [biom_artifact_id, job_id]) - TRN.execute() - return job_id, params - - -def transfer_file_to_artifact(analysis_id, a_timestamp, command_id, - data_type_id, params, artifact_type_id, - filepath_id): - """Creates a new artifact with the given filepath id - - Parameters - ---------- - analysis_id : int - The analysis id to attach the artifact - a_timestamp : datetime.datetime - The generated timestamp of the artifact - command_id : int - The command id of the artifact - data_type_id : int - The data type id of the artifact - params : str - The parameters of the artifact - artifact_type_id : int - The artifact type - filepath_id : int - The filepath id - - Returns - ------- - int - The artifact id - """ - with TRN: - # Add the row in the artifact table - # Magic number 4: Visibility -> sandbox - sql = """INSERT INTO qiita.artifact - (generated_timestamp, command_id, data_type_id, - command_parameters, visibility_id, artifact_type_id, - submitted_to_vamps) - VALUES (%s, %s, %s, %s, %s, %s, %s) - RETURNING artifact_id""" - TRN.add(sql, [a_timestamp, command_id, data_type_id, params, 4, - artifact_type_id, False]) - artifact_id = TRN.execute_fetchlast() - # Link the artifact with its file - sql = """INSERT INTO qiita.artifact_filepath (artifact_id, filepath_id) - VALUES (%s, %s)""" - TRN.add(sql, [artifact_id, filepath_id]) - # Link the artifact with the analysis - sql = """INSERT INTO qiita.analysis_artifact - (analysis_id, artifact_id) - VALUES (%s, %s)""" - TRN.add(sql, [analysis_id, artifact_id]) - - return artifact_id - - -def create_rarefied_biom_artifact(analysis, srare_cmd_id, biom_data, params, - parent_biom_artifact_id, rarefaction_job_id, - srare_cmd_out_id): - """Creates the rarefied biom artifact - - Parameters - ---------- - analysis : dict - The analysis information - srare_cmd_id : int - The command id of "Single Rarefaction" - biom_data : dict - The biom information - params : str - The processing parameters - parent_biom_artifact_id : int - The parent biom artifact id - rarefaction_job_id : str - The job id of the rarefaction job - srare_cmd_out_id : int - The id of the single rarefaction output - - Returns - ------- - int - The artifact id - """ - with TRN: - # Transfer the file to an artifact - # Magic number 7: artifact type -> biom - artifact_id = transfer_file_to_artifact( - analysis['analysis_id'], analysis['timestamp'], srare_cmd_id, - biom_data['data_type_id'], params, 7, biom_data['filepath_id']) - # Link the artifact with its parent - sql = """INSERT INTO qiita.parent_artifact (artifact_id, parent_id) - VALUES (%s, %s)""" - TRN.add(sql, [artifact_id, parent_biom_artifact_id]) - # Link the artifact as the job output - sql = """INSERT INTO qiita.artifact_output_processing_job - (artifact_id, processing_job_id, command_output_id) - VALUES (%s, %s, %s)""" - TRN.add(sql, [artifact_id, rarefaction_job_id, srare_cmd_out_id]) - return artifact_id - - -def transfer_job(analysis, command_id, params, input_artifact_id, job_data, - cmd_out_id, biom_data, output_artifact_type_id): - """Transfers the job from the old structure to the plugin structure - - Parameters - ---------- - analysis : dict - The analysis information - command_id : int - The id of the command executed - params : str - The parameters used in the job - input_artifact_id : int - The id of the input artifact - job_data : dict - The job information - cmd_out_id : int - The id of the command's output - biom_data : dict - The biom information - output_artifact_type_id : int - The type of the output artifact - """ - with TRN: - # Create the job - # Add the row in the processing job table - # Magic number 3: status -> success - sql = """INSERT INTO qiita.processing_job - (email, command_id, command_parameters, - processing_job_status_id) - VALUES (%s, %s, %s, %s) - RETURNING processing_job_id""" - TRN.add(sql, [analysis['email'], command_id, params, 3]) - job_id = TRN.execute_fetchlast() - - # Link the job with the input artifact - sql = """INSERT INTO qiita.artifact_processing_job - (artifact_id, processing_job_id) - VALUES (rarefied_biom_id, proc_job_id)""" - TRN.add(sql, [input_artifact_id, job_id]) - - # Check if the executed job has results and add them - sql = """SELECT EXISTS(SELECT * - FROM qiita.job_results_filepath - WHERE job_id = %s)""" - TRN.add(sql, [job_data['job_id']]) - if TRN.execute_fetchlast(): - # There are results for the current job. - # Transfer the job files to a new artifact - sql = """SELECT filepath_id - FROM qiita.job_results_filepath - WHERE job_id = %s""" - TRN.add(sql, job_data['job_id']) - filepath_id = TRN.execute_fetchlast() - artifact_id = transfer_file_to_artifact( - analysis['analysis_id'], analysis['timestamp'], command_id, - biom_data['data_type_id'], params, output_artifact_type_id, - filepath_id) - - # Link the artifact with its parent - sql = """INSERT INTO qiita.parent_artifact (artifact_id, parent_id) - VALUES (%s, %s)""" - TRN.add(sql, [artifact_id, input_artifact_id]) - # Link the artifact as the job output - sql = """INSERT INTO qiita.artifact_output_processing_job - (artifact_id, processing_job_id, command_output_id) - VALUES (%s, %s, %s)""" - TRN.add(sql, [artifact_id, job_id, cmd_out_id]) - TRN.exeucte() - else: - # There are no results on the current job, so mark it as - # error - if job_data.log_id is None: - # Magic number 2 - we are not using any other severity - # level, so keep using number 2 - sql = """INSERT INTO qiita.logging (time, severity_id, msg) - VALUES (%s, %s, %s) - RETURNING logging_id""" - TRN.add(sql, [analysis['timestamp'], 2, - "Unknown error - patch 47"]) - else: - log_id = job_data['log_id'] - - # Magic number 4 -> status -> error - sql = """UPDATE qiita.processing_job - SET processing_job_status_id = 4, logging_id = %s - WHERE processing_job_id = %s""" - TRN.add(sql, [log_id, job_id]) - - -# The new commands that we are going to add generate new artifact types. -# These new artifact types are going to be added to a different plugin. -# In interest of time and given that the artifact type system is going to -# change in the near future, we feel that the easiest way to transfer -# the current analyses results is by creating 3 different types of -# artifacts: (1) distance matrix -> which will include the distance matrix, -# the principal coordinates and the emperor plots; (2) rarefaction -# curves -> which will include all the files generated by alpha rarefaction -# and (3) taxonomy summary, which will include all the files generated -# by summarize_taxa_through_plots.py - -# Step 1: Create the new type -with TRN: - # Magic number 2 -> The "artifact definition" software type - sql = """INSERT INTO qiita.software - (name, version, description, environment_script, start_script, - software_type_id) - VALUES ('Diversity types', '0.1.0', - 'Diversity artifacts type plugin', - 'source activate qiita', 'start_diversity_types', 2) - RETURNING software_id""" - TRN.add(sql) - divtype_id = TRN.execute_fetchlast() - - # Step 2: Create the validate and HTML generator commands - sql = """INSERT INTO qiita.software_command (software_id, name, description) - VALUES (%s, %s, %s) - RETURNING command_id""" - TRN.add(sql, [divtype_id, 'Validate', - 'Validates a new artifact of the given diversity type']) - validate_cmd_id = TRN.execute_fetchlast() - TRN.add(sql, [divtype_id, 'Generate HTML summary', - 'Generates the HTML summary of a given diversity type']) - html_summary_cmd_id = TRN.execute_fetchlast() - - # Step 3: Add the parameters for the previous commands - sql = """INSERT INTO qiita.command_parameter - (command_id, parameter_name, parameter_type, required) - VALUES (%s, %s, %s, %s)""" - sql_args = [(validate_cmd_id, 'files', 'string', True), - (validate_cmd_id, 'artifact_type', 'string', True), - (html_summary_cmd_id, 'input_data', 'artifact', True)] - TRN.add(sql, sql_args, many=True) - - # Step 4: Add the new artifact types - sql = """INSERT INTO qiita.artifact_type ( - artifact_type, description, can_be_submitted_to_ebi, - can_be_submitted_to_vamps) - VALUES (%s, %s, %s, %s) - RETURNING artifact_type_id""" - TRN.add(sql, ['distance_matrix', 'Distance matrix holding pairwise ' - 'distance between samples', False, False]) - dm_atype_id = TRN.execute_fetchlast() - TRN.add(sql, ['rarefaction_curves', 'Rarefaction curves', False, False]) - rc_atype_id = TRN.execute_fetchlast() - TRN.add(sql, ['taxa_summary', 'Taxa summary plots', False, False]) - ts_atype_id = TRN.execute_fetchlast() - - # Step 5: Associate each artifact with the filetypes that it accepts - # At this time we are going to add them as directories, just as it is done - # right now. We can make it fancier with the new type system. - # Magic number 8: the filepath_type_id for the directory - sql = """INSERT INTO qiita.artifact_type_filepath_type - (artifact_type_id, filepath_type_id, required) - VALUES (%s, %s, %s)""" - sql_args = [[dm_atype_id, 8, True], - [rc_atype_id, 8, True], - [ts_atype_id, 8, True]] - TRN.add(sql, sql_args, many=True) - - # Step 6: Associate the plugin with the types that it defines - sql = """INSERT INTO qiita.software_artifact_type - (software_id, artifact_type_id) - VALUES (%s, %s)""" - sql_args = [[divtype_id, dm_atype_id], - [divtype_id, rc_atype_id], - [divtype_id, ts_atype_id]] - TRN.add(sql, sql_args, many=True) - - # Step 7: Create the new entries for the data directory - sql = """INSERT INTO qiita.data_directory - (data_type, mountpoint, subdirectory, active) - VALUES (%s, %s, %s, %s)""" - sql_args = [['distance_matrix', 'distance_matrix', True, True], - ['rarefaction_curves', 'rarefaction_curves', True, True], - ['taxa_summary', 'taxa_summary', True, True]] - TRN.add(sql, sql_args, many=True) - - # Create the new commands that execute the current analyses. In qiita, - # the only commands that where available are Summarize Taxa, Beta - # Diversity and Alpha Rarefaction. The system was executing rarefaction - # by default, but it should be a different step in the analysis process - # so we are going to create a command for it too. These commands are going - # to be part of the QIIME plugin, so we are going to first retrieve the - # id of the QIIME 1.9.1 plugin, which for sure exists cause it was added - # in patch 33 and there is no way of removing plugins - - # Step 1: Get the QIIME plugin id - sql = """SELECT software_id - FROM qiita.software - WHERE name = 'QIIME' AND version = '1.9.1'""" - TRN.add(sql) - qiime_id = TRN.execute_fetchlast() - - # Step 2: Insert the new commands in the software_command table - sql = """INSERT INTO qiita.software_command (software_id, name, description) - VALUES (%s, %s, %s) - RETURNING command_id""" - TRN.add(sql, [qiime_id, 'Summarize Taxa', 'Plots taxonomy summaries at ' - 'different taxonomy levels']) - sum_taxa_cmd_id = TRN.execute_fetchlast() - TRN.add(sql, [qiime_id, 'Beta Diversity', - 'Computes and plots beta diversity results']) - bdiv_cmd_id = TRN.execute_fetchlast() - TRN.add(sql, [qiime_id, 'Alpha Rarefaction', - 'Computes and plots alpha rarefaction results']) - arare_cmd_id = TRN.execute_fetchlast() - TRN.add(sql, [qiime_id, 'Single Rarefaction', - 'Rarefies the input table by random sampling without ' - 'replacement']) - srare_cmd_id = TRN.execute_fetchlast() - - # Step 3: Insert the parameters for each command - sql = """INSERT INTO qiita.command_parameter - (command_id, parameter_name, parameter_type, required, - default_value) - VALUES (%s, %s, %s, %s, %s) - RETURNING command_parameter_id""" - sql_args = [ - # Summarize Taxa - (sum_taxa_cmd_id, 'metadata_category', 'string', False, ''), - (sum_taxa_cmd_id, 'sort', 'bool', False, 'False'), - # Beta Diversity - (bdiv_cmd_id, 'tree', 'string', False, ''), - (bdiv_cmd_id, 'metrics', - 'mchoice:["abund_jaccard","binary_chisq","binary_chord",' - '"binary_euclidean","binary_hamming","binary_jaccard",' - '"binary_lennon","binary_ochiai","binary_otu_gain","binary_pearson",' - '"binary_sorensen_dice","bray_curtis","bray_curtis_faith",' - '"bray_curtis_magurran","canberra","chisq","chord","euclidean",' - '"gower","hellinger","kulczynski","manhattan","morisita_horn",' - '"pearson","soergel","spearman_approx","specprof","unifrac",' - '"unifrac_g","unifrac_g_full_tree","unweighted_unifrac",' - '"unweighted_unifrac_full_tree","weighted_normalized_unifrac",' - '"weighted_unifrac"]', False, '["binary_jaccard","bray_curtis"]'), - # Alpha rarefaction - (arare_cmd_id, 'tree', 'string', False, ''), - (arare_cmd_id, 'num_steps', 'integer', False, 10), - (arare_cmd_id, 'min_rare_depth', 'integer', False, 10), - (arare_cmd_id, 'max_rare_depth', 'integer', False, 'Default'), - # Single rarefaction - (srare_cmd_id, 'depth', 'integer', True, None), - (srare_cmd_id, 'subsample_multinomial', 'bool', False, 'False') - ] - TRN.add(sql, sql_args, many=True) - - TRN.add(sql, [sum_taxa_cmd_id, 'biom_table', 'artifact', True, None]) - sum_taxa_cmd_param_id = TRN.execute_fetchlast() - TRN.add(sql, [bdiv_cmd_id, 'biom_table', 'artifact', True, None]) - bdiv_cmd_param_id = TRN.execute_fetchlast() - TRN.add(sql, [arare_cmd_id, 'biom_table', 'artifact', True, None]) - arare_cmd_param_id = TRN.execute_fetchlast() - TRN.add(sql, [srare_cmd_id, 'biom_table', 'artifact', True, None]) - srare_cmd_param_id = TRN.execute_fetchlast() - - # Step 4: Connect the artifact parameters with the artifact types that - # they accept - sql = """SELECT artifact_type_id - FROM qiita.artifact_type - WHERE artifact_type = 'BIOM'""" - TRN.add(sql) - biom_atype_id = TRN.execute_fetchlast() - - sql = """INSERT INTO qiita.parameter_artifact_type - (command_parameter_id, artifact_type_id) - VALUES (%s, %s)""" - sql_args = [[sum_taxa_cmd_param_id, biom_atype_id], - [bdiv_cmd_param_id, biom_atype_id], - [arare_cmd_param_id, biom_atype_id], - [srare_cmd_param_id, biom_atype_id]] - TRN.add(sql, sql_args, many=True) - - # Step 5: Add the outputs of the command. - sql = """INSERT INTO qiita.command_output - (name, command_id, artifact_type_id) - VALUES (%s, %s, %s) - RETURNING command_output_id""" - TRN.add(sql, ['taxa_summary', sum_taxa_cmd_id, ts_atype_id]) - sum_taxa_cmd_out_id = TRN.execute_fetchlast() - TRN.add(sql, ['distance_matrix', bdiv_cmd_id, dm_atype_id]) - bdiv_cmd_out_id = TRN.execute_fetchlast() - TRN.add(sql, ['rarefaction_curves', arare_cmd_id, rc_atype_id]) - arare_cmd_out_id = TRN.execute_fetchlast() - TRN.add(sql, ['rarefied_table', srare_cmd_id, biom_atype_id]) - srare_cmd_out_id = TRN.execute_fetchlast() - -# At this point we are ready to start transferring the data from the old -# structures to the new structures. Overview of the procedure: -# Step 1: Add initial set of artifacts up to rarefied table -# Step 2: Transfer the "analisys jobs" to processing jobs and create -# the analysis artifacts -db_dir = get_db_files_base_dir() -with TRN: - sql = "SELECT * FROM qiita.analysis" - TRN.add(sql) - analysis_info = TRN.execute_fetchindex() - - # Loop through all the analysis - for analysis in analysis_info: - # Step 1: Add the inital set of artifacts. An analysis starts with - # a set of BIOM artifacts. - sql = """SELECT * - FROM qiita.analysis_filepath - JOIN qiita.filepath USING (filepath_id) - JOIN qiita.filepath_type USING (filepath_type_id) - WHERE analysis_id = %s AND filepath_type = 'biom'""" - TRN.add(sql, [analysis['analysis_id']]) - analysis_bioms = TRN.execute_fetchindex() - - # Loop through all the biom tables associated with the current analysis - # so we can create the initial set of artifacts - for biom_data in analysis_bioms: - # Get the path of the BIOM table - sql = """SELECT filepath, mountpoint - FROM qiita.filepath - JOIN qiita.data_directory USING (data_directory_id) - WHERE filepath_id = %s""" - TRN.add(sql, [biom_data['filepath_id']]) - # Magic number 0: There is only a single row in the query result - fp_info = TRN.execute_fetchindex()[0] - filepath = join(db_dir, fp_info['mountpoint'], fp_info['filepath']) - - # We need to check if the BIOM table has been rarefied or not - table = load_table(filepath) - depths = set(table.sum(axis='sample')) - if len(depths) == 1: - # The BIOM table was rarefied - # Create the initial unrarefied artifact - initial_biom_artifact_id = create_non_rarefied_biom_artifact( - analysis, biom_data, table) - # Create the rarefaction job - rarefaction_job_id, params = create_rarefaction_job( - depths.pop(), initial_biom_artifact_id, analysis, - srare_cmd_id) - # Create the rarefied artifact - rarefied_biom_artifact_id = create_rarefied_biom_artifact( - analysis, srare_cmd_id, biom_data, params, - initial_biom_artifact_id, rarefaction_job_id, - srare_cmd_out_id) - else: - # The BIOM table was not rarefied, use current table as initial - initial_biom_id = transfer_file_to_artifact() - - # Loop through all the jobs that used this biom table as input - sql = """SELECT * - FROM qiita.job - WHERE reverse(split_part(reverse( - options::json->>'--otu_table_fp'), '/', 1)) = %s""" - TRN.add(sql, [filepath]) - analysis_jobs = TRN.execute_fetchindex() - for job_data in analysis_jobs: - # Identify which command the current job exeucted - if job_data['command_id'] == 1: - # Taxa summaries - cmd_id = sum_taxa_cmd_id - params = ('{"biom_table":%d,"metadata_category":"",' - '"sort":false}' % initial_biom_id) - output_artifact_type_id = ts_atype_id - cmd_out_id = sum_taxa_cmd_out_id - elif job_data['command_id'] == 2: - # Beta diversity - cmd_id = bdiv_cmd_id - tree_fp = loads(job_data['options'])['--tree_fp'] - if tree_fp: - params = ('{"biom_table":%d,"tree":"%s","metrics":' - '["unweighted_unifrac","weighted_unifrac"]}' - % (initial_biom_id, tree_fp)) - else: - params = ('{"biom_table":%d,"metrics":["bray_curtis",' - '"gower","canberra","pearson"]}' - % initial_biom_id) - output_artifact_type_id = dm_atype_id - cmd_out_id = bdiv_cmd_out_id - else: - # Alpha rarefaction - cmd_id = arare_cmd_id - tree_fp = loads(job_data['options'])['--tree_fp'] - params = ('{"biom_table":%d,"tree":"%s","num_steps":"10",' - '"min_rare_depth":"10",' - '"max_rare_depth":"Default"}' - % (initial_biom_id, tree_fp)) - output_artifact_type_id = rc_atype_id - cmd_out_id = arare_cmd_out_id - - transfer_job(analysis, cmd_id, params, initial_biom_id, - job_data, cmd_out_id, biom_data, - output_artifact_type_id) - -errors = [] -with TRN: - # Unlink the analysis from the biom table filepaths - # Magic number 7 -> biom filepath type - sql = """DELETE FROM qiita.analysis_filepath - WHERE filepath_id IN (SELECT filepath_id - FROM qiita.filepath - WHERE filepath_type_id = 7)""" - TRN.add(sql) - TRN.execute() - - # Delete old structures that are not used anymore - tables = ["collection_job", "collection_analysis", "collection_users", - "collection", "collection_status", "analysis_workflow", - "analysis_chain", "analysis_job", "job_results_filepath", "job", - "job_status", "command_data_type", "command", "analysis_status"] - for table in tables: - TRN.add("DROP TABLE qiita.%s" % table) - try: - TRN.execute() - except Exception as e: - errors.append("Error deleting table %s: %s" % (table, str(e))) - -# Purge filepaths -try: - purge_filepaths() -except Exception as e: - errors.append("Error purging filepaths: %s" % str(e)) - -if errors: - print "\n".join(errors) +from qiita_db.study import Study + + +class ForRecursion(object): + """for some strange reason, my guess is how we are executing the patches + recursion doesn't work directly so decided to use a class to make it + work""" + + @classmethod + def change_status(cls, artifact, status): + for a in artifact.children: + try: + a.visibility = status + except: + # print so we know which changes failed and we can deal by hand + print "failed aid: %d, status %s" % (artifact.id, status) + return + cls.change_status(a, status) + + +studies = Study.get_by_status('private').union( + Study.get_by_status('public')).union(Study.get_by_status('sandbox')) +# just getting the base artifacts, no parents +artifacts = {a for s in studies for a in s.artifacts() if not a.parents} + +# inheriting status +fr = ForRecursion +for a in artifacts: + status = a.visibility + fr.change_status(a, status) diff --git a/qiita_db/support_files/patches/python_patches/48.py b/qiita_db/support_files/patches/python_patches/48.py new file mode 100644 index 000000000..e831f80ba --- /dev/null +++ b/qiita_db/support_files/patches/python_patches/48.py @@ -0,0 +1,56 @@ +# replacing all \t and \n for space as those chars brake QIIME + +from qiita_db.study import Study +from qiita_db.sql_connection import TRN + + +def searcher(df): + search = r"\t|\n" + + return [col for col in df + if df[col].str.contains(search, na=False, regex=True).any()] + + +studies = Study.get_by_status('private').union( + Study.get_by_status('public')).union(Study.get_by_status('sandbox')) + +# we will start search using pandas as is much easier and faster +# than using pgsql. remember that to_dataframe actually transforms what's +# in the db +to_fix = [] +for s in studies: + st = s.sample_template + if st is None: + continue + cols = searcher(st.to_dataframe()) + if cols: + to_fix.append((st, cols)) + + for pt in s.prep_templates(): + if pt is None: + continue + cols = searcher(pt.to_dataframe()) + if cols: + to_fix.append((pt, cols)) + + +# now let's fix the database and regenerate the files +for infofile, cols in to_fix: + with TRN: + for col in cols: + # removing tabs + sql = """UPDATE qiita.{0}{1} + SET {2} = replace({2}, chr(9), ' ')""".format( + infofile._table_prefix, infofile.id, col) + TRN.add(sql) + + # removing enters + sql = """UPDATE qiita.{0}{1} + SET {2} = regexp_replace( + {2}, E'[\\n\\r\\u2028]+', ' ', 'g' )""".format( + infofile._table_prefix, infofile.id, col) + TRN.add(sql) + + TRN.execute() + + infofile.generate_files() diff --git a/qiita_db/support_files/patches/python_patches/51.py b/qiita_db/support_files/patches/python_patches/51.py new file mode 100644 index 000000000..43f1b65a9 --- /dev/null +++ b/qiita_db/support_files/patches/python_patches/51.py @@ -0,0 +1,688 @@ +# The code is commented with details on the changes implemented here, +# but here is an overview of the changes needed to transfer the analysis +# data to the plugins structure: +# 1) Create a new type plugin to define the diversity types +# 2) Create the new commands on the existing QIIME plugin to execute the +# existing analyses (beta div, taxa summaries and alpha rarefaction) +# 3) Transfer all the data in the old structures to the plugin structures +# 4) Delete old structures + +from os.path import join, exists, basename +from os import makedirs +from json import loads + +from biom import load_table, Table +from biom.util import biom_open + +from qiita_db.sql_connection import TRN +from qiita_db.util import (get_db_files_base_dir, purge_filepaths, + get_mountpoint, compute_checksum) +from qiita_db.artifact import Artifact + +# Create some aux functions that are going to make the code more modular +# and easier to understand, since there is a fair amount of work to do to +# trasnfer the data from the old structure to the new one + + +def create_non_rarefied_biom_artifact(analysis, biom_data, rarefied_table): + """Creates the initial non-rarefied BIOM artifact of the analysis + + Parameters + ---------- + analysis : dict + Dictionary with the analysis information + biom_data : dict + Dictionary with the biom file information + rarefied_table : biom.Table + The rarefied BIOM table + + Returns + ------- + int + The id of the new artifact + """ + # The non rarefied biom artifact is the initial biom table of the analysis. + # This table does not currently exist anywhere, so we need to actually + # create the BIOM file. To create this BIOM file we need: (1) the samples + # and artifacts they come from and (2) whether the samples where + # renamed or not. (1) is on the database, but we need to inferr (2) from + # the existing rarefied BIOM table. Fun, fun... + + with TRN: + # Get the samples included in the BIOM table grouped by artifact id + # Note that the analysis contains a BIOM table per data type included + # in it, and the table analysis_sample does not differentiate between + # datatypes, so we need to check the data type in the artifact table + sql = """SELECT artifact_id, array_agg(sample_id) + FROM qiita.analysis_sample + JOIN qiita.artifact USING (artifact_id) + WHERE analysis_id = %s AND data_type_id = %s + GROUP BY artifact_id""" + TRN.add(sql, [analysis['analysis_id'], biom_data['data_type_id']]) + samples_by_artifact = TRN.execute_fetchindex() + + # Create an empty BIOM table to be the new master table + new_table = Table([], [], []) + ids_map = {} + for a_id, samples in samples_by_artifact: + # Get the filepath of the BIOM table from the artifact + artifact = Artifact(a_id) + biom_fp = None + for _, fp, fp_type in artifact.filepaths: + if fp_type == 'biom': + biom_fp = fp + # Note that we are sure that the biom table exists for sure, so + # no need to check if biom_fp is undefined + biom_table = load_table(biom_fp) + biom_table.filter(samples, axis='sample', inplace=True) + new_table = new_table.merge(biom_table) + ids_map.update({sid: "%d.%s" % (a_id, sid) + for sid in biom_table.ids()}) + + # Check if we need to rename the sample ids in the biom table + new_table_ids = set(new_table.ids()) + if not new_table_ids.issuperset(rarefied_table.ids()): + # We need to rename the sample ids + new_table.update_ids(ids_map, 'sample', True, True) + + sql = """INSERT INTO qiita.artifact + (generated_timestamp, data_type_id, visibility_id, + artifact_type_id, submitted_to_vamps) + VALUES (%s, %s, %s, %s, %s) + RETURNING artifact_id""" + # Magic number 4 -> visibility sandbox + # Magix number 7 -> biom artifact type + TRN.add(sql, [analysis['timestamp'], biom_data['data_type_id'], + 4, 7, False]) + artifact_id = TRN.execute_fetchlast() + # Associate the artifact with the analysis + sql = """INSERT INTO qiita.analysis_artifact + (analysis_id, artifact_id) + VALUES (%s, %s)""" + TRN.add(sql, [analysis['analysis_id'], artifact_id]) + # Link the artifact with its file + dd_id, mp = get_mountpoint('BIOM')[0] + dir_fp = join(get_db_files_base_dir(), mp, str(artifact_id)) + if not exists(dir_fp): + makedirs(dir_fp) + new_table_fp = join(dir_fp, "biom_table.biom") + with biom_open(new_table_fp, 'w') as f: + new_table.to_hdf5(f, "Generated by Qiita") + + sql = """INSERT INTO qiita.filepath + (filepath, filepath_type_id, checksum, + checksum_algorithm_id, data_directory_id) + VALUES (%s, %s, %s, %s, %s) + RETURNING filepath_id""" + # Magic number 7 -> filepath_type_id = 'biom' + # Magic number 1 -> the checksum algorithm id + TRN.add(sql, [basename(new_table_fp), 7, + compute_checksum(new_table_fp), 1, dd_id]) + fp_id = TRN.execute_fetchlast() + sql = """INSERT INTO qiita.artifact_filepath + (artifact_id, filepath_id) + VALUES (%s, %s)""" + TRN.add(sql, [artifact_id, fp_id]) + TRN.execute() + + return artifact_id + + +def create_rarefaction_job(depth, biom_artifact_id, analysis, srare_cmd_id): + """Create a new rarefaction job + + Parameters + ---------- + depth : int + The rarefaction depth + biom_artifact_id : int + The artifact id of the input rarefaction biom table + analysis : dict + Dictionary with the analysis information + srare_cmd_id : int + The command id of the single rarefaction command + + Returns + ------- + job_id : str + The job id + params : str + The job parameters + """ + # Add the row in the procesisng job table + params = ('{"depth":%d,"subsample_multinomial":false,"biom_table":%s}' + % (depth, biom_artifact_id)) + with TRN: + # magic number 3: status -> success + sql = """INSERT INTO qiita.processing_job + (email, command_id, command_parameters, + processing_job_status_id) + VALUES (%s, %s, %s, %s) + RETURNING processing_job_id""" + TRN.add(sql, [analysis['email'], srare_cmd_id, params, 3]) + job_id = TRN.execute_fetchlast() + # Step 1.2.b: Link the job with the input artifact + sql = """INSERT INTO qiita.artifact_processing_job + (artifact_id, processing_job_id) + VALUES (%s, %s)""" + TRN.add(sql, [biom_artifact_id, job_id]) + TRN.execute() + return job_id, params + + +def transfer_file_to_artifact(analysis_id, a_timestamp, command_id, + data_type_id, params, artifact_type_id, + filepath_id): + """Creates a new artifact with the given filepath id + + Parameters + ---------- + analysis_id : int + The analysis id to attach the artifact + a_timestamp : datetime.datetime + The generated timestamp of the artifact + command_id : int + The command id of the artifact + data_type_id : int + The data type id of the artifact + params : str + The parameters of the artifact + artifact_type_id : int + The artifact type + filepath_id : int + The filepath id + + Returns + ------- + int + The artifact id + """ + with TRN: + # Add the row in the artifact table + # Magic number 4: Visibility -> sandbox + sql = """INSERT INTO qiita.artifact + (generated_timestamp, command_id, data_type_id, + command_parameters, visibility_id, artifact_type_id, + submitted_to_vamps) + VALUES (%s, %s, %s, %s, %s, %s, %s) + RETURNING artifact_id""" + TRN.add(sql, [a_timestamp, command_id, data_type_id, params, 4, + artifact_type_id, False]) + artifact_id = TRN.execute_fetchlast() + # Link the artifact with its file + sql = """INSERT INTO qiita.artifact_filepath (artifact_id, filepath_id) + VALUES (%s, %s)""" + TRN.add(sql, [artifact_id, filepath_id]) + # Link the artifact with the analysis + sql = """INSERT INTO qiita.analysis_artifact + (analysis_id, artifact_id) + VALUES (%s, %s)""" + TRN.add(sql, [analysis_id, artifact_id]) + + return artifact_id + + +def create_rarefied_biom_artifact(analysis, srare_cmd_id, biom_data, params, + parent_biom_artifact_id, rarefaction_job_id, + srare_cmd_out_id): + """Creates the rarefied biom artifact + + Parameters + ---------- + analysis : dict + The analysis information + srare_cmd_id : int + The command id of "Single Rarefaction" + biom_data : dict + The biom information + params : str + The processing parameters + parent_biom_artifact_id : int + The parent biom artifact id + rarefaction_job_id : str + The job id of the rarefaction job + srare_cmd_out_id : int + The id of the single rarefaction output + + Returns + ------- + int + The artifact id + """ + with TRN: + # Transfer the file to an artifact + # Magic number 7: artifact type -> biom + artifact_id = transfer_file_to_artifact( + analysis['analysis_id'], analysis['timestamp'], srare_cmd_id, + biom_data['data_type_id'], params, 7, biom_data['filepath_id']) + # Link the artifact with its parent + sql = """INSERT INTO qiita.parent_artifact (artifact_id, parent_id) + VALUES (%s, %s)""" + TRN.add(sql, [artifact_id, parent_biom_artifact_id]) + # Link the artifact as the job output + sql = """INSERT INTO qiita.artifact_output_processing_job + (artifact_id, processing_job_id, command_output_id) + VALUES (%s, %s, %s)""" + TRN.add(sql, [artifact_id, rarefaction_job_id, srare_cmd_out_id]) + return artifact_id + + +def transfer_job(analysis, command_id, params, input_artifact_id, job_data, + cmd_out_id, biom_data, output_artifact_type_id): + """Transfers the job from the old structure to the plugin structure + + Parameters + ---------- + analysis : dict + The analysis information + command_id : int + The id of the command executed + params : str + The parameters used in the job + input_artifact_id : int + The id of the input artifact + job_data : dict + The job information + cmd_out_id : int + The id of the command's output + biom_data : dict + The biom information + output_artifact_type_id : int + The type of the output artifact + """ + with TRN: + # Create the job + # Add the row in the processing job table + # Magic number 3: status -> success + sql = """INSERT INTO qiita.processing_job + (email, command_id, command_parameters, + processing_job_status_id) + VALUES (%s, %s, %s, %s) + RETURNING processing_job_id""" + TRN.add(sql, [analysis['email'], command_id, params, 3]) + job_id = TRN.execute_fetchlast() + + # Link the job with the input artifact + sql = """INSERT INTO qiita.artifact_processing_job + (artifact_id, processing_job_id) + VALUES (rarefied_biom_id, proc_job_id)""" + TRN.add(sql, [input_artifact_id, job_id]) + + # Check if the executed job has results and add them + sql = """SELECT EXISTS(SELECT * + FROM qiita.job_results_filepath + WHERE job_id = %s)""" + TRN.add(sql, [job_data['job_id']]) + if TRN.execute_fetchlast(): + # There are results for the current job. + # Transfer the job files to a new artifact + sql = """SELECT filepath_id + FROM qiita.job_results_filepath + WHERE job_id = %s""" + TRN.add(sql, job_data['job_id']) + filepath_id = TRN.execute_fetchlast() + artifact_id = transfer_file_to_artifact( + analysis['analysis_id'], analysis['timestamp'], command_id, + biom_data['data_type_id'], params, output_artifact_type_id, + filepath_id) + + # Link the artifact with its parent + sql = """INSERT INTO qiita.parent_artifact (artifact_id, parent_id) + VALUES (%s, %s)""" + TRN.add(sql, [artifact_id, input_artifact_id]) + # Link the artifact as the job output + sql = """INSERT INTO qiita.artifact_output_processing_job + (artifact_id, processing_job_id, command_output_id) + VALUES (%s, %s, %s)""" + TRN.add(sql, [artifact_id, job_id, cmd_out_id]) + TRN.exeucte() + else: + # There are no results on the current job, so mark it as + # error + if job_data.log_id is None: + # Magic number 2 - we are not using any other severity + # level, so keep using number 2 + sql = """INSERT INTO qiita.logging (time, severity_id, msg) + VALUES (%s, %s, %s) + RETURNING logging_id""" + TRN.add(sql, [analysis['timestamp'], 2, + "Unknown error - patch 47"]) + else: + log_id = job_data['log_id'] + + # Magic number 4 -> status -> error + sql = """UPDATE qiita.processing_job + SET processing_job_status_id = 4, logging_id = %s + WHERE processing_job_id = %s""" + TRN.add(sql, [log_id, job_id]) + + +# The new commands that we are going to add generate new artifact types. +# These new artifact types are going to be added to a different plugin. +# In interest of time and given that the artifact type system is going to +# change in the near future, we feel that the easiest way to transfer +# the current analyses results is by creating 3 different types of +# artifacts: (1) distance matrix -> which will include the distance matrix, +# the principal coordinates and the emperor plots; (2) rarefaction +# curves -> which will include all the files generated by alpha rarefaction +# and (3) taxonomy summary, which will include all the files generated +# by summarize_taxa_through_plots.py + +# Step 1: Create the new type +with TRN: + # Magic number 2 -> The "artifact definition" software type + sql = """INSERT INTO qiita.software + (name, version, description, environment_script, start_script, + software_type_id) + VALUES ('Diversity types', '0.1.0', + 'Diversity artifacts type plugin', + 'source activate qiita', 'start_diversity_types', 2) + RETURNING software_id""" + TRN.add(sql) + divtype_id = TRN.execute_fetchlast() + + # Step 2: Create the validate and HTML generator commands + sql = """INSERT INTO qiita.software_command (software_id, name, description) + VALUES (%s, %s, %s) + RETURNING command_id""" + TRN.add(sql, [divtype_id, 'Validate', + 'Validates a new artifact of the given diversity type']) + validate_cmd_id = TRN.execute_fetchlast() + TRN.add(sql, [divtype_id, 'Generate HTML summary', + 'Generates the HTML summary of a given diversity type']) + html_summary_cmd_id = TRN.execute_fetchlast() + + # Step 3: Add the parameters for the previous commands + sql = """INSERT INTO qiita.command_parameter + (command_id, parameter_name, parameter_type, required) + VALUES (%s, %s, %s, %s)""" + sql_args = [(validate_cmd_id, 'files', 'string', True), + (validate_cmd_id, 'artifact_type', 'string', True), + (html_summary_cmd_id, 'input_data', 'artifact', True)] + TRN.add(sql, sql_args, many=True) + + # Step 4: Add the new artifact types + sql = """INSERT INTO qiita.artifact_type ( + artifact_type, description, can_be_submitted_to_ebi, + can_be_submitted_to_vamps) + VALUES (%s, %s, %s, %s) + RETURNING artifact_type_id""" + TRN.add(sql, ['distance_matrix', 'Distance matrix holding pairwise ' + 'distance between samples', False, False]) + dm_atype_id = TRN.execute_fetchlast() + TRN.add(sql, ['rarefaction_curves', 'Rarefaction curves', False, False]) + rc_atype_id = TRN.execute_fetchlast() + TRN.add(sql, ['taxa_summary', 'Taxa summary plots', False, False]) + ts_atype_id = TRN.execute_fetchlast() + + # Step 5: Associate each artifact with the filetypes that it accepts + # At this time we are going to add them as directories, just as it is done + # right now. We can make it fancier with the new type system. + # Magic number 8: the filepath_type_id for the directory + sql = """INSERT INTO qiita.artifact_type_filepath_type + (artifact_type_id, filepath_type_id, required) + VALUES (%s, %s, %s)""" + sql_args = [[dm_atype_id, 8, True], + [rc_atype_id, 8, True], + [ts_atype_id, 8, True]] + TRN.add(sql, sql_args, many=True) + + # Step 6: Associate the plugin with the types that it defines + sql = """INSERT INTO qiita.software_artifact_type + (software_id, artifact_type_id) + VALUES (%s, %s)""" + sql_args = [[divtype_id, dm_atype_id], + [divtype_id, rc_atype_id], + [divtype_id, ts_atype_id]] + TRN.add(sql, sql_args, many=True) + + # Step 7: Create the new entries for the data directory + sql = """INSERT INTO qiita.data_directory + (data_type, mountpoint, subdirectory, active) + VALUES (%s, %s, %s, %s)""" + sql_args = [['distance_matrix', 'distance_matrix', True, True], + ['rarefaction_curves', 'rarefaction_curves', True, True], + ['taxa_summary', 'taxa_summary', True, True]] + TRN.add(sql, sql_args, many=True) + + # Create the new commands that execute the current analyses. In qiita, + # the only commands that where available are Summarize Taxa, Beta + # Diversity and Alpha Rarefaction. The system was executing rarefaction + # by default, but it should be a different step in the analysis process + # so we are going to create a command for it too. These commands are going + # to be part of the QIIME plugin, so we are going to first retrieve the + # id of the QIIME 1.9.1 plugin, which for sure exists cause it was added + # in patch 33 and there is no way of removing plugins + + # Step 1: Get the QIIME plugin id + sql = """SELECT software_id + FROM qiita.software + WHERE name = 'QIIME' AND version = '1.9.1'""" + TRN.add(sql) + qiime_id = TRN.execute_fetchlast() + + # Step 2: Insert the new commands in the software_command table + sql = """INSERT INTO qiita.software_command (software_id, name, description) + VALUES (%s, %s, %s) + RETURNING command_id""" + TRN.add(sql, [qiime_id, 'Summarize Taxa', 'Plots taxonomy summaries at ' + 'different taxonomy levels']) + sum_taxa_cmd_id = TRN.execute_fetchlast() + TRN.add(sql, [qiime_id, 'Beta Diversity', + 'Computes and plots beta diversity results']) + bdiv_cmd_id = TRN.execute_fetchlast() + TRN.add(sql, [qiime_id, 'Alpha Rarefaction', + 'Computes and plots alpha rarefaction results']) + arare_cmd_id = TRN.execute_fetchlast() + TRN.add(sql, [qiime_id, 'Single Rarefaction', + 'Rarefies the input table by random sampling without ' + 'replacement']) + srare_cmd_id = TRN.execute_fetchlast() + + # Step 3: Insert the parameters for each command + sql = """INSERT INTO qiita.command_parameter + (command_id, parameter_name, parameter_type, required, + default_value) + VALUES (%s, %s, %s, %s, %s) + RETURNING command_parameter_id""" + sql_args = [ + # Summarize Taxa + (sum_taxa_cmd_id, 'metadata_category', 'string', False, ''), + (sum_taxa_cmd_id, 'sort', 'bool', False, 'False'), + # Beta Diversity + (bdiv_cmd_id, 'tree', 'string', False, ''), + (bdiv_cmd_id, 'metrics', + 'mchoice:["abund_jaccard","binary_chisq","binary_chord",' + '"binary_euclidean","binary_hamming","binary_jaccard",' + '"binary_lennon","binary_ochiai","binary_otu_gain","binary_pearson",' + '"binary_sorensen_dice","bray_curtis","bray_curtis_faith",' + '"bray_curtis_magurran","canberra","chisq","chord","euclidean",' + '"gower","hellinger","kulczynski","manhattan","morisita_horn",' + '"pearson","soergel","spearman_approx","specprof","unifrac",' + '"unifrac_g","unifrac_g_full_tree","unweighted_unifrac",' + '"unweighted_unifrac_full_tree","weighted_normalized_unifrac",' + '"weighted_unifrac"]', False, '["binary_jaccard","bray_curtis"]'), + # Alpha rarefaction + (arare_cmd_id, 'tree', 'string', False, ''), + (arare_cmd_id, 'num_steps', 'integer', False, 10), + (arare_cmd_id, 'min_rare_depth', 'integer', False, 10), + (arare_cmd_id, 'max_rare_depth', 'integer', False, 'Default'), + # Single rarefaction + (srare_cmd_id, 'depth', 'integer', True, None), + (srare_cmd_id, 'subsample_multinomial', 'bool', False, 'False') + ] + TRN.add(sql, sql_args, many=True) + + TRN.add(sql, [sum_taxa_cmd_id, 'biom_table', 'artifact', True, None]) + sum_taxa_cmd_param_id = TRN.execute_fetchlast() + TRN.add(sql, [bdiv_cmd_id, 'biom_table', 'artifact', True, None]) + bdiv_cmd_param_id = TRN.execute_fetchlast() + TRN.add(sql, [arare_cmd_id, 'biom_table', 'artifact', True, None]) + arare_cmd_param_id = TRN.execute_fetchlast() + TRN.add(sql, [srare_cmd_id, 'biom_table', 'artifact', True, None]) + srare_cmd_param_id = TRN.execute_fetchlast() + + # Step 4: Connect the artifact parameters with the artifact types that + # they accept + sql = """SELECT artifact_type_id + FROM qiita.artifact_type + WHERE artifact_type = 'BIOM'""" + TRN.add(sql) + biom_atype_id = TRN.execute_fetchlast() + + sql = """INSERT INTO qiita.parameter_artifact_type + (command_parameter_id, artifact_type_id) + VALUES (%s, %s)""" + sql_args = [[sum_taxa_cmd_param_id, biom_atype_id], + [bdiv_cmd_param_id, biom_atype_id], + [arare_cmd_param_id, biom_atype_id], + [srare_cmd_param_id, biom_atype_id]] + TRN.add(sql, sql_args, many=True) + + # Step 5: Add the outputs of the command. + sql = """INSERT INTO qiita.command_output + (name, command_id, artifact_type_id) + VALUES (%s, %s, %s) + RETURNING command_output_id""" + TRN.add(sql, ['taxa_summary', sum_taxa_cmd_id, ts_atype_id]) + sum_taxa_cmd_out_id = TRN.execute_fetchlast() + TRN.add(sql, ['distance_matrix', bdiv_cmd_id, dm_atype_id]) + bdiv_cmd_out_id = TRN.execute_fetchlast() + TRN.add(sql, ['rarefaction_curves', arare_cmd_id, rc_atype_id]) + arare_cmd_out_id = TRN.execute_fetchlast() + TRN.add(sql, ['rarefied_table', srare_cmd_id, biom_atype_id]) + srare_cmd_out_id = TRN.execute_fetchlast() + +# At this point we are ready to start transferring the data from the old +# structures to the new structures. Overview of the procedure: +# Step 1: Add initial set of artifacts up to rarefied table +# Step 2: Transfer the "analisys jobs" to processing jobs and create +# the analysis artifacts +db_dir = get_db_files_base_dir() +with TRN: + sql = "SELECT * FROM qiita.analysis" + TRN.add(sql) + analysis_info = TRN.execute_fetchindex() + + # Loop through all the analysis + for analysis in analysis_info: + # Step 1: Add the inital set of artifacts. An analysis starts with + # a set of BIOM artifacts. + sql = """SELECT * + FROM qiita.analysis_filepath + JOIN qiita.filepath USING (filepath_id) + JOIN qiita.filepath_type USING (filepath_type_id) + WHERE analysis_id = %s AND filepath_type = 'biom'""" + TRN.add(sql, [analysis['analysis_id']]) + analysis_bioms = TRN.execute_fetchindex() + + # Loop through all the biom tables associated with the current analysis + # so we can create the initial set of artifacts + for biom_data in analysis_bioms: + # Get the path of the BIOM table + sql = """SELECT filepath, mountpoint + FROM qiita.filepath + JOIN qiita.data_directory USING (data_directory_id) + WHERE filepath_id = %s""" + TRN.add(sql, [biom_data['filepath_id']]) + # Magic number 0: There is only a single row in the query result + fp_info = TRN.execute_fetchindex()[0] + filepath = join(db_dir, fp_info['mountpoint'], fp_info['filepath']) + + # We need to check if the BIOM table has been rarefied or not + table = load_table(filepath) + depths = set(table.sum(axis='sample')) + if len(depths) == 1: + # The BIOM table was rarefied + # Create the initial unrarefied artifact + initial_biom_artifact_id = create_non_rarefied_biom_artifact( + analysis, biom_data, table) + # Create the rarefaction job + rarefaction_job_id, params = create_rarefaction_job( + depths.pop(), initial_biom_artifact_id, analysis, + srare_cmd_id) + # Create the rarefied artifact + rarefied_biom_artifact_id = create_rarefied_biom_artifact( + analysis, srare_cmd_id, biom_data, params, + initial_biom_artifact_id, rarefaction_job_id, + srare_cmd_out_id) + else: + # The BIOM table was not rarefied, use current table as initial + initial_biom_id = transfer_file_to_artifact() + + # Loop through all the jobs that used this biom table as input + sql = """SELECT * + FROM qiita.job + WHERE reverse(split_part(reverse( + options::json->>'--otu_table_fp'), '/', 1)) = %s""" + TRN.add(sql, [filepath]) + analysis_jobs = TRN.execute_fetchindex() + for job_data in analysis_jobs: + # Identify which command the current job exeucted + if job_data['command_id'] == 1: + # Taxa summaries + cmd_id = sum_taxa_cmd_id + params = ('{"biom_table":%d,"metadata_category":"",' + '"sort":false}' % initial_biom_id) + output_artifact_type_id = ts_atype_id + cmd_out_id = sum_taxa_cmd_out_id + elif job_data['command_id'] == 2: + # Beta diversity + cmd_id = bdiv_cmd_id + tree_fp = loads(job_data['options'])['--tree_fp'] + if tree_fp: + params = ('{"biom_table":%d,"tree":"%s","metrics":' + '["unweighted_unifrac","weighted_unifrac"]}' + % (initial_biom_id, tree_fp)) + else: + params = ('{"biom_table":%d,"metrics":["bray_curtis",' + '"gower","canberra","pearson"]}' + % initial_biom_id) + output_artifact_type_id = dm_atype_id + cmd_out_id = bdiv_cmd_out_id + else: + # Alpha rarefaction + cmd_id = arare_cmd_id + tree_fp = loads(job_data['options'])['--tree_fp'] + params = ('{"biom_table":%d,"tree":"%s","num_steps":"10",' + '"min_rare_depth":"10",' + '"max_rare_depth":"Default"}' + % (initial_biom_id, tree_fp)) + output_artifact_type_id = rc_atype_id + cmd_out_id = arare_cmd_out_id + + transfer_job(analysis, cmd_id, params, initial_biom_id, + job_data, cmd_out_id, biom_data, + output_artifact_type_id) + +errors = [] +with TRN: + # Unlink the analysis from the biom table filepaths + # Magic number 7 -> biom filepath type + sql = """DELETE FROM qiita.analysis_filepath + WHERE filepath_id IN (SELECT filepath_id + FROM qiita.filepath + WHERE filepath_type_id = 7)""" + TRN.add(sql) + TRN.execute() + + # Delete old structures that are not used anymore + tables = ["collection_job", "collection_analysis", "collection_users", + "collection", "collection_status", "analysis_workflow", + "analysis_chain", "analysis_job", "job_results_filepath", "job", + "job_status", "command_data_type", "command", "analysis_status"] + for table in tables: + TRN.add("DROP TABLE qiita.%s" % table) + try: + TRN.execute() + except Exception as e: + errors.append("Error deleting table %s: %s" % (table, str(e))) + +# Purge filepaths +try: + purge_filepaths() +except Exception as e: + errors.append("Error purging filepaths: %s" % str(e)) + +if errors: + print "\n".join(errors) diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql index a04051ba4..24f46b77a 100644 --- a/qiita_db/support_files/populate_test_db.sql +++ b/qiita_db/support_files/populate_test_db.sql @@ -11,7 +11,7 @@ INSERT INTO qiita.qiita_user (email, user_level_id, password, name, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Dude', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344'), - ('shared@foo.bar', 3, + ('shared@foo.bar', 4, '$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe', 'Shared', 'Nowhere University', '123 fake st, Apt 0, Faketown, CO 80302', '111-222-3344'), diff --git a/qiita_db/support_files/qiita-db.dbs b/qiita_db/support_files/qiita-db.dbs index 6e65c748b..ca22db34c 100644 --- a/qiita_db/support_files/qiita-db.dbs +++ b/qiita_db/support_files/qiita-db.dbs @@ -863,6 +863,26 @@ + + + + + + + + + + + + + + + + + + + +
What portals are available to show a study in @@ -1491,6 +1511,23 @@ Controlled Vocabulary]]>
+ + + + + + + + + + + + + + + + +
Links shared studies to users they are shared with @@ -1584,85 +1621,99 @@ Controlled Vocabulary]]> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + analysis tables @@ -1785,4 +1836,4 @@ ALTER TABLE oauth_software ADD CONSTRAINT fk_oauth_software FOREIGN KEY ( client ]]> - \ No newline at end of file + diff --git a/qiita_db/support_files/qiita-db.html b/qiita_db/support_files/qiita-db.html index cb6b7ce1b..b7b76b61e 100644 --- a/qiita_db/support_files/qiita-db.html +++ b/qiita_db/support_files/qiita-db.html @@ -97,7 +97,7 @@ - +
+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table analysis_users
Links analyses to the users they are shared with
analysis_id bigint NOT NULL
email varchar NOT NULL
Indexes
idx_analysis_users primary key ON analysis_id, email
idx_analysis_users_analysis ON analysis_id
idx_analysis_users_email ON email
Foreign Keys
fk_analysis_users_analysis ( analysis_id ) ref analysis (analysis_id)
fk_analysis_users_user ( email ) ref qiita_user (email)
+

@@ -2224,23 +2282,81 @@

- - + + - - + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table column_ontology
This table relates a column with an ontology.
Table analysis_filepath
Stores link between analysis and the data file used for the analysis.
column_name varchar NOT NULL analysis_id bigint NOT NULL
ontology_short_name varchar NOT NULL filepath_id bigint NOT NULL
bioportal_id integer NOT NULL data_type_id bigint
Indexes
idx_analysis_filepath ON analysis_id
idx_analysis_filepath_0 ON filepath_id
idx_analysis_filepath_1 primary key ON analysis_id, filepath_id
idx_analysis_filepath_2 ON data_type_id
Foreign Keys
fk_analysis_filepath ( analysis_id ) ref analysis (analysis_id)
fk_analysis_filepath_0 ( filepath_id ) ref filepath (filepath_id)
fk_analysis_filepath_1 ( data_type_id ) ref data_type (data_type_id)
+ +

+ + + + + + + + + + + + + + + + + + + @@ -2290,6 +2406,64 @@
Table column_ontology
This table relates a column with an ontology.
column_name varchar NOT NULL
ontology_short_name varchar NOT NULL
bioportal_id integer NOT NULL
+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table ontology
ontology_id bigint NOT NULL
ontology varchar NOT NULL
fully_loaded bool NOT NULL
fullname varchar
query_url varchar
source_url varchar
definition text
load_date date NOT NULL
Indexes
pk_ontology primary key ON ontology_id
idx_ontology unique ON ontology
+

@@ -2718,6 +2892,76 @@
+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table portal_type
What portals are available to show a study in
portal_type_id bigserial NOT NULL
portal varchar NOT NULL
portal_description varchar NOT NULL
Indexes
pk_portal_type primary key ON portal_type_id
+ +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table analysis_portal
Controls what analyses are visible on what portals
analysis_id bigint NOT NULL
portal_type_id bigint NOT NULL
Indexes
idx_analysis_portal ON analysis_id
idx_analysis_portal_0 ON portal_type_id
Foreign Keys
fk_analysis_portal ( analysis_id ) ref analysis (analysis_id)
fk_analysis_portal_0 ( portal_type_id ) ref portal_type (portal_type_id)
+

@@ -2974,6 +3218,63 @@
+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table analysis_sample
analysis_id bigint NOT NULL
artifact_id bigint NOT NULL
sample_id varchar NOT NULL
Indexes
idx_analysis_sample ON analysis_id
idx_analysis_sample_0 ON artifact_id
idx_analysis_sample_1 ON sample_id
pk_analysis_sample primary key ON analysis_id, artifact_id, sample_id
Foreign Keys
fk_analysis_sample_analysis ( analysis_id ) ref analysis (analysis_id)
fk_analysis_sample ( sample_id ) ref study_sample (sample_id)
fk_analysis_sample_artifact ( artifact_id ) ref artifact (artifact_id)
+

@@ -3282,146 +3583,18 @@

- - + - - - + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table study_person
Contact information for the various people involved in a study
Table environmental_package
study_person_id bigserial NOT NULL environmental_package_name varchar NOT NULL The name of the environmental package
namemetadata_table varchar NOT NULL
email varchar NOT NULL
affiliation varchar NOT NULL The institution with which this person is affiliated
address varchar( 100 )
phone varchar
Indexes
pk_study_person primary key ON study_person_id
idx_study_person unique ON name, affiliation
- -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table study_experimental_factor
EFO ontological link of experimental factors to studies
study_id bigint NOT NULL
efo_id bigint NOT NULL
Indexes
idx_study_experimental_factor primary key ON study_id, efo_id
idx_study_experimental_factor_0 ON study_id
Foreign Keys
fk_study_experimental_factor ( study_id ) ref study (study_id)
- -

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table study_environmental_package
Holds the 1 to many relationship between the study and the environmental_package
study_id bigint NOT NULL
environmental_package_name varchar NOT NULL
Indexes
pk_study_environmental_package primary key ON study_id, environmental_package_name
idx_study_environmental_package ON study_id
idx_study_environmental_package_0 ON environmental_package_name
Foreign Keys
fk_study_environmental_package ( study_id ) ref study (study_id)
fk_study_environmental_package_0 ( environmental_package_name ) ref environmental_package (environmental_package_name)
- -

- - - - - - - - - - - - - - + @@ -3431,49 +3604,6 @@
Table environmental_package
environmental_package_name varchar NOT NULL The name of the environmental package
metadata_table varchar NOT NULL Contains the name of the table that contains the pre-defined metadata columns for the environmental package Contains the name of the table that contains the pre-defined metadata columns for the environmental package
Indexes
pk_environmental_package primary key
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table investigation_study
investigation_id bigint NOT NULL
study_id bigint NOT NULL
Indexes
idx_investigation_study primary key ON investigation_id, study_id
idx_investigation_study_investigation ON investigation_id
idx_investigation_study_study ON study_id
Foreign Keys
fk_investigation_study ( investigation_id ) ref investigation (investigation_id)
fk_investigation_study_study ( study_id ) ref study (study_id)
-

@@ -3563,6 +3693,34 @@
+

+ + + + + + + + + + + + + + + + + + + + + + + + + +
Table data_type
data_type_id bigserial NOT NULL
data_type varchar NOT NULL Data type (16S, metabolome, etc) the job will use
Indexes
pk_data_type primary key ON data_type_id
idx_data_type unique ON data_type
+

@@ -3708,39 +3866,6 @@
-

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Table timeseries_type
timeseries_type_id bigserial NOT NULL
timeseries_type varchar NOT NULL
intervention_type varchar NOT NULL DEFO 'None'
Indexes
pk_timeseries_type primary key ON timeseries_type_id
idx_timeseries_type unique ON timeseries_type, intervention_type
-

@@ -3811,27 +3936,115 @@

- + + - + + + + + + - + - - - + + + - - + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table ebi_run_accession
Table analysis
Holds analysis information
sample_idanalysis_id bigserial NOT NULL Unique identifier for analysis
email varchar NOT NULL Email for user who owns the analysis
artifact_id bigint NOT NULL name varchar NOT NULL Name of the analysis
ebi_run_accession bigint NOT NULL description varchar NOT NULL
Indexes
idx_ebi_run_accession primary key ON sample_id, artifact_id, ebi_run_accession
pmid varchar PMID of paper from the analysis
timestamp timestamptz DEFO current_timestamp
dflt bool NOT NULL DEFO false
portal_type_id bigint NOT NULL
logging_id bigint
Indexes
pk_analysis primary key ON analysis_id
idx_analysis_email ON email
idx_analysis ON portal_type_id
idx_analysis_0 ON logging_id
Foreign Keys
fk_analysis_user ( email ) ref qiita_user (email)
fk_analysis ( portal_type_id ) ref portal_type (portal_type_id)
fk_analysis_logging ( logging_id ) ref logging (logging_id)
+ +

+ + + + + + + + + + + + + + + + + + + + + + + @@ -5299,42 +5512,42 @@

Table ebi_run_accession
sample_id varchar NOT NULL
artifact_id bigint NOT NULL
ebi_run_accession bigint NOT NULL
Indexes
idx_ebi_run_accession primary key ON sample_id, artifact_id, ebi_run_accession
idx_ebi_run_accession
- - + + - + - + - - + + - - + + - - + + - - + + - - + + @@ -5343,96 +5556,47 @@

Table analysis_users
Links analyses to the users they are shared with
Table study_environmental_package
Holds the 1 to many relationship between the study and the environmental_package
analysis_idstudy_id bigint NOT NULL
emailenvironmental_package_name varchar NOT NULL
Indexes
idx_analysis_users primary key ON analysis_id, email
pk_study_environmental_package primary key ON study_id, environmental_package_name
idx_analysis_users_analysis ON analysis_id
idx_study_environmental_package ON study_id
idx_analysis_users_email ON email
idx_study_environmental_package_0 ON environmental_package_name
Foreign Keys
fk_analysis_users_analysis ( analysis_id ) ref analysis (analysis_id) fk_study_environmental_package ( study_id ) ref study (study_id)
fk_analysis_users_user ( email ) ref qiita_user (email) fk_study_environmental_package_0 ( environmental_package_name ) ref environmental_package (environmental_package_name)
- - + + - - - - - - - - - - - - - - - - + + - - - + + - - + + - -
Table analysis_portal
Controls what analyses are visible on what portals
Table study_person
Contact information for the various people involved in a study
analysis_id bigint NOT NULL
portal_type_id bigint NOT NULL
Indexes
idx_analysis_portal ON analysis_id
idx_analysis_portal_0 ON portal_type_idstudy_person_id bigserial NOT NULL
Foreign Keys
fk_analysis_portal ( analysis_id ) ref analysis (analysis_id) name varchar NOT NULL
fk_analysis_portal_0 ( portal_type_id ) ref portal_type (portal_type_id) email varchar NOT NULL
- -

- - - - - - - - - + + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - + + @@ -5441,85 +5605,41 @@

Table analysis_filepath
Stores link between analysis and the data file used for the analysis.
analysis_id bigint NOT NULL affiliation varchar NOT NULL The institution with which this person is affiliated
filepath_id bigint NOT NULL address varchar( 100 )
data_type_id bigint phone varchar
Indexes
idx_analysis_filepath ON analysis_id
idx_analysis_filepath_0 ON filepath_id
idx_analysis_filepath_1 primary key ON analysis_id, filepath_id
idx_analysis_filepath_2 ON data_type_id
Foreign Keys
fk_analysis_filepath ( analysis_id ) ref analysis (analysis_id)
fk_analysis_filepath_0 ( filepath_id ) ref filepath (filepath_id)
pk_study_person primary key ON study_person_id
fk_analysis_filepath_1 ( data_type_id ) ref data_type (data_type_id)
idx_study_person unique ON name, affiliation
- + - + - + - - - - - - - - - - - + + - - + + - - + + - - - - - - - - - - - - - - - -
Table analysis_sample
Table investigation_study
analysis_idinvestigation_id bigint NOT NULL
artifact_idstudy_id bigint NOT NULL
sample_id varchar NOT NULL
Indexes
idx_analysis_sample ON analysis_id
idx_analysis_sample_0 ON artifact_id
idx_investigation_study primary key ON investigation_id, study_id
idx_analysis_sample_1 ON sample_id
idx_investigation_study_investigation ON investigation_id
pk_analysis_sample primary key ON analysis_id, artifact_id, sample_id
idx_investigation_study_study ON study_id
Foreign Keys
fk_analysis_sample_analysis ( analysis_id ) ref analysis (analysis_id)
fk_analysis_sample ( sample_id ) ref study_sample (sample_id)
fk_analysis_sample_artifact ( artifact_id ) ref artifact (artifact_id)
- -

- - - - - - - - - - - - - - + + - - - - - - - + + @@ -5528,56 +5648,41 @@

Table portal_type
What portals are available to show a study in
portal_type_id bigserial NOT NULL
portal varchar NOT NULL fk_investigation_study ( investigation_id ) ref investigation (investigation_id)
portal_description varchar NOT NULL
Indexes
pk_portal_type primary key ON portal_type_idfk_investigation_study_study ( study_id ) ref study (study_id)
- + - + - - - - - - - + + - - - + + + - - - + + - - - + + + - - + + - - - - - - - - - - - + + @@ -5586,41 +5691,33 @@

Table ontology
Table per_study_tags
ontology_idstudy_tag_id bigint NOT NULL
ontology varchar NOT NULL
fully_loaded bool NOT NULL study_id integer NOT NULL
fullname varchar
Indexes
pk_per_study_tags primary key ON study_tag_id, study_id
query_url varchar
idx_per_study_tags ON study_id
source_url varchar
idx_per_study_tags ON study_tag_id
Foreign Keys
definition text fk_per_study_tags_study ( study_id ) ref study (study_id)
load_date date NOT NULL
Indexes
pk_ontology primary key ON ontology_id
idx_ontology unique ON ontologyfk_per_study_tags_study_tags ( study_tag_id ) ref study_tags (study_tag_id)
- + + - + - + - - - - - - + + - - + + - - - - - - - + + @@ -5629,114 +5726,31 @@

Table analysis_artifact
Table study_experimental_factor
EFO ontological link of experimental factors to studies
analysis_idstudy_id bigint NOT NULL
artifact_idefo_id bigint NOT NULL
Indexes
idx_analysis_artifact ON analysis_id
idx_analysis_artifact ON artifact_id
idx_study_experimental_factor primary key ON study_id, efo_id
idx_analysis_artifact_0 primary key ON analysis_id, artifact_id
idx_study_experimental_factor_0 ON study_id
Foreign Keys
fk_analysis_artifact_analysis ( analysis_id ) ref analysis (analysis_id)
fk_analysis_artifact_artifact ( artifact_id ) ref artifact (artifact_id) fk_study_experimental_factor ( study_id ) ref study (study_id)
- + - + - - - - - - - - - - - - - - -
Table data_type
Table timeseries_type
data_type_idtimeseries_type_id bigserial NOT NULL
data_type varchar NOT NULL Data type (16S, metabolome, etc) the job will use
Indexes
pk_data_type primary key ON data_type_id
idx_data_type unique ON data_type
- -

- - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - + + @@ -5745,41 +5759,41 @@

Table analysis
Holds analysis information
analysis_id bigserial NOT NULL Unique identifier for analysis
email varchar NOT NULL Email for user who owns the analysis
name varchar NOT NULL Name of the analysis
descriptiontimeseries_type varchar NOT NULL
pmid varchar PMID of paper from the analysis
timestamp timestamptz DEFO current_timestamp
dflt bool NOT NULL DEFO false
portal_type_id bigint NOT NULL
logging_id bigint intervention_type varchar NOT NULL DEFO 'None'
Indexes
pk_analysis primary key ON analysis_id
idx_analysis_email ON email
idx_analysis ON portal_type_id
idx_analysis_0 ON logging_id
Foreign Keys
fk_analysis_user ( email ) ref qiita_user (email)
fk_analysis ( portal_type_id ) ref portal_type (portal_type_id)
pk_timeseries_type primary key ON timeseries_type_id
fk_analysis_logging ( logging_id ) ref logging (logging_id)
idx_timeseries_type unique ON timeseries_type, intervention_type
- + - - + + - + - - - + + + - - + + + - - + + - - - - + + + - - + + diff --git a/qiita_db/test/test_analysis.py b/qiita_db/test/test_analysis.py index 3b31fff9b..2fbce2187 100644 --- a/qiita_db/test/test_analysis.py +++ b/qiita_db/test/test_analysis.py @@ -1,14 +1,9 @@ from unittest import TestCase, main -from tempfile import mkstemp -from os import remove, close +from os import remove from os.path import exists, join, basename from shutil import move -from time import sleep -from datetime import datetime -from future.utils import viewitems from biom import load_table -import pandas as pd from pandas.util.testing import assert_frame_equal from functools import partial import numpy.testing as npt diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py index 2423e9d4e..a9173440e 100644 --- a/qiita_db/test/test_artifact.py +++ b/qiita_db/test/test_artifact.py @@ -46,7 +46,7 @@ def test_iter_public(self): def test_create_type(self): obs = qdb.artifact.Artifact.types() exp = [['BIOM', 'BIOM table'], - ['Demultiplexed', 'Demultiplexed and QC sequeneces'], + ['Demultiplexed', 'Demultiplexed and QC sequences'], ['FASTA', None], ['FASTA_Sanger', None], ['FASTQ', None], ['SFF', None], ['per_sample_FASTQ', None], ['distance_matrix', 'Distance matrix holding pairwise ' @@ -61,7 +61,7 @@ def test_create_type(self): obs = qdb.artifact.Artifact.types() exp = [['BIOM', 'BIOM table'], - ['Demultiplexed', 'Demultiplexed and QC sequeneces'], + ['Demultiplexed', 'Demultiplexed and QC sequences'], ['FASTA', None], ['FASTA_Sanger', None], ['FASTQ', None], ['SFF', None], ['per_sample_FASTQ', None], ['distance_matrix', 'Distance matrix holding pairwise ' @@ -733,7 +733,7 @@ def test_create_processed(self): self.assertEqual(obs.name, 'noname') self.assertTrue(before < obs.timestamp < datetime.now()) self.assertEqual(obs.processing_parameters, exp_params) - self.assertEqual(obs.visibility, 'sandbox') + self.assertEqual(obs.visibility, 'private') self.assertEqual(obs.artifact_type, "Demultiplexed") self.assertEqual(obs.data_type, qdb.artifact.Artifact(1).data_type) self.assertTrue(obs.can_be_submitted_to_ebi) @@ -765,7 +765,7 @@ def test_create_copy_files(self): self.assertEqual(obs.name, 'noname') self.assertTrue(before < obs.timestamp < datetime.now()) self.assertEqual(obs.processing_parameters, exp_params) - self.assertEqual(obs.visibility, 'sandbox') + self.assertEqual(obs.visibility, 'private') self.assertEqual(obs.artifact_type, "Demultiplexed") self.assertEqual(obs.data_type, qdb.artifact.Artifact(1).data_type) self.assertTrue(obs.can_be_submitted_to_ebi) @@ -797,7 +797,7 @@ def test_create_biom(self): self.assertEqual(obs.name, 'noname') self.assertTrue(before < obs.timestamp < datetime.now()) self.assertEqual(obs.processing_parameters, exp_params) - self.assertEqual(obs.visibility, 'sandbox') + self.assertEqual(obs.visibility, 'private') self.assertEqual(obs.artifact_type, 'BIOM') self.assertEqual(obs.data_type, qdb.artifact.Artifact(2).data_type) self.assertFalse(obs.can_be_submitted_to_ebi) diff --git a/qiita_db/test/test_base.py b/qiita_db/test/test_base.py index d001e5f1a..db606013f 100644 --- a/qiita_db/test/test_base.py +++ b/qiita_db/test/test_base.py @@ -45,8 +45,6 @@ def test_check_subclass_error(self): # Checked through the __init__ call with self.assertRaises(IncompetentQiitaDeveloperError): qdb.base.QiitaObject(1) - with self.assertRaises(IncompetentQiitaDeveloperError): - qdb.base.QiitaStatusObject(1) def test_check_id(self): """Correctly checks if an id exists on the database""" diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index 4feec4d12..02c770b16 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -10,6 +10,7 @@ import pandas as pd +from moi import r_client from qiita_core.qiita_settings import qiita_config from qiita_core.util import qiita_test_checker @@ -32,47 +33,37 @@ def _set_artifact_public(self): self.conn_handler.execute( "UPDATE qiita.artifact SET visibility_id=2") - def _unshare_studies(self): - self.conn_handler.execute("DELETE FROM qiita.study_users") - - def _unshare_analyses(self): - self.conn_handler.execute("DELETE FROM qiita.analysis_users") - - def test_get_accessible_filepath_ids(self): + def test_validate_filepath_access_by_user(self): self._set_artifact_private() # shared has access to all study files and analysis files - - obs = qdb.meta_util.get_accessible_filepath_ids( - qdb.user.User('shared@foo.bar')) - self.assertItemsEqual(obs, { - 1, 2, 3, 4, 5, 9, 12, 16, 17, 18, 19, 20, 21}) + user = qdb.user.User('shared@foo.bar') + for i in [1, 2, 3, 4, 5, 9, 12, 15, 16, 17, 18, 19, 20, 21]: + self.assertTrue(qdb.meta_util.validate_filepath_access_by_user( + user, i)) # Now shared should not have access to the study files - self._unshare_studies() - obs = qdb.meta_util.get_accessible_filepath_ids( - qdb.user.User('shared@foo.bar')) - self.assertItemsEqual(obs, {16}) + qdb.study.Study(1).unshare(user) + for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]: + self.assertFalse(qdb.meta_util.validate_filepath_access_by_user( + user, i)) + for i in [15, 16]: + self.assertTrue(qdb.meta_util.validate_filepath_access_by_user( + user, i)) # Now shared should not have access to any files - self._unshare_analyses() - obs = qdb.meta_util.get_accessible_filepath_ids( - qdb.user.User('shared@foo.bar')) - self.assertEqual(obs, set()) + qdb.analysis.Analysis(1).unshare(user) + for i in [1, 2, 3, 4, 5, 9, 12, 15, 16, 17, 18, 19, 20, 21]: + self.assertFalse(qdb.meta_util.validate_filepath_access_by_user( + user, i)) # Now shared has access to public study files self._set_artifact_public() - obs = qdb.meta_util.get_accessible_filepath_ids( - qdb.user.User('shared@foo.bar')) - self.assertEqual( - obs, {1, 2, 3, 4, 5, 9, 12, 15, 16, 17, 18, 19, 20, 21, 22}) + for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]: + self.assertTrue(qdb.meta_util.validate_filepath_access_by_user( + user, i)) # Test that it doesn't break: if the SampleTemplate hasn't been added - exp = {1, 2, 3, 4, 5, 9, 12, 15, 16, 17, 18, 19, 20, 21, 22} - obs = qdb.meta_util.get_accessible_filepath_ids( - qdb.user.User('test@foo.bar')) - self.assertEqual(obs, exp) - info = { "timeseries_type_id": 1, "metadata_complete": True, @@ -86,28 +77,31 @@ def test_get_accessible_filepath_ids(self): "principal_investigator_id": 1, "lab_person_id": 1 } - qdb.study.Study.create( + study = qdb.study.Study.create( qdb.user.User('test@foo.bar'), "Test study", [1], info) - obs = qdb.meta_util.get_accessible_filepath_ids( - qdb.user.User('test@foo.bar')) - self.assertEqual(obs, exp) + for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]: + self.assertTrue(qdb.meta_util.validate_filepath_access_by_user( + user, i)) # test in case there is a prep template that failed self.conn_handler.execute( "INSERT INTO qiita.prep_template (data_type_id) VALUES (2)") - obs = qdb.meta_util.get_accessible_filepath_ids( - qdb.user.User('test@foo.bar')) - self.assertEqual(obs, exp) + for i in [1, 2, 3, 4, 5, 9, 12, 17, 18, 19, 20, 21]: + self.assertTrue(qdb.meta_util.validate_filepath_access_by_user( + user, i)) # admin should have access to everything - count = self.conn_handler.execute_fetchone( - "SELECT last_value FROM qiita.filepath_filepath_id_seq")[0] - exp = set(range(1, count + 1)) - exp.discard(13) - exp.discard(14) - obs = qdb.meta_util.get_accessible_filepath_ids( - qdb.user.User('admin@foo.bar')) - self.assertEqual(obs, exp) + admin = qdb.user.User('admin@foo.bar') + fids = self.conn_handler.execute_fetchall( + "SELECT filepath_id FROM qiita.filepath") + for i in fids: + self.assertTrue(qdb.meta_util.validate_filepath_access_by_user( + admin, i[0])) + + # returning to origina sharing + qdb.study.Study(1).share(user) + qdb.analysis.Analysis(1).share(user) + qdb.study.Study.delete(study.id) def test_get_lat_longs(self): exp = [ @@ -173,7 +167,7 @@ def test_get_lat_longs_EMP_portal(self): } md_ext = pd.DataFrame.from_dict(md, orient='index', dtype=str) - qdb.metadata_template.sample_template.SampleTemplate.create( + st = qdb.metadata_template.sample_template.SampleTemplate.create( md_ext, study) qiita_config.portal = 'EMP' @@ -182,7 +176,46 @@ def test_get_lat_longs_EMP_portal(self): exp = [[42.42, 41.41]] self.assertItemsEqual(obs, exp) - + qdb.metadata_template.sample_template.SampleTemplate.delete(st.id) + qdb.study.Study.delete(study.id) + + def test_update_redis_stats(self): + qdb.meta_util.update_redis_stats() + + portal = qiita_config.portal + vals = [ + ('number_studies', {'sanbox': '0', 'public': '1', + 'private': '0'}, r_client.hgetall), + ('number_of_samples', {'sanbox': '0', 'public': '27', + 'private': '0'}, r_client.hgetall), + ('num_users', '4', r_client.get), + ('lat_longs', EXP_LAT_LONG, r_client.get), + ('num_studies_ebi', '1', r_client.get), + ('num_samples_ebi', '27', r_client.get), + ('number_samples_ebi_prep', '54', r_client.get) + # not testing img/time for simplicity + # ('img', r_client.get), + # ('time', r_client.get) + ] + for k, exp, f in vals: + redis_key = '%s:stats:%s' % (portal, k) + self.assertEqual(f(redis_key), exp) + + +EXP_LAT_LONG = ( + '[[60.1102854322, 74.7123248382], [23.1218032799, 42.838497795],' + ' [3.21190859967, 26.8138925876], [74.0894932572, 65.3283470202],' + ' [53.5050692395, 31.6056761814], [12.6245524972, 96.0693176066],' + ' [43.9614715197, 82.8516734159], [10.6655599093, 70.784770579],' + ' [78.3634273709, 74.423907894], [82.8302905615, 86.3615778099],' + ' [44.9725384282, 66.1920014699], [4.59216095574, 63.5115213108],' + ' [57.571893782, 32.5563076447], [40.8623799474, 6.66444220187],' + ' [95.2060749748, 27.3592668624], [38.2627021402, 3.48274264219],' + ' [13.089194595, 92.5274472082], [84.0030227585, 66.8954849864],' + ' [68.51099627, 2.35063674718], [29.1499460692, 82.1270418227],' + ' [35.2374368957, 68.5041623253], [12.7065957714, 84.9722975792],' + ' [0.291867635913, 68.5945325743], [85.4121476399, 15.6526750776],' + ' [68.0991287718, 34.8360987059]]') if __name__ == '__main__': main() diff --git a/qiita_db/test/test_reference.py b/qiita_db/test/test_reference.py index e7cf286c1..d733d276f 100644 --- a/qiita_db/test/test_reference.py +++ b/qiita_db/test/test_reference.py @@ -38,21 +38,20 @@ def tearDown(self): def test_create(self): """Correctly creates the rows in the DB for the reference""" - fp_count = qdb.util.get_count('qiita.filepath') # Check that the returned object has the correct id obs = qdb.reference.Reference.create( self.name, self.version, self.seqs_fp, self.tax_fp, self.tree_fp) self.assertEqual(obs.id, 3) - seqs_id = fp_count + 1 - tax_id = fp_count + 2 - tree_id = fp_count + 3 - # Check that the information on the database is correct obs = self.conn_handler.execute_fetchall( "SELECT * FROM qiita.reference WHERE reference_id=3") - exp = [[3, self.name, self.version, seqs_id, tax_id, tree_id]] - self.assertEqual(obs, exp) + self.assertEqual(obs[0][1], self.name) + self.assertEqual(obs[0][2], self.version) + + seqs_id = obs[0][3] + tax_id = obs[0][4] + tree_id = obs[0][5] # Check that the filepaths have been correctly added to the DB obs = self.conn_handler.execute_fetchall( diff --git a/qiita_db/test/test_software.py b/qiita_db/test/test_software.py index 0b1ea84c7..369d6d0cc 100644 --- a/qiita_db/test/test_software.py +++ b/qiita_db/test/test_software.py @@ -359,7 +359,10 @@ def test_description(self): def test_commands(self): exp = [qdb.software.Command(1), qdb.software.Command(2), qdb.software.Command(3)] - self.assertEqual(qdb.software.Software(1).commands, exp) + obs = qdb.software.Software(1).commands + self.assertEqual(len(obs), 7) + for e in exp: + self.assertIn(e, obs) def test_get_command(self): s = qdb.software.Software(1) diff --git a/qiita_db/test/test_study.py b/qiita_db/test/test_study.py index 5f2cbcd66..2ccf23c67 100644 --- a/qiita_db/test/test_study.py +++ b/qiita_db/test/test_study.py @@ -832,6 +832,33 @@ def test_environmental_packages_sandboxed(self): with self.assertRaises(qdb.exceptions.QiitaDBStatusError): self.study.environmental_packages = ['air'] + def test_study_tags(self): + # inserting new tags + user = qdb.user.User('test@foo.bar') + tags = ['this is my tag', 'I want GOLD!!'] + qdb.study.Study.insert_tags(user, tags) + + # testing that insertion went fine + obs = qdb.study.Study.get_tags() + exp = [[i + 1, tag] for i, tag in enumerate(tags)] + self.assertEqual(obs, exp) + + # assigning the tags to study + study = qdb.study.Study(1) + self.assertEqual(study.tags, []) + study.tags = [tig for tig, tag in obs] + # and checking that everything went fine + self.assertEqual(obs, study.tags) + + # making sure that everything is overwritten + obs.pop() + study.tags = [tig for tig, tag in obs] + self.assertEqual(obs, study.tags) + + # cleaning tags + study.tags = [] + self.assertEqual(study.tags, []) + if __name__ == "__main__": main() diff --git a/qiita_db/test/test_user.py b/qiita_db/test/test_user.py index 449f1535d..f15d70459 100644 --- a/qiita_db/test/test_user.py +++ b/qiita_db/test/test_user.py @@ -450,14 +450,26 @@ def test_user_artifacts(self): qdb.artifact.Artifact(7)]} self.assertEqual(obs, exp) - def test_jobs(self): + def test_jobs_all(self): PJ = qdb.processing_job.ProcessingJob + ignore_status = [] # generates expected jobs - jobs = qdb.user.User('shared@foo.bar').jobs() + jobs = qdb.user.User('shared@foo.bar').jobs(ignore_status) self.assertEqual(jobs, [ PJ('d19f76ee-274e-4c1b-b3a2-a12d73507c55'), PJ('b72369f9-a886-4193-8d3d-f7b504168e75')]) + # no jobs + self.assertEqual(qdb.user.User('admin@foo.bar').jobs( + ignore_status), []) + + def test_jobs_defaults(self): + PJ = qdb.processing_job.ProcessingJob + # generates expected jobs + jobs = qdb.user.User('shared@foo.bar').jobs() + self.assertEqual(jobs, [ + PJ('d19f76ee-274e-4c1b-b3a2-a12d73507c55')]) + # no jobs self.assertEqual(qdb.user.User('admin@foo.bar').jobs(), []) diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 4c22e8f57..190f2e056 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -8,12 +8,13 @@ from unittest import TestCase, main from tempfile import mkstemp -from os import close, remove, makedirs +from os import close, remove, makedirs, mkdir from os.path import join, exists, basename from shutil import rmtree from datetime import datetime from functools import partial from string import punctuation +from tarfile import open as topen import pandas as pd @@ -370,6 +371,20 @@ def _common_purge_filpeaths_test(self): def test_purge_filepaths(self): self._common_purge_filpeaths_test() + def test_empty_trash_upload_folder(self): + # creating file to delete so we know it actually works + study_id = '1' + uploads_fp = join(qdb.util.get_mountpoint("uploads")[0][1], study_id) + trash = join(uploads_fp, 'trash') + if not exists(trash): + mkdir(trash) + fp = join(trash, 'my_file_to_delete.txt') + open(fp, 'w').close() + + self.assertTrue(exists(fp)) + qdb.util.empty_trash_upload_folder() + self.assertFalse(exists(fp)) + def test_purge_filepaths_null_cols(self): # For more details about the source of the issue that motivates this # test: http://www.depesz.com/2008/08/13/nulls-vs-not-in/ @@ -726,6 +741,42 @@ def test_supported_filepath_types(self): exp = [["biom", True], ["directory", False], ["log", False]] self.assertItemsEqual(obs, exp) + def test_generate_biom_and_metadata_release(self): + tgz, txt = qdb.util.generate_biom_and_metadata_release('private') + self.files_to_remove.extend([tgz, txt]) + + tmp = topen(tgz, "r:gz") + tgz_obs = [ti.name for ti in tmp] + tmp.close() + tgz_exp = [ + 'processed_data/1_study_1001_closed_reference_otu_table.biom', + 'templates/1_19700101-000000.txt', + 'templates/1_prep_1_19700101-000000.txt', + 'processed_data/1_study_1001_closed_reference_otu_table.biom', + 'templates/1_19700101-000000.txt', + 'templates/1_prep_1_19700101-000000.txt', + 'processed_data/1_study_1001_closed_reference_otu_table_' + 'Silva.biom', 'templates/1_19700101-000000.txt', + 'templates/1_prep_1_19700101-000000.txt'] + self.assertEqual(tgz_obs, tgz_exp) + + tmp = open(txt) + txt_obs = tmp.readlines() + tmp.close() + txt_exp = [ + 'biom_fp\tsample_fp\tprep_fp\tqiita_artifact_id\tcommand\n', + 'processed_data/1_study_1001_closed_reference_otu_table.biom\ttem' + 'plates/1_19700101-000000.txt\ttemplates/1_prep_1_19700101-000000' + '.txt\t4\tPick closed-reference OTUs, Split libraries FASTQ\n', + 'processed_data/1_study_1001_closed_reference_otu_table.biom\ttem' + 'plates/1_19700101-000000.txt\ttemplates/1_prep_1_19700101-000000' + '.txt\t5\tPick closed-reference OTUs, Split libraries FASTQ\n', + 'processed_data/1_study_1001_closed_reference_otu_table_Silva.bio' + 'm\ttemplates/1_19700101-000000.txt\ttemplates/1_prep_1_19700101-' + '000000.txt\t6\tPick closed-reference OTUs, Split libraries ' + 'FASTQ\n'] + self.assertEqual(txt_obs, txt_exp) + @qiita_test_checker() class UtilTests(TestCase): diff --git a/qiita_db/user.py b/qiita_db/user.py index 3c9335c3d..46b29ece8 100644 --- a/qiita_db/user.py +++ b/qiita_db/user.py @@ -660,11 +660,13 @@ def delete_messages(self, messages): qdb.sql_connection.TRN.add(sql) qdb.sql_connection.TRN.execute() - def jobs(self): + def jobs(self, ignore_status=['success']): """Return jobs created by the user Parameters ---------- + ignore_status, list of str + don't retieve jobs that have one of these status Returns ------- @@ -672,11 +674,29 @@ def jobs(self): """ with qdb.sql_connection.TRN: - sql_info = [self._id] sql = """SELECT processing_job_id FROM qiita.processing_job + LEFT JOIN qiita.processing_job_status + USING (processing_job_status_id) WHERE email = %s - ORDER BY heartbeat DESC""" + """ + + if ignore_status: + sql_info = [self._id, tuple(ignore_status)] + sql += " AND processing_job_status NOT IN %s" + else: + sql_info = [self._id] + + sql += """ + ORDER BY CASE processing_job_status + WHEN 'in_construction' THEN 1 + WHEN 'running' THEN 2 + WHEN 'queued' THEN 3 + WHEN 'waiting' THEN 4 + WHEN 'error' THEN 5 + WHEN 'success' THEN 6 + END, heartbeat DESC""" + qdb.sql_connection.TRN.add(sql, sql_info) return [qdb.processing_job.ProcessingJob(p[0]) for p in qdb.sql_connection.TRN.execute_fetchindex()] diff --git a/qiita_db/util.py b/qiita_db/util.py index ac2480bb8..2d956d3ce 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -54,8 +54,10 @@ from json import dumps from datetime import datetime from itertools import chain +from tarfile import open as topen from qiita_core.exceptions import IncompetentQiitaDeveloperError +from qiita_core.configuration_manager import ConfigurationManager import qiita_db as qdb @@ -714,9 +716,24 @@ def path_builder(db_dir, filepath, mountpoint, subdirectory, obj_id): for fpid, fp, fp_type_, m, s in results] -def purge_filepaths(): +def _rm_files(TRN, fp): + # Remove the data + if exists(fp): + if isdir(fp): + func = rmtree + else: + func = remove + TRN.add_post_commit_func(func, fp) + + +def purge_filepaths(delete_files=True): r"""Goes over the filepath table and remove all the filepaths that are not used in any place + + Parameters + ---------- + delete_files : bool + if True it will actually delete the files, if False print """ with qdb.sql_connection.TRN: # Get all the (table, column) pairs that reference to the filepath @@ -739,30 +756,58 @@ def purge_filepaths(): union_str = " UNION ".join( ["SELECT %s FROM qiita.%s WHERE %s IS NOT NULL" % (col, table, col) for table, col in qdb.sql_connection.TRN.execute_fetchindex()]) - # Get all the filepaths from the filepath table that are not - # referenced from any place in the database - sql = """SELECT filepath_id, filepath, filepath_type, data_directory_id - FROM qiita.filepath FP JOIN qiita.filepath_type FPT - ON FP.filepath_type_id = FPT.filepath_type_id - WHERE filepath_id NOT IN (%s)""" % union_str - qdb.sql_connection.TRN.add(sql) + if union_str: + # Get all the filepaths from the filepath table that are not + # referenced from any place in the database + sql = """SELECT filepath_id, filepath, filepath_type, data_directory_id + FROM qiita.filepath FP JOIN qiita.filepath_type FPT + ON FP.filepath_type_id = FPT.filepath_type_id + WHERE filepath_id NOT IN (%s)""" % union_str + qdb.sql_connection.TRN.add(sql) # We can now go over and remove all the filepaths sql = "DELETE FROM qiita.filepath WHERE filepath_id=%s" db_results = qdb.sql_connection.TRN.execute_fetchindex() for fp_id, fp, fp_type, dd_id in db_results: - qdb.sql_connection.TRN.add(sql, [fp_id]) + if delete_files: + qdb.sql_connection.TRN.add(sql, [fp_id]) + fp = join(get_mountpoint_path_by_id(dd_id), fp) + _rm_files(qdb.sql_connection.TRN, fp) + else: + print fp, fp_type - # Remove the data - fp = join(get_mountpoint_path_by_id(dd_id), fp) - if exists(fp): - if fp_type == 'directory': - func = rmtree - else: - func = remove - qdb.sql_connection.TRN.add_post_commit_func(func, fp) + if delete_files: + qdb.sql_connection.TRN.execute() - qdb.sql_connection.TRN.execute() + +def empty_trash_upload_folder(delete_files=True): + r"""Delete all files in the trash folder inside each of the upload + folders + + Parameters + ---------- + delete_files : bool + if True it will actually delete the files, if False print + """ + gfp = partial(join, get_db_files_base_dir()) + with qdb.sql_connection.TRN: + sql = """SELECT mountpoint + FROM qiita.data_directory + WHERE data_type = 'uploads'""" + qdb.sql_connection.TRN.add(sql) + + for mp in qdb.sql_connection.TRN.execute_fetchflatten(): + for path, dirs, files in walk(gfp(mp)): + if path.endswith('/trash'): + if delete_files: + for f in files: + fp = join(path, f) + _rm_files(qdb.sql_connection.TRN, fp) + else: + print files + + if delete_files: + qdb.sql_connection.TRN.execute() def move_filepaths_to_upload_folder(study_id, filepaths): @@ -1463,3 +1508,87 @@ def generate_study_list(study_ids, build_samples, public_only=False): infolist.append(info) return infolist + + +def generate_biom_and_metadata_release(study_status='public'): + """Generate a list of biom/meatadata filepaths and a tgz of those files + + Parameters + ---------- + study_status : str, optional + The study status to search for. Note that this should always be set + to 'public' but having this exposed as helps with testing. The other + options are 'private' and 'sandbox' + + Returns + ------- + str, str + tgz_name: the filepath of the new generated tgz + txt_name: the filepath of the new generated txt + """ + studies = qdb.study.Study.get_by_status(study_status) + qiita_config = ConfigurationManager() + working_dir = qiita_config.working_dir + portal = qiita_config.portal + bdir = qdb.util.get_db_files_base_dir() + bdir_len = len(bdir) + 1 + + data = [] + for s in studies: + # [0] latest is first, [1] only getting the filepath + sample_fp = s.sample_template.get_filepaths()[0][1] + if sample_fp.startswith(bdir): + sample_fp = sample_fp[bdir_len:] + + for a in s.artifacts(artifact_type='BIOM'): + if a.processing_parameters is None: + continue + + cmd_name = a.processing_parameters.command.name + + # this loop is necessary as in theory an artifact can be + # generated from multiple prep info files + human_cmd = [] + for p in a.parents: + pp = p.processing_parameters + pp_cmd_name = pp.command.name + if pp_cmd_name == 'Trimming': + human_cmd.append('%s @ %s' % ( + cmd_name, str(pp.values['length']))) + else: + human_cmd.append('%s, %s' % (cmd_name, pp_cmd_name)) + human_cmd = ', '.join(human_cmd) + + for _, fp, fp_type in a.filepaths: + if fp_type != 'biom' or 'only-16s' in fp: + continue + if fp.startswith(bdir): + fp = fp[bdir_len:] + # format: (biom_fp, sample_fp, prep_fp, qiita_artifact_id, + # human readable name) + for pt in a.prep_templates: + for _, prep_fp in pt.get_filepaths(): + if 'qiime' not in prep_fp: + break + if prep_fp.startswith(bdir): + prep_fp = prep_fp[bdir_len:] + data.append((fp, sample_fp, prep_fp, a.id, human_cmd)) + + # writing text and tgz file + ts = datetime.now().strftime('%m%d%y-%H%M%S') + tgz_dir = join(working_dir, 'releases') + if not exists(tgz_dir): + makedirs(tgz_dir) + tgz_name = join(tgz_dir, '%s-%s-%s.tgz' % (portal, study_status, ts)) + txt_name = join(tgz_dir, '%s-%s-%s.txt' % (portal, study_status, ts)) + with open(txt_name, 'w') as txt, topen(tgz_name, "w|gz") as tgz: + # writing header for txt + txt.write("biom_fp\tsample_fp\tprep_fp\tqiita_artifact_id\tcommand\n") + for biom_fp, sample_fp, prep_fp, artifact_id, human_cmd in data: + txt.write("%s\t%s\t%s\t%s\t%s\n" % ( + biom_fp, sample_fp, prep_fp, artifact_id, human_cmd)) + tgz.add(join(bdir, biom_fp), arcname=biom_fp, recursive=False) + tgz.add(join(bdir, sample_fp), arcname=sample_fp, recursive=False) + tgz.add(join(bdir, prep_fp), arcname=prep_fp, recursive=False) + + return tgz_name, txt_name diff --git a/qiita_pet/handlers/api_proxy/prep_template.py b/qiita_pet/handlers/api_proxy/prep_template.py index a3a5cc887..506162ded 100644 --- a/qiita_pet/handlers/api_proxy/prep_template.py +++ b/qiita_pet/handlers/api_proxy/prep_template.py @@ -299,7 +299,8 @@ def prep_template_summary_get_req(prep_id, user_id): Format {'status': status, 'message': message, 'num_samples': value, - 'category': [(val1, count1), (val2, count2), ...], ...} + 'category': [(val1, count1), (val2, count2), ...], + 'editable': bool} """ exists = _check_prep_template_exists(int(prep_id)) if exists['status'] != 'success': @@ -309,17 +310,21 @@ def prep_template_summary_get_req(prep_id, user_id): access_error = check_access(prep.study_id, user_id) if access_error: return access_error + + editable = Study(prep.study_id).can_edit(User(user_id)) df = prep.to_dataframe() out = {'num_samples': df.shape[0], - 'summary': {}, + 'summary': [], 'status': 'success', - 'message': ''} + 'message': '', + 'editable': editable} - cols = list(df.columns) + cols = sorted(list(df.columns)) for column in cols: counts = df[column].value_counts() - out['summary'][str(column)] = [(str(key), counts[key]) - for key in natsorted(counts.index)] + out['summary'].append( + (str(column), [(str(key), counts[key]) + for key in natsorted(counts.index)])) return out @@ -417,10 +422,11 @@ def prep_template_patch_req(user_id, req_op, req_path, req_value=None, Returns ------- - dict of {str, str} + dict of {str, str, str} A dictionary with the following keys: - status: str, whether if the request is successful or not - message: str, if the request is unsuccessful, a human readable error + - row_id: str, the row_id that we tried to delete """ req_path = [v for v in req_path.split('/') if v] if req_op == 'replace': @@ -458,13 +464,15 @@ def prep_template_patch_req(user_id, req_op, req_path, req_value=None, return {'status': status, 'message': msg} elif req_op == 'remove': - # The structure of the path should be /prep_id/{columns|samples}/name - if len(req_path) != 3: + # The structure of the path should be: + # /prep_id/row_id/{columns|samples}/name + if len(req_path) != 4: return {'status': 'error', 'message': 'Incorrect path parameter'} prep_id = int(req_path[0]) - attribute = req_path[1] - attr_id = req_path[2] + row_id = req_path[1] + attribute = req_path[2] + attr_id = req_path[3] # Check if the user actually has access to the study pt = PrepTemplate(prep_id) @@ -478,12 +486,13 @@ def prep_template_patch_req(user_id, req_op, req_path, req_value=None, # Store the job id attaching it to the sample template id r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id, dumps({'job_id': job_id, 'is_qiita_job': False})) - return {'status': 'success', 'message': ''} + return {'status': 'success', 'message': '', 'row_id': row_id} else: return {'status': 'error', 'message': 'Operation "%s" not supported. ' 'Current supported operations: replace, remove' - % req_op} + % req_op, + 'row_id': '0'} def prep_template_samples_get_req(prep_id, user_id): diff --git a/qiita_pet/handlers/api_proxy/sample_template.py b/qiita_pet/handlers/api_proxy/sample_template.py index 0adeeeac6..e6bf953b2 100644 --- a/qiita_pet/handlers/api_proxy/sample_template.py +++ b/qiita_pet/handlers/api_proxy/sample_template.py @@ -521,13 +521,15 @@ def sample_template_patch_request(user_id, req_op, req_path, req_value=None, if req_op == 'remove': req_path = [v for v in req_path.split('/') if v] - if len(req_path) != 3: + # format: study_id/row_id/column|sample/attribute_id + if len(req_path) != 4: return {'status': 'error', 'message': 'Incorrect path parameter'} st_id = req_path[0] - attribute = req_path[1] - attr_id = req_path[2] + row_id = req_path[1] + attribute = req_path[2] + attr_id = req_path[3] # Check if the user actually has access to the template st = SampleTemplate(st_id) @@ -542,9 +544,10 @@ def sample_template_patch_request(user_id, req_op, req_path, req_value=None, r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % st_id, dumps({'job_id': job_id})) - return {'status': 'success', 'message': ''} + return {'status': 'success', 'message': '', 'row_id': row_id} else: return {'status': 'error', 'message': 'Operation "%s" not supported. ' - 'Current supported operations: remove' % req_op} + 'Current supported operations: remove' % req_op, + 'row_id': 0} diff --git a/qiita_pet/handlers/api_proxy/studies.py b/qiita_pet/handlers/api_proxy/studies.py index 0cae2abb1..98b4bdebf 100644 --- a/qiita_pet/handlers/api_proxy/studies.py +++ b/qiita_pet/handlers/api_proxy/studies.py @@ -16,6 +16,7 @@ from qiita_db.util import (supported_filepath_types, get_files_from_uploads_folders) from qiita_pet.handlers.api_proxy.util import check_access +from qiita_core.exceptions import IncompetentQiitaDeveloperError def data_types_get_req(): @@ -198,7 +199,7 @@ def study_prep_get_req(study_id, user_id): def study_files_get_req(user_id, study_id, prep_template_id, artifact_type): """Returns the uploaded files for the study id categorized by artifact_type - It retrieves the files uploaded for the given study and tries to do a + It retrieves the files uploaded for the given study and tries to guess on how those files should be added to the artifact of the given type. Uses information on the prep template to try to do a better guess. @@ -234,31 +235,46 @@ def study_files_get_req(user_id, study_id, prep_template_id, artifact_type): remaining = [] uploaded = get_files_from_uploads_folders(study_id) - pt = PrepTemplate(prep_template_id).to_dataframe() + pt = PrepTemplate(prep_template_id) + if pt.study_id != study_id: + raise IncompetentQiitaDeveloperError( + "The requested prep id (%d) doesn't belong to the study " + "(%d)" % (pt.study_id, study_id)) + + pt = pt.to_dataframe() ftypes_if = (ft.startswith('raw_') for ft, _ in supp_file_types if ft != 'raw_sff') if any(ftypes_if) and 'run_prefix' in pt.columns: prep_prefixes = tuple(set(pt['run_prefix'])) num_prefixes = len(prep_prefixes) - for _, filename in uploaded: - if filename.startswith(prep_prefixes): - selected.append(filename) + # sorting prefixes by length to avoid collisions like: 100 1002 + # 10003 + prep_prefixes = sorted(prep_prefixes, key=len, reverse=True) + # group files by prefix + sfiles = {p: [f for _, f in uploaded if f.startswith(p)] + for p in prep_prefixes} + inuse = [y for x in sfiles.values() for y in x] + remaining.extend([f for _, f in uploaded if f not in inuse]) + supp_file_types_len = len(supp_file_types) + + for k, v in viewitems(sfiles): + len_files = len(v) + # if the number of files in the k group is larger than the + # available columns add to the remaining group, if not put them in + # the selected group + if len_files > supp_file_types_len: + remaining.extend(v) else: - remaining.append(filename) + v.sort() + selected.append(v) else: num_prefixes = 0 remaining = [f for _, f in uploaded] - # At this point we can't do anything smart about selecting by default - # the files for each type. The only thing that we can do is assume that - # the first in the supp_file_types list is the default one where files - # should be added in case of 'run_prefix' being present - file_types = [(fp_type, req, []) for fp_type, req in supp_file_types[1:]] - first = supp_file_types[0] - # Note that this works even if `run_prefix` is not in the prep template - # because selected is initialized to the empty list - file_types.insert(0, (first[0], first[1], selected)) + # get file_types, format: filetype, required, list of files + file_types = [(t, req, [x[i] for x in selected if i+1 <= len(x)]) + for i, (t, req) in enumerate(supp_file_types)] # Create a list of artifacts that the user has access to, in case that # he wants to import the files from another artifact diff --git a/qiita_pet/handlers/api_proxy/tests/test_artifact.py b/qiita_pet/handlers/api_proxy/tests/test_artifact.py index f6876f524..badcd9f06 100644 --- a/qiita_pet/handlers/api_proxy/tests/test_artifact.py +++ b/qiita_pet/handlers/api_proxy/tests/test_artifact.py @@ -102,7 +102,7 @@ def test_artifact_types_get_req(self): exp = {'message': '', 'status': 'success', 'types': [['BIOM', 'BIOM table'], - ['Demultiplexed', 'Demultiplexed and QC sequeneces'], + ['Demultiplexed', 'Demultiplexed and QC sequences'], ['FASTA', None], ['FASTA_Sanger', None], ['FASTQ', None], diff --git a/qiita_pet/handlers/api_proxy/tests/test_prep_template.py b/qiita_pet/handlers/api_proxy/tests/test_prep_template.py index c33a1219f..9b825518f 100644 --- a/qiita_pet/handlers/api_proxy/tests/test_prep_template.py +++ b/qiita_pet/handlers/api_proxy/tests/test_prep_template.py @@ -206,40 +206,9 @@ def test_prep_template_graph_get_req_no_exists(self): def test_prep_template_summary_get_req(self): obs = prep_template_summary_get_req(1, 'test@foo.bar') - exp = {'summary': { - 'experiment_center': [('ANL', 27)], - 'center_name': [('ANL', 27)], - 'run_center': [('ANL', 27)], - 'run_prefix': [('s_G1_L001_sequences', 27)], - 'primer': [('GTGCCAGCMGCCGCGGTAA', 27)], - 'target_gene': [('16S rRNA', 27)], - 'sequencing_meth': [('Sequencing by synthesis', 27)], - 'run_date': [('8/1/12', 27)], - 'platform': [('Illumina', 27)], - 'pcr_primers': [('FWD:GTGCCAGCMGCCGCGGTAA; ' - 'REV:GGACTACHVGGGTWTCTAAT', 27)], - 'library_construction_protocol': [( - 'This analysis was done as in Caporaso et al 2011 Genome ' - 'research. The PCR primers (F515/R806) were developed against ' - 'the V4 region of the 16S rRNA (both bacteria and archaea), ' - 'which we determined would yield optimal community clustering ' - 'with reads of this length using a procedure similar to that ' - 'of ref. 15. [For reference, this primer pair amplifies the ' - 'region 533_786 in the Escherichia coli strain 83972 sequence ' - '(greengenes accession no. prokMSA_id:470367).] The reverse ' - 'PCR primer is barcoded with a 12-base error-correcting Golay ' - 'code to facilitate multiplexing of up to 1,500 samples per ' - 'lane, and both PCR primers contain sequencer adapter ' - 'regions.', 27)], - 'experiment_design_description': [( - 'micro biome of soil and rhizosphere of cannabis plants from ' - 'CA', 27)], - 'study_center': [('CCME', 27)], - 'center_project_name': [], - 'sample_center': [('ANL', 27)], - 'samp_size': [('.25,g', 27)], - 'qiita_prep_id': [('1', 27)], - 'barcode': [ + exp = { + 'status': 'success', 'message': '', + 'summary': [('barcode', [ ('AACTCCTGTGGA', 1), ('ACCTCAGTCAAG', 1), ('ACGCACATACAA', 1), ('AGCAGGCACGAA', 1), ('AGCGCTCACATC', 1), ('ATATCGCGATGA', 1), ('ATGGCCTGACTA', 1), ('CATACACGCACC', 1), ('CCACCCAGTAAC', 1), @@ -248,15 +217,44 @@ def test_prep_template_summary_get_req(self): ('CGTAGAGCTCTC', 1), ('CGTGCACAATTG', 1), ('GATAGCACTCGT', 1), ('GCGGACTATTCA', 1), ('GTCCGCAAGTTA', 1), ('TAATGGTCGTAG', 1), ('TAGCGCGAACTT', 1), ('TCGACCAAACAC', 1), ('TGAGTGGTCTGT', 1), - ('TGCTACAGACGT', 1), ('TGGTTATGGCAC', 1), ('TTGCACCGTCGA', 1)], - 'emp_status': [('EMP', 27)], - 'illumina_technology': [('MiSeq', 27)], - 'experiment_title': [('Cannabis Soil Microbiome', 27)], - 'target_subfragment': [('V4', 27)], - 'instrument_model': [('Illumina MiSeq', 27)]}, - 'num_samples': 27, - 'status': 'success', - 'message': ''} + ('TGCTACAGACGT', 1), ('TGGTTATGGCAC', 1), ('TTGCACCGTCGA', 1) + ]), ('center_name', [('ANL', 27)]), ('center_project_name', []), + ('emp_status', [('EMP', 27)]), + ('experiment_center', [('ANL', 27)]), + ('experiment_design_description', [ + ('micro biome of soil and rhizosphere of cannabis plants ' + 'from CA', 27)]), + ('experiment_title', [('Cannabis Soil Microbiome', 27)]), + ('illumina_technology', [('MiSeq', 27)]), + ('instrument_model', [('Illumina MiSeq', 27)]), + ('library_construction_protocol', [ + ('This analysis was done as in Caporaso et al 2011 Genome ' + 'research. The PCR primers (F515/R806) were developed ' + 'against the V4 region of the 16S rRNA (both bacteria ' + 'and archaea), which we determined would yield optimal ' + 'community clustering with reads of this length using a ' + 'procedure similar to that of ref. 15. [For reference, ' + 'this primer pair amplifies the region 533_786 in the ' + 'Escherichia coli strain 83972 sequence (greengenes ' + 'accession no. prokMSA_id:470367).] The reverse PCR ' + 'primer is barcoded with a 12-base error-correcting ' + 'Golay code to facilitate multiplexing of up to 1,500 ' + 'samples per lane, and both PCR primers contain ' + 'sequencer adapter regions.', 27)]), + ('pcr_primers', [( + 'FWD:GTGCCAGCMGCCGCGGTAA; REV:GGACTACHVGGGTWTCTAAT', 27)]), + ('platform', [('Illumina', 27)]), + ('primer', [('GTGCCAGCMGCCGCGGTAA', 27)]), + ('qiita_prep_id', [('1', 27)]), ('run_center', [('ANL', 27)]), + ('run_date', [('8/1/12', 27)]), + ('run_prefix', [('s_G1_L001_sequences', 27)]), + ('samp_size', [('.25,g', 27)]), + ('sample_center', [('ANL', 27)]), + ('sequencing_meth', [('Sequencing by synthesis', 27)]), + ('study_center', [('CCME', 27)]), + ('target_gene', [('16S rRNA', 27)]), + ('target_subfragment', [('V4', 27)])], + 'editable': True, 'num_samples': 27} self.assertEqual(obs, exp) def test_prep_template_summary_get_req_no_access(self): @@ -476,8 +474,8 @@ def test_prep_template_patch_req(self): # Delete a prep template column obs = prep_template_patch_req( 'test@foo.bar', 'remove', - '/%s/columns/target_subfragment/' % pt.id) - exp = {'status': 'success', 'message': ''} + '/%s/10/columns/target_subfragment/' % pt.id) + exp = {'status': 'success', 'message': '', 'row_id': '10'} self.assertEqual(obs, exp) self._wait_for_parallel_job('prep_template_%s' % pt.id) self.assertNotIn('target_subfragment', pt.categories()) @@ -489,7 +487,8 @@ def test_prep_template_patch_req(self): 'Cancer Genomics') exp = {'status': 'error', 'message': 'Operation "add" not supported. ' - 'Current supported operations: replace, remove'} + 'Current supported operations: replace, remove', + 'row_id': '0'} self.assertEqual(obs, exp) # Incorrect path parameter obs = prep_template_patch_req( diff --git a/qiita_pet/handlers/api_proxy/tests/test_sample_template.py b/qiita_pet/handlers/api_proxy/tests/test_sample_template.py index 4e6daadcc..d7f226189 100644 --- a/qiita_pet/handlers/api_proxy/tests/test_sample_template.py +++ b/qiita_pet/handlers/api_proxy/tests/test_sample_template.py @@ -525,28 +525,28 @@ def test_sample_template_meta_cats_get_req_no_template(self): def test_sample_template_patch_request(self): # Wrong operation operation obs = sample_template_patch_request( - "test@foo.bar", "add", "/1/columns/season_environment/") + "test@foo.bar", "add", "/1/10/columns/season_environment/") exp = {'status': 'error', 'message': 'Operation "add" not supported. ' - 'Current supported operations: remove'} + 'Current supported operations: remove', + 'row_id': 0} self.assertEqual(obs, exp) # Wrong path parameter obs = sample_template_patch_request( - "test@foo.bar", "remove", "/columns/season_environment/") + "test@foo.bar", "remove", "10/columns/season_environment/") exp = {'status': 'error', 'message': 'Incorrect path parameter'} self.assertEqual(obs, exp) # No access obs = sample_template_patch_request( - "demo@microbio.me", "remove", "/1/columns/season_environment/") + "demo@microbio.me", "remove", "/1/10/columns/season_environment/") exp = {'status': 'error', 'message': 'User does not have access to study'} self.assertEqual(obs, exp) # Success obs = sample_template_patch_request( - "test@foo.bar", "remove", "/1/columns/season_environment/") - exp = {'status': 'success', - 'message': ''} + "test@foo.bar", "remove", "/1/10/columns/season_environment/") + exp = {'status': 'success', 'message': '', 'row_id': '10'} self.assertEqual(obs, exp) # This is needed so the clean up works - this is a distributed system diff --git a/qiita_pet/handlers/api_proxy/tests/test_studies.py b/qiita_pet/handlers/api_proxy/tests/test_studies.py index 0a0e427a0..e700d37f3 100644 --- a/qiita_pet/handlers/api_proxy/tests/test_studies.py +++ b/qiita_pet/handlers/api_proxy/tests/test_studies.py @@ -7,15 +7,16 @@ # ----------------------------------------------------------------------------- from unittest import TestCase, main from datetime import datetime -from os.path import exists, join, basename, isdir -from os import remove, close, mkdir +from os.path import exists, join, isdir +from os import remove from shutil import rmtree -from tempfile import mkstemp, mkdtemp +from tempfile import mkdtemp import pandas as pd import numpy.testing as npt from qiita_core.util import qiita_test_checker +from qiita_core.exceptions import IncompetentQiitaDeveloperError import qiita_db as qdb from qiita_pet.handlers.api_proxy.studies import ( data_types_get_req, study_get_req, study_prep_get_req, study_delete_req, @@ -237,6 +238,8 @@ def test_study_prep_get_req(self): for i in range(4, 0, -1): qdb.artifact.Artifact(i).visibility = "private" + qdb.metadata_template.prep_template.PrepTemplate.delete(pt.id) + def test_study_prep_get_req_failed_EBI(self): temp_dir = mkdtemp() self._clean_up_files.append(temp_dir) @@ -282,7 +285,9 @@ def test_study_prep_get_req_failed_EBI(self): } metadata = pd.DataFrame.from_dict(metadata_dict, orient='index', dtype=str) - qdb.metadata_template.sample_template.SampleTemplate.create( + npt.assert_warns( + qdb.exceptions.QiitaDBWarning, + qdb.metadata_template.sample_template.SampleTemplate.create, metadata, study) # (C) @@ -334,6 +339,8 @@ def test_study_prep_get_req_failed_EBI(self): 'status': 'success'} self.assertEqual(obs, exp) + qdb.metadata_template.prep_template.PrepTemplate.delete(pt.id) + def test_study_prep_get_req_no_access(self): obs = study_prep_get_req(1, 'demo@microbio.me') exp = {'status': 'error', @@ -409,6 +416,7 @@ def test_study_files_get_req(self): 'Cannabis Soils (1) - Raw data 1 (1)')]} self.assertEqual(obs, exp) + # adding a new study for further testing info = { "timeseries_type_id": 1, "metadata_complete": True, @@ -422,58 +430,103 @@ def test_study_files_get_req(self): "principal_investigator_id": qdb.study.StudyPerson(3), "lab_person_id": qdb.study.StudyPerson(1) } - new_study = qdb.study.Study.create( qdb.user.User('test@foo.bar'), "Some New Study to get files", [1], info) - obs = study_files_get_req('test@foo.bar', new_study.id, 1, 'FASTQ') - exp = {'status': 'success', - 'message': '', - 'remaining': [], - 'file_types': [('raw_barcodes', True, []), - ('raw_forward_seqs', True, []), - ('raw_reverse_seqs', False, [])], - 'num_prefixes': 1, - 'artifacts': [(1, 'Identification of the Microbiomes for ' - 'Cannabis Soils (1) - Raw data 1 (1)')]} + # check that you can't call a this function using two unrelated + # study_id and prep_template_id + with self.assertRaises(IncompetentQiitaDeveloperError): + study_files_get_req('test@foo.bar', new_study.id, 1, 'FASTQ') + + def test_study_files_get_req_multiple(self): + study_id = 1 + # adding a new prep for testing + PREP = qdb.metadata_template.prep_template.PrepTemplate + prep_info_dict = { + 'SKB7.640196': {'run_prefix': 'test_1'}, + 'SKB8.640193': {'run_prefix': 'test_2'} + } + prep_info = pd.DataFrame.from_dict(prep_info_dict, + orient='index', dtype=str) + pt = npt.assert_warns( + qdb.exceptions.QiitaDBWarning, PREP.create, prep_info, + qdb.study.Study(study_id), "Metagenomic") + + # getting the upload folder so we can test + study_upload_dir = join( + qdb.util.get_mountpoint("uploads")[0][1], str(study_id)) + + # adding just foward per sample FASTQ to the upload folder + filenames = ['test_1.R1.fastq.gz', 'test_2.R1.fastq.gz'] + for f in filenames: + fpt = join(study_upload_dir, f) + open(fpt, 'w', 0).close() + self._clean_up_files.append(fpt) + obs = study_files_get_req( + 'shared@foo.bar', 1, pt.id, 'per_sample_FASTQ') + exp = { + 'status': 'success', 'num_prefixes': 2, 'artifacts': [], + 'remaining': ['uploaded_file.txt'], 'message': '', + 'file_types': [ + ('raw_forward_seqs', True, + ['test_2.R1.fastq.gz', 'test_1.R1.fastq.gz']), + ('raw_reverse_seqs', False, [])]} self.assertEqual(obs, exp) - obs = study_files_get_req('admin@foo.bar', new_study.id, 1, 'FASTQ') - exp = {'status': 'success', - 'message': '', - 'remaining': [], - 'file_types': [('raw_barcodes', True, []), - ('raw_forward_seqs', True, []), - ('raw_reverse_seqs', False, [])], - 'num_prefixes': 1, - 'artifacts': []} + # let's add reverse + filenames = ['test_1.R2.fastq.gz', 'test_2.R2.fastq.gz'] + for f in filenames: + fpt = join(study_upload_dir, f) + open(fpt, 'w', 0).close() + self._clean_up_files.append(fpt) + obs = study_files_get_req( + 'shared@foo.bar', 1, pt.id, 'per_sample_FASTQ') + exp = {'status': 'success', 'num_prefixes': 2, 'artifacts': [], + 'remaining': ['uploaded_file.txt'], 'message': '', + 'file_types': [('raw_forward_seqs', True, + ['test_2.R1.fastq.gz', 'test_1.R1.fastq.gz']), + ('raw_reverse_seqs', False, + ['test_2.R2.fastq.gz', 'test_1.R2.fastq.gz'])]} self.assertEqual(obs, exp) - # Create some 'sff' files - upload_dir = qdb.util.get_mountpoint("uploads")[0][1] - study_upload_dir = join(upload_dir, str(new_study.id)) - fps = [] - - for i in range(2): - if not exists(study_upload_dir): - mkdir(study_upload_dir) - fd, fp = mkstemp(suffix=".sff", dir=study_upload_dir) - close(fd) - with open(fp, 'w') as f: - f.write('\n') - fps.append(fp) - - self._clean_up_files.extend(fps) + # let's an extra file that matches + filenames = ['test_1.R3.fastq.gz'] + for f in filenames: + fpt = join(study_upload_dir, f) + open(fpt, 'w', 0).close() + self._clean_up_files.append(fpt) + obs = study_files_get_req( + 'shared@foo.bar', 1, pt.id, 'per_sample_FASTQ') + exp = {'status': 'success', 'num_prefixes': 2, 'artifacts': [], + 'remaining': ['test_1.R1.fastq.gz', 'test_1.R2.fastq.gz', + 'test_1.R3.fastq.gz', 'uploaded_file.txt'], + 'message': '', + 'file_types': [('raw_forward_seqs', True, + ['test_2.R1.fastq.gz']), + ('raw_reverse_seqs', False, + ['test_2.R2.fastq.gz'])]} + self.assertEqual(obs, exp) - obs = study_files_get_req('test@foo.bar', new_study.id, 1, 'SFF') - exp = {'status': 'success', + # now if we select FASTQ we have 3 columns so the extra file should go + # to the 3rd column + obs = study_files_get_req( + 'shared@foo.bar', 1, pt.id, 'FASTQ') + exp = {'status': 'success', 'num_prefixes': 2, + 'remaining': ['uploaded_file.txt'], 'message': '', - 'remaining': [basename(fpath) for fpath in sorted(fps)], - 'file_types': [('raw_sff', True, [])], - 'num_prefixes': 0, - 'artifacts': []} + 'artifacts': [(1, 'Identification of the Microbiomes for ' + 'Cannabis Soils (1) - Raw data 1 (1)')], + 'file_types': [ + ('raw_barcodes', True, + ['test_2.R1.fastq.gz', 'test_1.R1.fastq.gz']), + ('raw_forward_seqs', True, + ['test_2.R2.fastq.gz', 'test_1.R2.fastq.gz']), + ('raw_reverse_seqs', False, ['test_1.R3.fastq.gz'])]} self.assertEqual(obs, exp) + PREP.delete(pt.id) + + if __name__ == '__main__': main() diff --git a/qiita_pet/handlers/api_proxy/tests/test_user.py b/qiita_pet/handlers/api_proxy/tests/test_user.py index ac5e55bfa..8823bcce7 100644 --- a/qiita_pet/handlers/api_proxy/tests/test_user.py +++ b/qiita_pet/handlers/api_proxy/tests/test_user.py @@ -46,24 +46,8 @@ def test_user_jobs_get_req(self): 'threads': 1, 'sortmerna_coverage': 0.97}, 'name': 'Pick closed-reference OTUs', - 'processing_job_workflow_id': ''}, - {'id': 'b72369f9-a886-4193-8d3d-f7b504168e75', - 'status': 'success', - 'heartbeat': '2015-11-22 21:15:00', - 'params': { - 'max_barcode_errors': 1.5, - 'sequence_max_n': 0, - 'max_bad_run_length': 3, - 'phred_offset': u'auto', - 'rev_comp': False, - 'phred_quality_threshold': 3, - 'input_data': 1, - 'rev_comp_barcode': False, - 'rev_comp_mapping_barcodes': True, - 'min_per_read_length_fraction': 0.75, - 'barcode_type': u'golay_12'}, - 'name': 'Split libraries FASTQ', - 'processing_job_workflow_id': 1}]} + 'step': 'generating demux file', + 'processing_job_workflow_id': ''}]} self.assertEqual(obs, exp) diff --git a/qiita_pet/handlers/api_proxy/user.py b/qiita_pet/handlers/api_proxy/user.py index d5edd2ceb..ab5a1fd80 100644 --- a/qiita_pet/handlers/api_proxy/user.py +++ b/qiita_pet/handlers/api_proxy/user.py @@ -11,26 +11,26 @@ @execute_as_transaction -def user_jobs_get_req(user): +def user_jobs_get_req(user, limit=30): """Gets the json of jobs Parameters ---------- - prep_id : int - PrepTemplate id to get info for - user_id : str - User requesting the sample template info + user : User + The user from which you want to return all jobs + limit : int, optional + Maximum jobs to send, negative values will return all Returns ------- dict of objects {'status': status, 'message': message, - 'template': {sample: {column: value, ...}, ...} + 'template': {{column: value, ...}, ...} """ response = [] - for j in user.jobs(): + for i, j in enumerate(user.jobs()): name = j.command.name hb = j.heartbeat hb = "" if hb is None else hb.strftime("%Y-%m-%d %H:%M:%S") @@ -42,6 +42,7 @@ def user_jobs_get_req(user): 'params': j.parameters.values, 'status': j.status, 'heartbeat': hb, + 'step': j.step, 'processing_job_workflow_id': wid}) return {'status': 'success', diff --git a/qiita_pet/handlers/download.py b/qiita_pet/handlers/download.py index 1a113e928..bbf10699f 100644 --- a/qiita_pet/handlers/download.py +++ b/qiita_pet/handlers/download.py @@ -5,7 +5,7 @@ from .base_handlers import BaseHandler from qiita_pet.exceptions import QiitaPetAuthorizationError from qiita_db.util import filepath_id_to_rel_path -from qiita_db.meta_util import get_accessible_filepath_ids +from qiita_db.meta_util import validate_filepath_access_by_user from qiita_core.util import execute_as_transaction @@ -13,15 +13,13 @@ class DownloadHandler(BaseHandler): @authenticated @execute_as_transaction def get(self, filepath_id): - filepath_id = int(filepath_id) - # Check access to file - accessible_filepaths = get_accessible_filepath_ids(self.current_user) + fid = int(filepath_id) - if filepath_id not in accessible_filepaths: + if not validate_filepath_access_by_user(self.current_user, fid): raise QiitaPetAuthorizationError( - self.current_user, 'filepath id %s' % str(filepath_id)) + self.current_user, 'filepath id %s' % str(fid)) - relpath = filepath_id_to_rel_path(filepath_id) + relpath = filepath_id_to_rel_path(fid) fname = basename(relpath) # If we don't have nginx, write a file that indicates this diff --git a/qiita_pet/handlers/stats.py b/qiita_pet/handlers/stats.py index 5a82c04a0..4164d0fc9 100644 --- a/qiita_pet/handlers/stats.py +++ b/qiita_pet/handlers/stats.py @@ -7,63 +7,36 @@ from qiita_core.util import execute_as_transaction from qiita_core.qiita_settings import qiita_config -from qiita_db.util import get_count from qiita_db.study import Study -from qiita_db.meta_util import get_lat_longs from .base_handlers import BaseHandler class StatsHandler(BaseHandler): @execute_as_transaction def _get_stats(self, callback): - # check if the key exists in redis - redis_lats_key = '%s:stats:sample_lats' % qiita_config.portal - redis_longs_key = '%s:stats:sample_longs' % qiita_config.portal - lats = r_client.lrange(redis_lats_key, 0, -1) - longs = r_client.lrange(redis_longs_key, 0, -1) - if not (lats and longs): - # if we don't have them, then fetch from disk and add to the - # redis server with a 24-hour expiration - lat_longs = get_lat_longs() - lats = [float(x[0]) for x in lat_longs] - longs = [float(x[1]) for x in lat_longs] - with r_client.pipeline() as pipe: - for latitude, longitude in lat_longs: - # storing as a simple data structure, hopefully this - # doesn't burn us later - pipe.rpush(redis_lats_key, latitude) - pipe.rpush(redis_longs_key, longitude) + stats = {} + # checking values from redis + portal = qiita_config.portal + vals = [ + ('number_studies', r_client.hgetall), + ('number_of_samples', r_client.hgetall), + ('num_users', r_client.get), + ('lat_longs', r_client.get), + ('num_studies_ebi', r_client.get), + ('num_samples_ebi', r_client.get), + ('number_samples_ebi_prep', r_client.get), + ('img', r_client.get), + ('time', r_client.get)] + for k, f in vals: + redis_key = '%s:stats:%s' % (portal, k) + stats[k] = f(redis_key) - # set the key to expire in 24 hours, so that we limit the - # number of times we have to go to the database to a reasonable - # amount - r_client.expire(redis_lats_key, 86400) - r_client.expire(redis_longs_key, 86400) - - pipe.execute() - else: - # If we do have them, put the redis results into the same structure - # that would come back from the database - longs = [float(x) for x in longs] - lats = [float(x) for x in lats] - lat_longs = zip(lats, longs) - - # Get the number of studies - num_studies = get_count('qiita.study') - - # Get the number of samples - num_samples = len(lats) - - # Get the number of users - num_users = get_count('qiita.qiita_user') - - callback([num_studies, num_samples, num_users, lat_longs]) + callback(stats) @coroutine @execute_as_transaction def get(self): - num_studies, num_samples, num_users, lat_longs = \ - yield Task(self._get_stats) + stats = yield Task(self._get_stats) # Pull a random public study from the database public_studies = Study.get_by_status('public') @@ -79,8 +52,14 @@ def get(self): random_study_id = study.id self.render('stats.html', - num_studies=num_studies, num_samples=num_samples, - num_users=num_users, lat_longs=lat_longs, + number_studies=stats['number_studies'], + number_of_samples=stats['number_of_samples'], + num_users=stats['num_users'], + lat_longs=eval(stats['lat_longs']), + num_studies_ebi=stats['num_studies_ebi'], + num_samples_ebi=stats['num_samples_ebi'], + number_samples_ebi_prep=stats['number_samples_ebi_prep'], + img=stats['img'], time=stats['time'], random_study_info=random_study_info, random_study_title=random_study_title, random_study_id=random_study_id) diff --git a/qiita_pet/handlers/study_handlers/prep_template.py b/qiita_pet/handlers/study_handlers/prep_template.py index 573ad36e5..78bc6dade 100644 --- a/qiita_pet/handlers/study_handlers/prep_template.py +++ b/qiita_pet/handlers/study_handlers/prep_template.py @@ -44,9 +44,11 @@ class PrepTemplateSummaryAJAX(BaseHandler): @authenticated def get(self): prep_id = to_int(self.get_argument('prep_id')) + res = prep_template_summary_get_req(prep_id, self.current_user.id) + self.render('study_ajax/prep_summary_table.html', pid=prep_id, - stats=res['summary']) + stats=res['summary'], editable=res['editable']) class PrepTemplateAJAX(BaseHandler): @@ -54,11 +56,14 @@ class PrepTemplateAJAX(BaseHandler): def get(self): """Send formatted summary page of prep template""" prep_id = to_int(self.get_argument('prep_id')) + row_id = self.get_argument('row_id', '0') res = prep_template_ajax_get_req(self.current_user.id, prep_id) res['prep_id'] = prep_id + res['row_id'] = row_id # Escape the message just in case javascript breaking characters in it res['alert_message'] = url_escape(res['alert_message']) + self.render('study_ajax/prep_summary.html', **res) diff --git a/qiita_pet/handlers/study_handlers/sample_template.py b/qiita_pet/handlers/study_handlers/sample_template.py index eb035260d..5387bd486 100644 --- a/qiita_pet/handlers/study_handlers/sample_template.py +++ b/qiita_pet/handlers/study_handlers/sample_template.py @@ -74,6 +74,8 @@ class SampleTemplateAJAX(BaseHandler): def get(self): """Send formatted summary page of sample template""" study_id = self.get_argument('study_id') + row_id = self.get_argument('row_id', '0') + files = [f for _, f in get_files_from_uploads_folders(study_id) if f.endswith(('txt', 'tsv'))] data_types = sorted(data_types_get_req()['data_types']) @@ -95,6 +97,7 @@ def get(self): stats['files'] = files stats['study_id'] = study_id stats['data_types'] = data_types + stats['row_id'] = row_id # URL encode in case message has javascript-breaking characters in it stats['alert_message'] = url_escape(stats['alert_message']) self.render('study_ajax/sample_summary.html', **stats) diff --git a/qiita_pet/static/css/style.css b/qiita_pet/static/css/style.css index dcb209a1a..db6cc183d 100644 --- a/qiita_pet/static/css/style.css +++ b/qiita_pet/static/css/style.css @@ -1,9 +1,38 @@ +#qiita-main { + position: relative; + height: 100%; + width: 100%; +} +#qiita-processing { + position: absolute; + height: 100%; + width: 0%; + right: 0; + top: 0; +} #template-content{ padding: 10px; height: 100%; width: 100%; } +td.more-info-processing-jobs{ + cursor: pointer; + background: url('../img//details_open.png') no-repeat center center; +} +tr.shown td.more-info-processing-jobs { + background: url('../img//details_close.png') no-repeat center center; +} + +.blinking-message { + animation: blinker 2s linear infinite; +} + +@keyframes blinker { + 50% { opacity: 0.3; background: #CCC;} + 100% { opacity: 1.0; background: #FFF;} +} + /* table in the study description that holds the investigation type elements */ .investigation-type-table td { padding-top: 6px; diff --git a/qiita_pet/static/img/details_close.png b/qiita_pet/static/img/details_close.png new file mode 100644 index 000000000..fcc23c63e Binary files /dev/null and b/qiita_pet/static/img/details_close.png differ diff --git a/qiita_pet/static/img/details_open.png b/qiita_pet/static/img/details_open.png new file mode 100644 index 000000000..6f034d0f2 Binary files /dev/null and b/qiita_pet/static/img/details_open.png differ diff --git a/qiita_pet/static/js/qiita.js b/qiita_pet/static/js/qiita.js index 414c790ed..f9d555fc2 100644 --- a/qiita_pet/static/js/qiita.js +++ b/qiita_pet/static/js/qiita.js @@ -29,18 +29,50 @@ function bootstrapAlert(message, severity, timeout){ alertDiv.append('

Need help? Send us an email.

'); } - $('body').prepend(alertDiv); + $('#qiita-main').prepend(alertDiv); if(timeout > 0) { window.setTimeout(function() { $('#alert-message').alert('close'); }, timeout); } } +function format_extra_info_processing_jobs ( data ) { + // `data` is the original data object for the row + // 0: blank +/- button + // 1: heartbeat + // 2: name + // 3: status + // 4: step + // 5: id + // 6: params + // 7: processing_job_workflow_id + + let row = '
Table analysis_processing_job
Table study_tags
analysis_id bigint NOT NULL study_tag_id bigserial NOT NULL
processing_job_idstudy_tag varchar NOT NULL
Indexes
idx_analysis_processing_job primary key ON analysis_id, processing_job_id
email varchar NOT NULL
idx_analysis_processing_job ON analysis_id
Indexes
pk_study_tags primary key ON study_tag_id
idx_analysis_processing_job ON processing_job_id
idx_study_tags unique ON study_tag
Foreign Keys
fk_analysis_processing_job ( analysis_id ) ref analysis (analysis_id)
idx_study_tags ON email
Foreign Keys
fk_analysis_processing_job_pj ( processing_job_id ) ref processing_job (processing_job_id) fk_study_tags_qiita_user ( email ) ref qiita_user (email)
'+ + ''+ + ''+ + ''+ + ''+ + ''+ + ''+ + ''; + if (data[7] !== '' && data[3] === 'in_construction') { + row += ''+ + '' + ''; + } + row += '
ID:'+ data[5] +'
Parameters:
'+ data[6] +'
'+ + ''+ + '
'; + + return row +} + + function show_hide(div) { $('#' + div).toggle(); } - function delete_analysis(aname, analysis_id) { if (confirm('Are you sure you want to delete analysis: ' + aname + '?')) { var form = $("
") @@ -58,3 +90,21 @@ function delete_analysis(aname, analysis_id) { form.submit(); } } + +function show_hide_process_list() { + if ($("#qiita-main").width() == $("#qiita-main").parent().width()) { + // let's update the job list + processing_jobs_vue.update_processing_job_data(); + $("#qiita-main").width("76%"); + $("#user-studies-table").width("76%"); + $("#studies-table").width("76%"); + $("#qiita-processing").width("24%"); + $("#qiita-processing").show(); + } else { + $("#qiita-main").width("100%"); + $("#user-studies-table").width("100%"); + $("#studies-table").width("100%"); + $("#qiita-processing").width("0%"); + $("#qiita-processing").hide(); + } +} diff --git a/qiita_pet/static/vendor/fonts/glyphicons-halflings-regular.woff2 b/qiita_pet/static/vendor/fonts/glyphicons-halflings-regular.woff2 new file mode 100644 index 000000000..64539b54c Binary files /dev/null and b/qiita_pet/static/vendor/fonts/glyphicons-halflings-regular.woff2 differ diff --git a/qiita_pet/static/vendor/js/vue.min.js b/qiita_pet/static/vendor/js/vue.min.js new file mode 100644 index 000000000..0e552982d --- /dev/null +++ b/qiita_pet/static/vendor/js/vue.min.js @@ -0,0 +1,8 @@ +/*! + * Vue.js v2.1.6 + * (c) 2014-2016 Evan You + * Released under the MIT License. + */ +!function(e,t){"object"==typeof exports&&"undefined"!=typeof module?module.exports=t():"function"==typeof define&&define.amd?define(t):e.Vue=t()}(this,function(){"use strict";function e(e){return null==e?"":"object"==typeof e?JSON.stringify(e,null,2):String(e)}function t(e){var t=parseFloat(e,10);return t||0===t?t:e}function n(e,t){for(var n=Object.create(null),r=e.split(","),i=0;i-1)return e.splice(n,1)}}function i(e,t){return Yr.call(e,t)}function a(e){return"string"==typeof e||"number"==typeof e}function o(e){var t=Object.create(null);return function(n){var r=t[n];return r||(t[n]=e(n))}}function s(e,t){function n(n){var r=arguments.length;return r?r>1?e.apply(t,arguments):e.call(t,n):e.call(t)}return n._length=e.length,n}function c(e,t){t=t||0;for(var n=e.length-t,r=new Array(n);n--;)r[n]=e[n+t];return r}function l(e,t){for(var n in t)e[n]=t[n];return e}function u(e){return null!==e&&"object"==typeof e}function f(e){return ri.call(e)===ii}function d(e){for(var t={},n=0;n=0&&Li[n].id>e.id;)n--;Li.splice(Math.max(n,Ri)+1,0,e)}else Li.push(e);Mi||(Mi=!0,_i(B))}}function V(e){Ui.clear(),J(e,Ui)}function J(e,t){var n,r,i=Array.isArray(e);if((i||u(e))&&Object.isExtensible(e)){if(e.__ob__){var a=e.__ob__.dep.id;if(t.has(a))return;t.add(a)}if(i)for(n=e.length;n--;)J(e[n],t);else for(r=Object.keys(e),n=r.length;n--;)J(e[r[n]],t)}}function K(e){e._watchers=[];var t=e.$options;t.props&&q(e,t.props),t.methods&&Y(e,t.methods),t.data?W(e):k(e._data={},!0),t.computed&&Z(e,t.computed),t.watch&&Q(e,t.watch)}function q(e,t){var n=e.$options.propsData||{},r=e.$options._propKeys=Object.keys(t),i=!e.$parent;Si.shouldConvert=i;for(var a=function(i){var a=r[i];A(e,a,R(a,t,n,e))},o=0;o1?c(n):n;for(var r=c(arguments,1),i=0,a=n.length;i-1:e.test(t)}function qe(e){var t={};t.get=function(){return si},Object.defineProperty(e,"config",t),e.util=Ni,e.set=O,e.delete=S,e.nextTick=_i,e.options=Object.create(null),si._assetTypes.forEach(function(t){e.options[t+"s"]=Object.create(null)}),e.options._base=e,l(e.options.components,Yi),Be(e),ze(e),Ve(e),Je(e)}function We(e){for(var t=e.data,n=e,r=e;r.child;)r=r.child._vnode,r.data&&(t=Ze(r.data,t));for(;n=n.parent;)n.data&&(t=Ze(t,n.data));return Ge(t)}function Ze(e,t){return{staticClass:Ye(e.staticClass,t.staticClass),class:e.class?[e.class,t.class]:t.class}}function Ge(e){var t=e.class,n=e.staticClass;return n||t?Ye(n,Qe(t)):""}function Ye(e,t){return e?t?e+" "+t:e:t||""}function Qe(e){var t="";if(!e)return t;if("string"==typeof e)return e;if(Array.isArray(e)){for(var n,r=0,i=e.length;r-1?pa[e]=t.constructor===window.HTMLUnknownElement||t.constructor===window.HTMLElement:pa[e]=/HTMLUnknownElement/.test(t.toString())}function tt(e){if("string"==typeof e){if(e=document.querySelector(e),!e)return document.createElement("div")}return e}function nt(e,t){var n=document.createElement(e);return"select"!==e?n:(t.data&&t.data.attrs&&"multiple"in t.data.attrs&&n.setAttribute("multiple","multiple"),n)}function rt(e,t){return document.createElementNS(ca[e],t)}function it(e){return document.createTextNode(e)}function at(e){return document.createComment(e)}function ot(e,t,n){e.insertBefore(t,n)}function st(e,t){e.removeChild(t)}function ct(e,t){e.appendChild(t)}function lt(e){return e.parentNode}function ut(e){return e.nextSibling}function ft(e){return e.tagName}function dt(e,t){e.textContent=t}function pt(e,t,n){e.setAttribute(t,n)}function vt(e,t){var n=e.data.ref;if(n){var i=e.context,a=e.child||e.elm,o=i.$refs;t?Array.isArray(o[n])?r(o[n],a):o[n]===a&&(o[n]=void 0):e.data.refInFor?Array.isArray(o[n])&&o[n].indexOf(a)<0?o[n].push(a):o[n]=[a]:o[n]=a}}function ht(e){return null==e}function mt(e){return null!=e}function gt(e,t){return e.key===t.key&&e.tag===t.tag&&e.isComment===t.isComment&&!e.data==!t.data}function yt(e,t,n){var r,i,a={};for(r=t;r<=n;++r)i=e[r].key,mt(i)&&(a[i]=r);return a}function _t(e){function t(e){return new zi(O.tagName(e).toLowerCase(),{},[],void 0,e)}function r(e,t){function n(){0===--n.listeners&&i(e)}return n.listeners=t,n}function i(e){var t=O.parentNode(e);t&&O.removeChild(t,e)}function o(e,t,n,r,i){if(e.isRootInsert=!i,!s(e,t,n,r)){var a=e.data,o=e.children,c=e.tag;mt(c)?(e.elm=e.ns?O.createElementNS(e.ns,c):O.createElement(c,e),v(e),u(e,o,t),mt(a)&&d(e,t),l(n,e.elm,r)):e.isComment?(e.elm=O.createComment(e.text),l(n,e.elm,r)):(e.elm=O.createTextNode(e.text),l(n,e.elm,r))}}function s(e,t,n,r){var i=e.data;if(mt(i)){var a=mt(e.child)&&i.keepAlive;if(mt(i=i.hook)&&mt(i=i.init)&&i(e,!1,n,r),mt(e.child))return p(e,t),a&&c(e,t,n,r),!0}}function c(e,t,n,r){for(var i,a=e;a.child;)if(a=a.child._vnode,mt(i=a.data)&&mt(i=i.transition)){for(i=0;id?(l=ht(n[m+1])?null:n[m+1].elm,h(e,l,n,f,m,r)):f>m&&g(e,t,u,d)}function b(e,t,n,r){if(e!==t){if(t.isStatic&&e.isStatic&&t.key===e.key&&(t.isCloned||t.isOnce))return t.elm=e.elm,void(t.child=e.child);var i,a=t.data,o=mt(a);o&&mt(i=a.hook)&&mt(i=i.prepatch)&&i(e,t);var s=t.elm=e.elm,c=e.children,l=t.children;if(o&&f(t)){for(i=0;i-1?t.split(/\s+/).forEach(function(t){return e.classList.add(t)}):e.classList.add(t);else{var n=" "+e.getAttribute("class")+" ";n.indexOf(" "+t+" ")<0&&e.setAttribute("class",(n+t).trim())}}function It(e,t){if(t&&t.trim())if(e.classList)t.indexOf(" ")>-1?t.split(/\s+/).forEach(function(t){return e.classList.remove(t)}):e.classList.remove(t);else{for(var n=" "+e.getAttribute("class")+" ",r=" "+t+" ";n.indexOf(r)>=0;)n=n.replace(r," ");e.setAttribute("class",n.trim())}}function Ft(e){Fa(function(){Fa(e)})}function Ht(e,t){(e._transitionClasses||(e._transitionClasses=[])).push(t),Rt(e,t)}function Ut(e,t){e._transitionClasses&&r(e._transitionClasses,t),It(e,t)}function Bt(e,t,n){var r=zt(e,t),i=r.type,a=r.timeout,o=r.propCount;if(!i)return n();var s=i===La?Pa:Ia,c=0,l=function(){e.removeEventListener(s,u),n()},u=function(t){t.target===e&&++c>=o&&l()};setTimeout(function(){c0&&(n=La,u=o,f=a.length):t===Da?l>0&&(n=Da,u=l,f=c.length):(u=Math.max(o,l),n=u>0?o>l?La:Da:null,f=n?n===La?a.length:c.length:0);var d=n===La&&Ha.test(r[Ma+"Property"]);return{type:n,timeout:u,propCount:f,hasTransform:d}}function Vt(e,t){for(;e.length1,T=n._enterCb=Zt(function(){O&&Ut(n,w),T.cancelled?(O&&Ut(n,$), +A&&A(n)):k&&k(n),n._enterCb=null});e.data.show||ae(e.data.hook||(e.data.hook={}),"insert",function(){var t=n.parentNode,r=t&&t._pending&&t._pending[e.key];r&&r.context===e.context&&r.tag===e.tag&&r.elm._leaveCb&&r.elm._leaveCb(),C&&C(n,T)},"transition-insert"),x&&x(n),O&&(Ht(n,$),Ht(n,w),Ft(function(){Ut(n,$),T.cancelled||S||Bt(n,a,T)})),e.data.show&&(t&&t(),C&&C(n,T)),O||S||T()}}}function qt(e,t){function n(){m.cancelled||(e.data.show||((r.parentNode._pending||(r.parentNode._pending={}))[e.key]=e),l&&l(r),v&&(Ht(r,s),Ht(r,c),Ft(function(){Ut(r,s),m.cancelled||h||Bt(r,o,m)})),u&&u(r,m),v||h||m())}var r=e.elm;r._enterCb&&(r._enterCb.cancelled=!0,r._enterCb());var i=Wt(e.data.transition);if(!i)return t();if(!r._leaveCb&&1===r.nodeType){var a=i.css,o=i.type,s=i.leaveClass,c=i.leaveActiveClass,l=i.beforeLeave,u=i.leave,f=i.afterLeave,d=i.leaveCancelled,p=i.delayLeave,v=a!==!1&&!pi,h=u&&(u._length||u.length)>1,m=r._leaveCb=Zt(function(){r.parentNode&&r.parentNode._pending&&(r.parentNode._pending[e.key]=null),v&&Ut(r,c),m.cancelled?(v&&Ut(r,s),d&&d(r)):(t(),f&&f(r)),r._leaveCb=null});p?p(n):n()}}function Wt(e){if(e){if("object"==typeof e){var t={};return e.css!==!1&&l(t,Ua(e.name||"v")),l(t,e),t}return"string"==typeof e?Ua(e):void 0}}function Zt(e){var t=!1;return function(){t||(t=!0,e())}}function Gt(e,t){t.data.show||Kt(t)}function Yt(e,t,n){var r=t.value,i=e.multiple;if(!i||Array.isArray(r)){for(var a,o,s=0,c=e.options.length;s-1,o.selected!==a&&(o.selected=a);else if(h(Xt(o),r))return void(e.selectedIndex!==s&&(e.selectedIndex=s));i||(e.selectedIndex=-1)}}function Qt(e,t){for(var n=0,r=t.length;n',n.innerHTML.indexOf(t)>0}function pn(e){return eo=eo||document.createElement("div"),eo.innerHTML=e,eo.textContent}function vn(e,t){return t&&(e=e.replace(Zo,"\n")),e.replace(qo,"<").replace(Wo,">").replace(Go,"&").replace(Yo,'"')}function hn(e,t){function n(t){f+=t,e=e.substring(t)}function r(){var t=e.match(fo);if(t){var r={tagName:t[1],attrs:[],start:f};n(t[0].length);for(var i,a;!(i=e.match(po))&&(a=e.match(co));)n(a[0].length),r.attrs.push(a);if(i)return r.unarySlash=i[1],n(i[0].length),r.end=f,r}}function i(e){var n=e.tagName,r=e.unarySlash;l&&("p"===s&&io(n)&&a("",s),ro(n)&&s===n&&a("",n));for(var i=u(n)||"html"===n&&"head"===s||!!r,o=e.attrs.length,f=new Array(o),d=0;d=0&&c[a].tag.toLowerCase()!==o;a--);}else a=0;if(a>=0){for(var l=c.length-1;l>=a;l--)t.end&&t.end(c[l].tag,r,i);c.length=a,s=a&&c[a-1].tag}else"br"===n.toLowerCase()?t.start&&t.start(n,[],!0,r,i):"p"===n.toLowerCase()&&(t.start&&t.start(n,[],!1,r,i),t.end&&t.end(n,r,i))}for(var o,s,c=[],l=t.expectHTML,u=t.isUnaryTag||ai,f=0;e;){if(o=e,s&&Jo(s,t.sfc,c)){var d=s.toLowerCase(),p=Ko[d]||(Ko[d]=new RegExp("([\\s\\S]*?)(]*>)","i")),v=0,h=e.replace(p,function(e,n,r){return v=r.length,"script"!==d&&"style"!==d&&"noscript"!==d&&(n=n.replace(//g,"$1").replace(//g,"$1")),t.chars&&t.chars(n),""});f+=e.length-h.length,e=h,a("",d,f-v,f)}else{var m=e.indexOf("<");if(0===m){if(mo.test(e)){var g=e.indexOf("-->");if(g>=0){n(g+3);continue}}if(go.test(e)){var y=e.indexOf("]>");if(y>=0){n(y+2);continue}}var _=e.match(ho);if(_){n(_[0].length);continue}var b=e.match(vo);if(b){var $=f;n(b[0].length),a(b[0],b[1],$,f);continue}var w=r();if(w){i(w);continue}}var x=void 0,C=void 0,k=void 0;if(m>0){for(C=e.slice(m);!(vo.test(C)||fo.test(C)||mo.test(C)||go.test(C)||(k=C.indexOf("<",1),k<0));)m+=k,C=e.slice(m);x=e.substring(0,m),n(m)}m<0&&(x=e,e=""),t.chars&&x&&t.chars(x)}if(e===o&&t.chars){t.chars(e);break}}a()}function mn(e){function t(){(o||(o=[])).push(e.slice(v,i).trim()),v=i+1}var n,r,i,a,o,s=!1,c=!1,l=!1,u=!1,f=0,d=0,p=0,v=0;for(i=0;i=0&&(m=e.charAt(h)," "===m);h--);m&&/[\w$]/.test(m)||(u=!0)}}else void 0===a?(v=i+1,a=e.slice(0,i).trim()):t();if(void 0===a?a=e.slice(0,i).trim():0!==v&&t(),o)for(i=0;io&&a.push(JSON.stringify(e.slice(o,i)));var s=mn(r[1].trim());a.push("_s("+s+")"),o=i+r[0].length}return o=_o}function En(e){return 34===e||39===e}function jn(e){var t=1;for(xo=wo;!Tn();)if(e=Sn(),En(e))Nn(e);else if(91===e&&t++,93===e&&t--,0===t){Co=wo;break}}function Nn(e){for(var t=e;!Tn()&&(e=Sn(),e!==t););}function Ln(e,t){ko=t.warn||_n,Ao=t.getTagNamespace||ai,Oo=t.mustUseProp||ai,So=t.isPreTag||ai,To=bn(t.modules,"preTransformNode"),Eo=bn(t.modules,"transformNode"),jo=bn(t.modules,"postTransformNode"),No=t.delimiters;var n,r,i=[],a=t.preserveWhitespace!==!1,o=!1,s=!1;return hn(e,{expectHTML:t.expectHTML,isUnaryTag:t.isUnaryTag,shouldDecodeNewlines:t.shouldDecodeNewlines,start:function(e,a,c){function l(e){}var u=r&&r.ns||Ao(e);di&&"svg"===u&&(a=Yn(a));var f={type:1,tag:e,attrsList:a,attrsMap:Wn(a),parent:r,children:[]};u&&(f.ns=u),Gn(f)&&!gi()&&(f.forbidden=!0);for(var d=0;d-1:_q("+t+","+a+")"),Cn(e,"change","var $$a="+t+",$$el=$event.target,$$c=$$el.checked?("+a+"):("+o+");if(Array.isArray($$a)){var $$v="+(r?"_n("+i+")":i)+",$$i=_i($$a,$$v);if($$c){$$i<0&&("+t+"=$$a.concat($$v))}else{$$i>-1&&("+t+"=$$a.slice(0,$$i).concat($$a.slice($$i+1)))}}else{"+t+"=$$c}",null,!0)}function Ir(e,t,n){var r=n&&n.number,i=kn(e,"value")||"null";i=r?"_n("+i+")":i,$n(e,"checked","_q("+t+","+i+")"),Cn(e,"change",Ur(t,i),null,!0)}function Fr(e,t,n){var r=e.attrsMap.type,i=n||{},a=i.lazy,o=i.number,s=i.trim,c=a||di&&"range"===r?"change":"input",l=!a&&"range"!==r,u="input"===e.tag||"textarea"===e.tag,f=u?"$event.target.value"+(s?".trim()":""):s?"(typeof $event === 'string' ? $event.trim() : $event)":"$event";f=o||"number"===r?"_n("+f+")":f;var d=Ur(t,f);u&&l&&(d="if($event.target.composing)return;"+d),$n(e,"value",u?"_s("+t+")":"("+t+")"),Cn(e,c,d,null,!0),(s||o||"number"===r)&&Cn(e,"blur","$forceUpdate()")}function Hr(e,t,n){var r=n&&n.number,i='Array.prototype.filter.call($event.target.options,function(o){return o.selected}).map(function(o){var val = "_value" in o ? o._value : o.value;return '+(r?"_n(val)":"val")+"})"+(null==e.attrsMap.multiple?"[0]":""),a=Ur(t,i);Cn(e,"change",a,null,!0)}function Ur(e,t){var n=On(e);return null===n.idx?e+"="+t:"var $$exp = "+n.exp+", $$idx = "+n.idx+";if (!Array.isArray($$exp)){"+e+"="+t+"}else{$$exp.splice($$idx, 1, "+t+")}"}function Br(e,t){t.value&&$n(e,"textContent","_s("+t.value+")")}function zr(e,t){t.value&&$n(e,"innerHTML","_s("+t.value+")")}function Vr(e,t){return t=t?l(l({},ws),t):ws,jr(e,t)}function Jr(e,t,n){var r=(t&&t.warn||$i,t&&t.delimiters?String(t.delimiters)+e:e);if($s[r])return $s[r];var i={},a=Vr(e,t);i.render=Kr(a.render);var o=a.staticRenderFns.length;i.staticRenderFns=new Array(o);for(var s=0;s0,vi=fi&&fi.indexOf("edge/")>0,hi=fi&&fi.indexOf("android")>0,mi=fi&&/iphone|ipad|ipod|ios/.test(fi),gi=function(){return void 0===Wr&&(Wr=!ui&&"undefined"!=typeof global&&"server"===global.process.env.VUE_ENV),Wr},yi=ui&&window.__VUE_DEVTOOLS_GLOBAL_HOOK__,_i=function(){function e(){r=!1;var e=n.slice(0);n.length=0;for(var t=0;t1&&(t[n[0].trim()]=n[1].trim())}}),t}),Aa=/^--/,Oa=/\s*!important$/,Sa=function(e,t,n){Aa.test(t)?e.style.setProperty(t,n):Oa.test(n)?e.style.setProperty(t,n.replace(Oa,""),"important"):e.style[Ea(t)]=n},Ta=["Webkit","Moz","ms"],Ea=o(function(e){if(Xi=Xi||document.createElement("div"),e=Xr(e),"filter"!==e&&e in Xi.style)return e;for(var t=e.charAt(0).toUpperCase()+e.slice(1),n=0;n\/=]+)/,oo=/(?:=)/,so=[/"([^"]*)"+/.source,/'([^']*)'+/.source,/([^\s"'=<>`]+)/.source],co=new RegExp("^\\s*"+ao.source+"(?:\\s*("+oo.source+")\\s*(?:"+so.join("|")+"))?"),lo="[a-zA-Z_][\\w\\-\\.]*",uo="((?:"+lo+"\\:)?"+lo+")",fo=new RegExp("^<"+uo),po=/^\s*(\/?)>/,vo=new RegExp("^<\\/"+uo+"[^>]*>"),ho=/^]+>/i,mo=/^ - {% if message != '' %} -
- × -

- {% raw message %} -

+ + - {% end %} - - -
- {% block content %}{% end %}
- -