Let's get redbiom-ing #2118
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,163 @@ | ||
| from tornado.gen import coroutine, Task | ||
|
There was a problem hiding this comment. This file is missing the copyright header. |
||
|
|
||
| from qiita_core.util import execute_as_transaction | ||
|
|
||
| from .base_handlers import BaseHandler | ||
| from requests import ConnectionError | ||
| import redbiom.summarize | ||
|
There was a problem hiding this comment. Minor, but these imports are not following the PEP8 standard (and they're not flagged by pep8). The order of the imports should be:
Also, is there any problem on simply doing There was a problem hiding this comment.
In [1]: import scipy
In [2]: scipy.sparse.csr_matrix
---------------------------------------------------------------------------
AttributeError Traceback (most recent call last)
<ipython-input-2-c4d1267aee28> in <module>()
----> 1 scipy.sparse.csr_matrix
AttributeError: module 'scipy' has no attribute 'sparse'
In [3]: import scipy.sparse
In [4]: scipy.sparse.csr_matrix
Out[4]: scipy.sparse.csr.csr_matrixThere was a problem hiding this comment. changing blocks, thanks @wasade for your comment. |
||
| import redbiom.search | ||
| import redbiom._requests | ||
| import redbiom.util | ||
| import redbiom.fetch | ||
|
|
||
|
|
||
| class RedbiomPublicSearch(BaseHandler): | ||
| @execute_as_transaction | ||
| def get(self, search): | ||
| self.render('redbiom.html') | ||
|
|
||
| @execute_as_transaction | ||
| def _redbiom_search(self, query, search_on, callback): | ||
| error = False | ||
| message = '' | ||
| results = [] | ||
|
|
||
| try: | ||
| df = redbiom.summarize.contexts() | ||
| except ConnectionError: | ||
|
There was a problem hiding this comment. What if there is some other error raised? Are we allowing the 500 to be raised on the user screen or should we catch it? There was a problem hiding this comment. I'm not sure, I can see pros/cons on both (catch everything or just specific errors). I decided to go with specific so we can narrow down why it's failing. Note that we can see the errors in the logs if not caught. -- BTW I think we have caught all of them at this point. There was a problem hiding this comment. Ok, just wanted to make sure that this was the desired behavior. |
||
| error = True | ||
| message = 'Redbiom is down - contact admin, thanks!' | ||
|
|
||
| if not error: | ||
| contexts = df.ContextName.values | ||
| query = query.lower() | ||
| samples, categories = [], [] | ||
|
|
||
| if search_on == 'metadata': | ||
|
There was a problem hiding this comment. we may want an additional There was a problem hiding this comment. yup, let's chat this afternoon There was a problem hiding this comment. It seems that this block of code can be simplified to avoid code duplication: if search_on in ('metadata', 'categories'):
try:
samples = redbiom.serach.metadata_full(query, categories=(search_on == 'categories'))
except ... # Perform error handling as in the "categories" case.
elif search_on == 'observations':
... |
||
| try: | ||
| samples = redbiom.search.metadata_full( | ||
| query, categories=False) | ||
| except TypeError: | ||
|
There was a problem hiding this comment. Same here regarding error (will not comment again but applies to all) |
||
| error = True | ||
| message = ('Not a valid search: "%s", are you sure this ' | ||
| 'is a valid metadata value?' % query) | ||
| elif search_on == 'categories': | ||
| try: | ||
| categories = redbiom.search.metadata_full(query, | ||
| categories=True) | ||
| except ValueError: | ||
| error = True | ||
| message = ('Not a valid search: "%s", try a longer query' | ||
|
There was a problem hiding this comment. I'm not sure if this error message is helpful enough for the user, i.e. I don't know that "longer query" means. There was a problem hiding this comment. the error is basically because the query is actually too small (too few chars, for example searching for 'a' or 'tt' check the test) ... trying to improve the message, hope this helps |
||
| % query) | ||
| except TypeError: | ||
| error = True | ||
| message = ('Not a valid search: "%s", are you sure this ' | ||
| 'is a valid metadata category?' % query) | ||
| elif search_on == 'observations': | ||
| samples = [s.split('_', 1)[1] for context in contexts | ||
| for s in redbiom.util.samples_from_observations( | ||
| query.split(' '), True, context)] | ||
|
There was a problem hiding this comment. the There was a problem hiding this comment. yup, also within our discussion today. |
||
| else: | ||
| error = True | ||
| message = ('Incorrect search by: you can use observations ' | ||
| 'or metadata and you passed: %s' % search_on) | ||
|
|
||
| if not error: | ||
| import qiita_db as qdb | ||
| import qiita_db.sql_connection as qdbsc | ||
| if samples: | ||
| sql = """ | ||
| WITH main_query AS ( | ||
| SELECT study_title, study_id, artifact_id, | ||
| array_agg(DISTINCT sample_id) AS samples, | ||
| qiita.artifact_descendants(artifact_id) AS children | ||
| FROM qiita.study_prep_template | ||
| JOIN qiita.prep_template USING (prep_template_id) | ||
| JOIN qiita.prep_template_sample USING | ||
| (prep_template_id) | ||
| JOIN qiita.study USING (study_id) | ||
| WHERE sample_id IN %s | ||
| GROUP BY study_title, study_id, artifact_id) | ||
| SELECT study_title, study_id, samples, name, command_id, | ||
|
There was a problem hiding this comment. change: There was a problem hiding this comment. changing but not sure what's the benefit, can you expand? There was a problem hiding this comment. note that I was changing it and then realized that nr doesn't have all the members of the query added, for example cid/command_id, so this change then will require del nr['cid'] etc ... not worth it! |
||
| (main_query.children).artifact_id AS artifact_id | ||
| FROM main_query | ||
| JOIN qiita.artifact a ON | ||
| (main_query.children).artifact_id = a.artifact_id | ||
| JOIN qiita.artifact_type at ON ( | ||
| at.artifact_type_id = a.artifact_type_id | ||
| AND artifact_type = 'BIOM') | ||
| ORDER BY artifact_id | ||
| """ | ||
| with qdbsc.TRN: | ||
| qdbsc.TRN.add(sql, [tuple(samples)]) | ||
| results = [] | ||
| commands = {} | ||
| for row in qdbsc.TRN.execute_fetchindex(): | ||
| title, sid, samples, name, cid, aid = row | ||
| nr = {'study_title': title, 'study_id': sid, | ||
| 'artifact_id': aid, 'aname': name, | ||
| 'samples': samples} | ||
| if cid is not None: | ||
| if cid not in commands: | ||
| c = qdb.software.Command(cid) | ||
| commands[cid] = { | ||
| 'sfwn': c.software.name, | ||
| 'sfv': c.software.version, | ||
| 'cmdn': c.name | ||
| } | ||
| nr['command'] = commands[cid]['cmdn'] | ||
| nr['software'] = commands[cid]['sfwn'] | ||
| nr['version'] = commands[cid]['sfv'] | ||
| else: | ||
| nr['command'] = None | ||
| nr['software'] = None | ||
| nr['version'] = None | ||
| results.append(nr) | ||
| elif categories: | ||
| sql = """ | ||
| WITH get_studies AS ( | ||
| SELECT | ||
| trim(table_name, 'sample_')::int AS study_id, | ||
| array_agg(column_name::text) AS columns | ||
| FROM information_schema.columns | ||
| WHERE column_name IN %s | ||
| AND table_name LIKE 'sample_%%' | ||
| AND table_name NOT IN ( | ||
| 'prep_template', 'prep_template_sample') | ||
| GROUP BY table_name) | ||
| SELECT study_title, get_studies.study_id, columns | ||
| -- artifact_id, samples | ||
|
There was a problem hiding this comment. It looks like this line is commented out on the SQL, can you remove? |
||
| FROM get_studies | ||
| JOIN qiita.study ON get_studies.study_id = | ||
| qiita.study.study_id""" | ||
| with qdbsc.TRN: | ||
| results = [] | ||
| qdbsc.TRN.add(sql, [tuple(categories)]) | ||
| for title, sid, cols in qdbsc.TRN.execute_fetchindex(): | ||
| nr = {'study_title': title, 'study_id': sid, | ||
| 'artifact_id': None, 'aname': None, | ||
| 'samples': cols, 'command': ', '.join(cols), | ||
| 'software': None, 'version': None} | ||
| results.append(nr) | ||
| else: | ||
| error = True | ||
| message = 'No samples where found! Try again ...' | ||
| callback((results, message)) | ||
|
|
||
| @coroutine | ||
| @execute_as_transaction | ||
| def post(self, search): | ||
| search = self.get_argument('search', None) | ||
| search_on = self.get_argument('search_on', None) | ||
|
|
||
| data = [] | ||
| if search is not None and search and search != ' ': | ||
| if search_on in ('observations', 'metadata', 'categories'): | ||
| data, msg = yield Task( | ||
| self._redbiom_search, search, search_on) | ||
| else: | ||
| msg = 'Not a valid option for search_on' | ||
| else: | ||
| msg = 'Nothing to search for ...' | ||
|
|
||
| self.write({'status': 'success', 'message': msg, 'data': data}) | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think this line and the one below can be removed. Then instead of using
${qdbd}on the other lines you can simply use `pwd`/qiita_db/. According to lines 40, 42, and 43 that should work w/o a problemThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
k