Analysis refactor GUI part1 (#2076)

* fix #1505

* improving some GUI stuff

* improving some GUI stuff - missing lines

* addressing all comments

* ready for review

* fix #1987

* initial commit

* requested changes

* fix filter job list

* Fixing server cert (#2051)

* fix get_studies

* flake8

* fix #503

* fix #2010

* fix #1913

* fix errors

* addressing @josenavas comment

* flake8

* fix #1010

* fix #1066 (#2058)

* addressing @josenavas comments

* fix #1961

* fix #1837

* Automatic jobs & new stats (#2057)

* fix #814, fix #1636

* fixing error in test-env

* fixing stats.html call

* adding img

* addressing @josenavas comments

* rm for loops

* addresssing @ElDeveloper comments

* generalizing this functionality

* fix #1816

* fix #1959

* addressing @josenavas comments

* addressing @josenavas comments

* fixing error

* fixed?

* addressing @josenavas comments

* addressing @wasade comments

* fix flake8

* generate biom and metadata release (#2066)

* initial commit

* adding portal

* addressing @josenavas comments

* pid -> qiita_artifact_id

* addressing @josenavas comments

* addressing @ElDeveloper comments

* rm 50.sql

* database changes to fix 969

* adding delete

* addressing @josenavas comments

* addressing @ElDeveloper comments

* duh!

* fix generate_biom_and_metadata_release (#2072)

* fix generate_biom_and_metadata_release

* addressing @ElDeveloper comment

* Removing qiita ware code that will not be used anymore

* Organizing the handlers and new analysis description page

* Addressing @antgonza's comments

qiita_pet/handlers/analysis_handlers/__init__.py

@@ -0,0 +1,18 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) 2014--, The Qiita Development Team.
+#
+# Distributed under the terms of the BSD 3-clause License.
+#
+# The full license is in the file LICENSE, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from .util import check_analysis_access
+from .base_handlers import (CreateAnalysisHandler, AnalysisDescriptionHandler,
+                            AnalysisGraphHandler)
+from .listing_handlers import (ListAnalysesHandler, AnalysisSummaryAJAX,
+                               SelectedSamplesHandler)
+
+__all__ = ['CreateAnalysisHandler', 'AnalysisDescriptionHandler',
+           'AnalysisGraphHandler', 'ListAnalysesHandler',
+           'AnalysisSummaryAJAX', 'SelectedSamplesHandler',
+           'check_analysis_access']

qiita_pet/handlers/analysis_handlers/base_handlers.py

@@ -0,0 +1,105 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) 2014--, The Qiita Development Team.
+#
+# Distributed under the terms of the BSD 3-clause License.
+#
+# The full license is in the file LICENSE, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from tornado.web import authenticated
+
+from qiita_core.util import execute_as_transaction
+from qiita_core.qiita_settings import qiita_config
+from qiita_pet.handlers.base_handlers import BaseHandler
+from qiita_pet.handlers.analysis_handlers import check_analysis_access
+from qiita_pet.handlers.util import to_int
+from qiita_db.analysis import Analysis
+
+
+class CreateAnalysisHandler(BaseHandler):
+    @authenticated
+    @execute_as_transaction
+    def post(self):
+        name = self.get_argument('name')
+        desc = self.get_argument('description')
+        analysis = Analysis.create(self.current_user, name, desc,
+                                   from_default=True)
+
+        self.redirect(u"%s/analysis/description/%s/"
+                      % (qiita_config.portal_dir, analysis.id))
+
+
+class AnalysisDescriptionHandler(BaseHandler):
+    @authenticated
+    @execute_as_transaction
+    def get(self, analysis_id):
+        analysis = Analysis(analysis_id)
+        check_analysis_access(self.current_user, analysis)
+
+        self.render("analysis_description.html", analysis_name=analysis.name,
+                    analysis_id=analysis_id,
+                    analysis_description=analysis.description)
+
+
+def analyisis_graph_handler_get_request(analysis_id, user):
+    """Returns the graph information of the analysis
+
+    Parameters
+    ----------
+    analysis_id : int
+        The analysis id
+    user : qiita_db.user.User
+        The user performing the request
+
+    Returns
+    -------
+    dict with the graph information
+    """
+    analysis = Analysis(analysis_id)
+    # Check if the user actually has access to the analysis
+    check_analysis_access(user, analysis)
+
+    # A user has full access to the analysis if it is one of its private
+    # analyses, the analysis has been shared with the user or the user is a
+    # superuser or admin
+    full_access = (analysis in (user.private_analyses | user.shared_analyses)
+                   or user.level in {'superuser', 'admin'})
+
+    nodes = set()
+    edges = set()
+    # Loop through all the initial artifacts of the analysis
+    for a in analysis.artifacts:
+        g = a.descendants_with_jobs
+        # Loop through all the nodes in artifact descendants graph
+        for n in g.nodes():
+            # Get if the object is an artifact or a job
+            obj_type = n[0]
+            # Get the actual object
+            obj = n[1]
+            if obj_type == 'job':
+                name = obj.command.name
+            elif not full_access and not obj.visibility == 'public':
+                # The object is an artifact, it is not public and the user
+                # doesn't have full access, so we don't include it in the
+                # graph
+                continue
+            else:
+                name = '%s - %s' % (obj.name, obj.artifact_type)
+            nodes.add((obj_type, obj.id, name))
+
+        edges.update({(s[1].id, t[1].id) for s, t in g.edges()})
+
+    # Nodes and Edges are sets, but the set object can't be serialized using
+    # JSON. Transforming them to lists so when this is returned to the GUI
+    # over HTTP can be JSONized.
+    return {'edges': list(edges), 'nodes': list(nodes)}
+
+
+class AnalysisGraphHandler(BaseHandler):
+    @authenticated
+    @execute_as_transaction
+    def get(self):
+        analysis_id = to_int(self.get_argument('analysis_id'))
+        response = analyisis_graph_handler_get_request(
+            analysis_id, self.current_user)
+        self.write(response)

qiita_pet/handlers/analysis_handlers/listing_handlers.py

@@ -0,0 +1,135 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) 2014--, The Qiita Development Team.
+#
+# Distributed under the terms of the BSD 3-clause License.
+#
+# The full license is in the file LICENSE, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from functools import partial
+from json import dumps
+from collections import defaultdict
+from future.utils import viewitems
+
+from tornado.web import authenticated
+
+from qiita_core.qiita_settings import qiita_config
+from qiita_core.util import execute_as_transaction
+from qiita_pet.handlers.base_handlers import BaseHandler
+from qiita_pet.handlers.util import download_link_or_path
+from qiita_pet.handlers.analysis_handlers import check_analysis_access
+from qiita_pet.util import is_localhost
+from qiita_db.util import get_filepath_id
+from qiita_db.analysis import Analysis
+from qiita_db.logger import LogEntry
+from qiita_db.reference import Reference
+from qiita_db.artifact import Artifact
+
+
+class ListAnalysesHandler(BaseHandler):
+    @authenticated
+    @execute_as_transaction
+    def get(self):
+        message = self.get_argument('message', '')
+        level = self.get_argument('level', '')
+        user = self.current_user
+
+        analyses = user.shared_analyses | user.private_analyses
+
+        is_local_request = is_localhost(self.request.headers['host'])
+        gfi = partial(get_filepath_id, 'analysis')
+        dlop = partial(download_link_or_path, is_local_request)
+        mappings = {}
+        bioms = {}
+        tgzs = {}
+        for analysis in analyses:
+            _id = analysis.id
+            # getting mapping file
+            mapping = analysis.mapping_file
+            if mapping is not None:
+                mappings[_id] = dlop(mapping, gfi(mapping), 'mapping file')
+            else:
+                mappings[_id] = ''
+
+            bioms[_id] = ''
+            # getting tgz file
+            tgz = analysis.tgz
+            if tgz is not None:
+                tgzs[_id] = dlop(tgz, gfi(tgz), 'tgz file')
+            else:
+                tgzs[_id] = ''
+
+        self.render("list_analyses.html", analyses=analyses, message=message,
+                    level=level, is_local_request=is_local_request,
+                    mappings=mappings, bioms=bioms, tgzs=tgzs)
+
+    @authenticated
+    @execute_as_transaction
+    def post(self):
+        analysis_id = int(self.get_argument('analysis_id'))
+        analysis = Analysis(analysis_id)
+        analysis_name = analysis.name.decode('utf-8')
+
+        check_analysis_access(self.current_user, analysis)
+
+        try:
+            Analysis.delete(analysis_id)
+            msg = ("Analysis <b><i>%s</i></b> has been deleted." % (
+                analysis_name))
+            level = "success"
+        except Exception as e:
+            e = str(e)
+            msg = ("Couldn't remove <b><i>%s</i></b> analysis: %s" % (
+                analysis_name, e))
+            level = "danger"
+            LogEntry.create('Runtime', "Couldn't remove analysis ID %d: %s" %
+                            (analysis_id, e))
+
+        self.redirect(u"%s/analysis/list/?level=%s&message=%s"
+                      % (qiita_config.portal_dir, level, msg))
+
+
+class AnalysisSummaryAJAX(BaseHandler):
+    @authenticated
+    @execute_as_transaction
+    def get(self):
+        info = self.current_user.default_analysis.summary_data()
+        self.write(dumps(info))
+
+
+class SelectedSamplesHandler(BaseHandler):
+    @authenticated
+    @execute_as_transaction
+    def get(self):
+        # Format sel_data to get study IDs for the processed data
+        sel_data = defaultdict(dict)
+        proc_data_info = {}
+        sel_samps = self.current_user.default_analysis.samples
+        for aid, samples in viewitems(sel_samps):
+            a = Artifact(aid)
+            sel_data[a.study][aid] = samples
+            # Also get processed data info
+            processing_parameters = a.processing_parameters
+            if processing_parameters is None:
+                params = None
+                algorithm = None
+            else:
+                cmd = processing_parameters.command
+                params = processing_parameters.values
+                if 'reference' in params:
+                    ref = Reference(params['reference'])
+                    del params['reference']
+
+                    params['reference_name'] = ref.name
+                    params['reference_version'] = ref.version
+                algorithm = '%s (%s)' % (cmd.software.name, cmd.name)
+
+            proc_data_info[aid] = {
+                'processed_date': str(a.timestamp),
+                'algorithm': algorithm,
+                'data_type': a.data_type,
+                'params': params
+            }
+
+        self.render("analysis_selected.html", sel_data=sel_data,
+                    proc_info=proc_data_info)

qiita_pet/handlers/analysis_handlers/tests/__init__.py

@@ -0,0 +1,7 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) 2014--, The Qiita Development Team.
+#
+# Distributed under the terms of the BSD 3-clause License.
+#
+# The full license is in the file LICENSE, distributed with this software.
+# -----------------------------------------------------------------------------

qiita_pet/handlers/analysis_handlers/tests/test_base_handlers.py

@@ -0,0 +1,83 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) 2014--, The Qiita Development Team.
+#
+# Distributed under the terms of the BSD 3-clause License.
+#
+# The full license is in the file LICENSE, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from unittest import TestCase, main
+from json import loads
+
+from tornado.web import HTTPError
+
+from qiita_db.user import User
+from qiita_db.analysis import Analysis
+from qiita_pet.test.tornado_test_base import TestHandlerBase
+from qiita_pet.handlers.analysis_handlers.base_handlers import (
+    analyisis_graph_handler_get_request)
+
+
+class TestBaseHandlersUtils(TestCase):
+    def test_analyisis_graph_handler_get_request(self):
+        obs = analyisis_graph_handler_get_request(1, User('test@foo.bar'))
+        # The job id is randomly generated in the test environment. Gather
+        # it here. There is only 1 job in the first artifact of the analysis
+        job_id = Analysis(1).artifacts[0].jobs()[0].id
+        exp = {'edges': [(8, job_id), (job_id, 9)],
+               'nodes': [('job', job_id, 'Single Rarefaction'),
+                         ('artifact', 9, 'noname - BIOM'),
+                         ('artifact', 8, 'noname - BIOM')]}
+        self.assertItemsEqual(obs, exp)
+        self.assertItemsEqual(obs['edges'], exp['edges'])
+        self.assertItemsEqual(obs['nodes'], exp['nodes'])
+
+        # An admin has full access to the analysis
+        obs = analyisis_graph_handler_get_request(1, User('admin@foo.bar'))
+        self.assertItemsEqual(obs, exp)
+        self.assertItemsEqual(obs['edges'], exp['edges'])
+        self.assertItemsEqual(obs['nodes'], exp['nodes'])
+
+        # If the analysis is shared with the user he also has access
+        obs = analyisis_graph_handler_get_request(1, User('shared@foo.bar'))
+        self.assertItemsEqual(obs, exp)
+        self.assertItemsEqual(obs['edges'], exp['edges'])
+        self.assertItemsEqual(obs['nodes'], exp['nodes'])
+
+        # The user doesn't have access to the analysis
+        with self.assertRaises(HTTPError):
+            analyisis_graph_handler_get_request(1, User('demo@microbio.me'))
+
+
+class TestBaseHandlers(TestHandlerBase):
+    def test_post_create_analysis_handler(self):
+        args = {'name': 'New Test Analysis',
+                'description': 'Test Analysis Description'}
+        response = self.post('/analysis/create/', args)
+        self.assertRegexpMatches(
+            response.effective_url,
+            r"http://localhost:\d+/analysis/description/\d+/")
+        self.assertEqual(response.code, 200)
+
+    def test_get_analysis_description_handler(self):
+        response = self.get('/analysis/description/1/')
+        self.assertEqual(response.code, 200)
+
+    def test_get_analysis_graph_handler(self):
+        response = self.get('/analysis/description/graph/', {'analysis_id': 1})
+        self.assertEqual(response.code, 200)
+        # The job id is randomly generated in the test environment. Gather
+        # it here. There is only 1 job in the first artifact of the analysis
+        job_id = Analysis(1).artifacts[0].jobs()[0].id
+        obs = loads(response.body)
+        exp = {'edges': [[8, job_id], [job_id, 9]],
+               'nodes': [['job', job_id, 'Single Rarefaction'],
+                         ['artifact', 9, 'noname - BIOM'],
+                         ['artifact', 8, 'noname - BIOM']]}
+        self.assertItemsEqual(obs, exp)
+        self.assertItemsEqual(obs['edges'], exp['edges'])
+        self.assertItemsEqual(obs['nodes'], exp['nodes'])
+
+
+if __name__ == '__main__':
+    main()

qiita_pet/handlers/analysis_handlers/tests/test_listing_handlers.py

@@ -0,0 +1,32 @@
+# -----------------------------------------------------------------------------
+# Copyright (c) 2014--, The Qiita Development Team.
+#
+# Distributed under the terms of the BSD 3-clause License.
+#
+# The full license is in the file LICENSE, distributed with this software.
+# -----------------------------------------------------------------------------
+
+from unittest import main
+from json import loads
+
+from qiita_pet.test.tornado_test_base import TestHandlerBase
+
+
+class TestListingHandlers(TestHandlerBase):
+    def test_get_list_analyses_handler(self):
+        response = self.get('/analysis/list/')
+        self.assertEqual(response.code, 200)
+
+    def test_get_analysis_summary_ajax(self):
+        response = self.get('/analysis/dflt/sumary/')
+        self.assertEqual(response.code, 200)
+        self.assertEqual(loads(response.body),
+                         {"artifacts": 1, "studies": 1, "samples": 4})
+
+    def test_get_selected_samples_handler(self):
+        response = self.get('/analysis/selected/')
+        # Make sure page response loaded sucessfully
+        self.assertEqual(response.code, 200)
+
+if __name__ == '__main__':
+    main()