Merge branch 'release_18.01' into dev

client/galaxy/scripts/mvc/history/history-list.js

@@ -1,11 +1,60 @@
 import _l from "utils/localization";
-/** This class renders the grid list. */
+import AjaxQueue from "utils/ajax-queue";
 import Utils from "utils/utils";
+/** This class renders the grid list. */
 import GridView from "mvc/grid/grid-view";
 import HistoryModel from "mvc/history/history-model";
 import historyCopyDialog from "mvc/history/copy-dialog";
 
 var HistoryGridView = GridView.extend({
+    initialize: function(grid_config) {
+        this.ajaxQueue = new AjaxQueue.AjaxQueue();
+        GridView.prototype.initialize.call(this, grid_config);
+    },
+
+    init_grid_elements: function() {
+        const ajaxQueue = this.ajaxQueue;
+        ajaxQueue.stop();
+        GridView.prototype.init_grid_elements.call(this);
+        const fetchDetails = $.makeArray(
+            this.$el.find(".delayed-value-datasets_by_state").map((i, el) => {
+                return () => {
+                    const historyId = $(el).data("history-id");
+                    const url = `${
+                        Galaxy.root
+                    }api/histories/${historyId}?keys=nice_size,contents_active,contents_states`;
+                    const options = {};
+                    options.url = url;
+                    options.type = "GET";
+                    options.success = req => {
+                        const contentsStates = req["contents_states"];
+                        let stateHtml = "";
+                        for (let state of ["ok", "running", "queued", "new", "error"]) {
+                            const stateCount = contentsStates[state];
+                            if (stateCount) {
+                                stateHtml += `<div class="count-box state-color-${state}" title="Datasets in ${state} state">${stateCount}</div> `;
+                            }
+                        }
+                        const contentsActive = req["contents_active"];
+                        const deleted = contentsActive["deleted"];
+                        if (deleted) {
+                            stateHtml += `<div class="count-box state-color-deleted" title="Deleted datasets">${deleted}</div> `;
+                        }
+                        const hidden = contentsActive["hidden"];
+                        if (hidden) {
+                            stateHtml += `<div class="count-box state-color-hidden" title="Hidden datasets">${hidden}</div> `;
+                        }
+                        $(`.delayed-value-datasets_by_state[data-history-id='${historyId}']`).html(stateHtml);
+                        $(`.delayed-value-disk_size[data-history-id='${historyId}']`).html(req["nice_size"]);
+                    };
+                    var xhr = jQuery.ajax(options);
+                    return xhr;
+                };
+            })
+        );
+        fetchDetails.forEach(fn => ajaxQueue.add(fn));
+        ajaxQueue.start();
+    },
     _showCopyDialog: function(id) {
         var history = new HistoryModel.History({ id: id });
         history

client/galaxy/style/less/base.less

@@ -1178,8 +1178,15 @@ ul.manage-table-actions li {
 }
 
 .state-color-deleted {
-    border-color: @state-deleted-border;
-    background: @state-deleted-bg;
+    border-color: darken(@state-default-border, 30%);
+    border-style: dotted;
+    background: darken(@state-default-bg, 30%);
+}
+
+.state-color-hidden {
+    border-color: @state-default-border;
+    border-style: dotted;
+    background: @state-default-bg;
 }
 
 .state-fg-new {
@@ -1798,7 +1805,7 @@ div.toolTitleNoSection
     display: none;
 }
 
-// communication channel bootstrap modal 
+// communication channel bootstrap modal
 .chat-modal {
     overflow: hidden;
 }
@@ -1813,7 +1820,7 @@ div.toolTitleNoSection
 }
 
 .close-modal {
-    float: right; 
+    float: right;
     cursor: pointer;
 }
 

client/galaxy/style/less/galaxy_variables.less

@@ -1,5 +1,5 @@
 // Additional variables that are not used by bootstrap but are used
-// in Galaxy stylesheets 
+// in Galaxy stylesheets
 
 @base-bg: @white;
 @base-text-color: @black;
@@ -29,8 +29,8 @@
 @state-running-border: #AAAA66;
 @state-running-bg: #FFFFCC;
 
-@state-deleted-border: #330066;
-@state-deleted-bg: #3399FF;
+// @state-deleted-border: #330066;
+// @state-deleted-bg: #3399FF;
 
 // Theme, expects tmp-site-config.less written by grunt
 @import "tmp-site-config.less";

lib/galaxy/jobs/runners/pulsar.py

@@ -251,7 +251,7 @@ def check_watched_item(self, job_state):
         job_state = self._update_job_state_for_status(job_state, status)
         return job_state
 
-    def _update_job_state_for_status(self, job_state, pulsar_status):
+    def _update_job_state_for_status(self, job_state, pulsar_status, full_status=None):
         if pulsar_status == "complete":
             self.mark_as_finished(job_state)
             return None
@@ -261,7 +261,7 @@ def _update_job_state_for_status(self, job_state, pulsar_status):
             else:
                 message = LOST_REMOTE_ERROR
             if not job_state.job_wrapper.get_job().finished:
-                self.fail_job(job_state, message)
+                self.fail_job(job_state, message, full_status=full_status)
             return None
         if pulsar_status == "running" and not job_state.running:
             job_state.running = True
@@ -517,10 +517,15 @@ def finish_job(self, job_state):
             log.exception("Job wrapper finish method failed")
             job_wrapper.fail("Unable to finish job", exception=True)
 
-    def fail_job(self, job_state, message=GENERIC_REMOTE_ERROR):
+    def fail_job(self, job_state, message=GENERIC_REMOTE_ERROR, full_status=None):
         """Seperated out so we can use the worker threads for it."""
         self.stop_job(self.sa_session.query(self.app.model.Job).get(job_state.job_wrapper.job_id))
-        job_state.job_wrapper.fail(getattr(job_state, "fail_message", message))
+        stdout = ""
+        stderr = ""
+        if full_status:
+            stdout = full_status.get("stdout", "")
+            stderr = full_status.get("stderr", "")
+        job_state.job_wrapper.fail(getattr(job_state, "fail_message", message), stdout=stdout, stderr=stderr)
 
     def check_pid(self, pid):
         try:
@@ -744,7 +749,7 @@ def __async_update(self, full_status):
             job_id = full_status["job_id"]
             job, job_wrapper = self.app.job_manager.job_handler.job_queue.job_pair_for_id(job_id)
             job_state = self._job_state(job, job_wrapper)
-            self._update_job_state_for_status(job_state, full_status["status"])
+            self._update_job_state_for_status(job_state, full_status["status"], full_status=full_status)
         except Exception:
             log.exception("Failed to update Pulsar job status for job_id %s", job_id)
             raise

lib/galaxy/managers/libraries.py

@@ -3,7 +3,7 @@
 """
 import logging
 
-from sqlalchemy import and_, false, not_, or_, true
+from sqlalchemy import false, not_, or_, true
 from sqlalchemy.orm.exc import MultipleResultsFound
 from sqlalchemy.orm.exc import NoResultFound
 
@@ -109,8 +109,11 @@ def list(self, trans, deleted=False):
 
         :returns: query that will emit all accessible libraries
         :rtype:   sqlalchemy query
-        :returns: set of library ids that have restricted access (not public)
-        :rtype:   set
+        :returns: dict of 3 sets with available actions for user's accessible
+                  libraries and a set of ids of all public libraries. These are
+                  used for limiting the number of queries when dictifying the
+                  libraries later on.
+        :rtype:   dict
         """
         is_admin = trans.user_is_admin()
         query = trans.sa_session.query(trans.app.model.Library)
@@ -119,6 +122,7 @@ def list(self, trans, deleted=False):
             trans.sa_session.query(trans.model.LibraryPermissions).filter(
                 trans.model.LibraryPermissions.table.c.action == library_access_action
             ).distinct())}
+        prefetched_ids = {'restricted_library_ids': restricted_library_ids}
         if is_admin:
             if deleted is None:
                 #  Flag is not specified, do not filter on it.
@@ -129,18 +133,33 @@ def list(self, trans, deleted=False):
                 query = query.filter(trans.app.model.Library.table.c.deleted == false())
         else:
             #  Nonadmins can't see deleted libraries
+            query = query.filter(trans.app.model.Library.table.c.deleted == false())
             current_user_role_ids = [role.id for role in trans.get_current_user_roles()]
-            accessible_restricted_library_ids = [lp.library_id for lp in (
-                trans.sa_session.query(trans.model.LibraryPermissions).filter(
-                    and_(
-                        trans.model.LibraryPermissions.table.c.action == library_access_action,
-                        trans.model.LibraryPermissions.table.c.role_id.in_(current_user_role_ids)
-                    )))]
+            all_actions = trans.sa_session.query(trans.model.LibraryPermissions).filter(trans.model.LibraryPermissions.table.c.role_id.in_(current_user_role_ids))
+            library_add_action = trans.app.security_agent.permitted_actions.LIBRARY_ADD.action
+            library_modify_action = trans.app.security_agent.permitted_actions.LIBRARY_MODIFY.action
+            library_manage_action = trans.app.security_agent.permitted_actions.LIBRARY_MANAGE.action
+            accessible_restricted_library_ids = []
+            allowed_library_add_ids = set([])
+            allowed_library_modify_ids = set([])
+            allowed_library_manage_ids = set([])
+            for action in all_actions:
+                if action.action == library_access_action:
+                    accessible_restricted_library_ids.add(action.library_id)
+                if action.action == library_add_action:
+                    allowed_library_add_ids.add(action.library_id)
+                if action.action == library_modify_action:
+                    allowed_library_modify_ids.add(action.library_id)
+                if action.action == library_manage_action:
+                    allowed_library_manage_ids.add(action.library_id)
             query = query.filter(or_(
                 not_(trans.model.Library.table.c.id.in_(restricted_library_ids)),
                 trans.model.Library.table.c.id.in_(accessible_restricted_library_ids)
             ))
-        return query, restricted_library_ids
+            prefetched_ids['allowed_library_add_ids'] = allowed_library_add_ids
+            prefetched_ids['allowed_library_modify_ids'] = allowed_library_modify_ids
+            prefetched_ids['allowed_library_manage_ids'] = allowed_library_manage_ids
+        return query, prefetched_ids
 
     def secure(self, trans, library, check_accessible=True):
         """
@@ -172,30 +191,38 @@ def check_accessible(self, trans, library):
         else:
             return library
 
-    def get_library_dict(self, trans, library, restricted_library_ids=None):
+    def get_library_dict(self, trans, library, prefetched_ids=None):
         """
         Return library data in the form of a dictionary.
 
-        :param  library:                    library
-        :type   library:                    galaxy.model.Library
-        :param  restricted_library_ids:     ids of restricted libraries to speed up the
-                                            detection of public libraries
-        :type   restricted_library_ids:     list of ints
+        :param  library:         library
+        :type   library:         galaxy.model.Library
+        :param  prefetched_ids:  dict of 3 sets with available actions for user's accessible
+                                 libraries and a set of ids of all public libraries. These are
+                                 used for limiting the number of queries when dictifying a
+                                 set of libraries.
+        :type   prefetched_ids:  dict
 
         :returns:   dict with data about the library
         :rtype:     dictionary
         """
+        restricted_library_ids = prefetched_ids.get('restricted_library_ids', None) if prefetched_ids else None
+        allowed_library_add_ids = prefetched_ids.get('allowed_library_add_ids', None) if prefetched_ids else None
+        allowed_library_modify_ids = prefetched_ids.get('allowed_library_modify_ids', None) if prefetched_ids else None
+        allowed_library_manage_ids = prefetched_ids.get('allowed_library_manage_ids', None) if prefetched_ids else None
         library_dict = library.to_dict(view='element', value_mapper={'id': trans.security.encode_id, 'root_folder_id': trans.security.encode_id})
-        if restricted_library_ids and library.id in restricted_library_ids:
-            library_dict['public'] = False
-        else:
-            library_dict['public'] = True
+        library_dict['public'] = False if (restricted_library_ids and library.id in restricted_library_ids) else True
         library_dict['create_time_pretty'] = pretty_print_time_interval(library.create_time, precise=True)
         if not trans.user_is_admin():
-            current_user_roles = trans.get_current_user_roles()
-            library_dict['can_user_add'] = trans.app.security_agent.can_add_library_item(current_user_roles, library)
-            library_dict['can_user_modify'] = trans.app.security_agent.can_modify_library_item(current_user_roles, library)
-            library_dict['can_user_manage'] = trans.app.security_agent.can_manage_library_item(current_user_roles, library)
+            if prefetched_ids:
+                library_dict['can_user_add'] = True if (allowed_library_add_ids and library.id in allowed_library_add_ids) else False
+                library_dict['can_user_modify'] = True if (allowed_library_modify_ids and library.id in allowed_library_modify_ids) else False
+                library_dict['can_user_manage'] = True if (allowed_library_manage_ids and library.id in allowed_library_manage_ids) else False
+            else:
+                current_user_roles = trans.get_current_user_roles()
+                library_dict['can_user_add'] = trans.app.security_agent.can_add_library_item(current_user_roles, library)
+                library_dict['can_user_modify'] = trans.app.security_agent.can_modify_library_item(current_user_roles, library)
+                library_dict['can_user_manage'] = trans.app.security_agent.can_manage_library_item(current_user_roles, library)
         else:
             library_dict['can_user_add'] = True
             library_dict['can_user_modify'] = True

lib/galaxy/model/mapping.py

@@ -1574,6 +1574,10 @@ def simple_mapping(model, **kwds):
     average_rating=column_property(
         select([func.avg(model.HistoryRatingAssociation.table.c.rating)]).where(model.HistoryRatingAssociation.table.c.history_id == model.History.table.c.id),
         deferred=True
+    ),
+    users_shared_with_count=column_property(
+        select([func.count(model.HistoryUserShareAssociation.table.c.id)]).where(model.History.table.c.id == model.HistoryUserShareAssociation.table.c.history_id),
+        deferred=True
     )
 ))
 

lib/galaxy/tools/toolbox/filters/__init__.py

@@ -44,7 +44,7 @@ def build_filters(self, trans, **kwds):
                     elif name == 'toolbox_label_filters':
                         category = "label"
                     if category:
-                        validate = getattr(trans.app.config, 'user_%s_filters' % category, [])
+                        validate = getattr(trans.app.config, 'user_tool_%s_filters' % category, [])
                         self.__init_filters(category, user_filters, filters, validate=validate)
         else:
             if kwds.get("trackster", False):

lib/galaxy/web/framework/helpers/grids.py

@@ -822,22 +822,34 @@ def get_accepted_filters(self):
 class SharingStatusColumn(GridColumn):
     """ Grid column to indicate sharing status. """
 
+    def __init__(self, *args, **kwargs):
+        self.use_shared_with_count = kwargs.pop("use_shared_with_count", False)
+        super(SharingStatusColumn, self).__init__(*args, **kwargs)
+
     def get_value(self, trans, grid, item):
         # Delete items cannot be shared.
         if item.deleted:
             return ""
         # Build a list of sharing for this item.
         sharing_statuses = []
-        if item.users_shared_with:
+        if self._is_shared(item):
             sharing_statuses.append("Shared")
         if item.importable:
             sharing_statuses.append("Accessible")
         if item.published:
             sharing_statuses.append("Published")
         return ", ".join(sharing_statuses)
 
+    def _is_shared(self, item):
+        if self.use_shared_with_count:
+            # optimization to skip join for users_shared_with and loading in that data.
+            return item.users_shared_with_count > 0
+
+        return item.users_shared_with
+
     def get_link(self, trans, grid, item):
-        if not item.deleted and (item.users_shared_with or item.importable or item.published):
+        is_shared = self._is_shared(item)
+        if not item.deleted and (is_shared or item.importable or item.published):
             return dict(operation="share or publish", id=item.id)
         return None
 

lib/galaxy/webapps/galaxy/api/libraries.py

@@ -46,10 +46,10 @@ def index(self, trans, **kwd):
 
         """
         deleted = util.string_as_bool_or_none(kwd.get('deleted', None))
-        query, restricted_library_ids = self.library_manager.list(trans, deleted)
+        query, prefetched_ids = self.library_manager.list(trans, deleted)
         libraries = []
         for library in query:
-            libraries.append(self.library_manager.get_library_dict(trans, library, restricted_library_ids))
+            libraries.append(self.library_manager.get_library_dict(trans, library, prefetched_ids))
         return libraries
 
     def __decode_id(self, trans, encoded_id, object_name=None):

lib/galaxy/webapps/galaxy/api/users.py

@@ -708,11 +708,22 @@ def _add_filter_inputs(self, factory, filter_types, inputs, errors, filter_type,
             function = factory.build_filter_function(filter_name)
             if function is None:
                 errors['%s|%s' % (filter_type, filter_name)] = 'Filter function not found.'
+
+            short_description, description = None, None
+            doc_string = docstring_trim(function.__doc__)
+            split = doc_string.split('\n\n')
+            if split:
+                short_description = split[0]
+                if len(split) > 1:
+                    description = split[1]
+            else:
+                log.warning('No description specified in the __doc__ string for %s.' % filter_name)
+
             filter_inputs.append({
                 'type': 'boolean',
                 'name': filter_name,
-                'label': filter_name,
-                'help': docstring_trim(function.__doc__) or 'No description available.',
+                'label': short_description or filter_name,
+                'help': description or 'No description available.',
                 'value': 'true' if filter_name in filter_values else 'false'
             })
         if filter_inputs: