Merge pull request #6652 from davebx/ts_api_improvements

Ts api improvements

client/galaxy/scripts/admin.toolshed.js

@@ -30,7 +30,7 @@ var AdminToolshedRouter = Backbone.Router.extend({
         "status/r/:repositories": "status",
         status: "status",
         "categories/s/:tool_shed": "categories",
-        "category/s/:tool_shed/c/:cateory_id": "repositories",
+        "category/s/:tool_shed/c/:category_id/k/:sort_key/p/:page/t/:sort_order": "repositories",
         "repository/s/:tool_shed/r/:repository_id": "repository"
     },
 
@@ -71,10 +71,13 @@ var GalaxyAdminToolshedApp = Backbone.View.extend({
                 tool_shed: tool_shed.replace(/\//g, "%2f")
             });
         });
-        this.admin_toolshed_router.on("route:repositories", (tool_shed, category_id) => {
+        this.admin_toolshed_router.on("route:repositories", (tool_shed, category_id, sort_key='name', page=1, sort_order='asc') => {
             Galaxy.admintoolshedapp.adminShedCategoryView = new mod_repositories_view.Category({
                 tool_shed: tool_shed.replace(/\//g, "%2f"),
-                category_id: category_id
+                category_id: category_id,
+                page: page,
+                sort_order: sort_order,
+                sort_key: sort_key
             });
         });
         this.admin_toolshed_router.on("route:repository", (tool_shed, repository_id) => {

client/galaxy/scripts/mvc/toolshed/categories-view.js

@@ -87,7 +87,7 @@ var ToolShedCategories = Backbone.View.extend({
             "<% _.each(categories, function(category) { %>",
             "<tr>",
             "<td>",
-            '<a href="#/category/s/<%= tool_shed %>/c/<%= category.get("id") %>"><%= category.get("name") %></a>',
+            '<a href="#/category/s/<%= tool_shed %>/c/<%= category.get("id") %>/k/name/p/1/t/asc"><%= category.get("name") %></a>',
             "</td>",
             '<td><%= category.get("description") %></td>',
             '<td><%= category.get("repositories") %></td>',

client/galaxy/scripts/mvc/toolshed/repositories-view.js

@@ -10,23 +10,51 @@ var ToolShedCategoryContentsView = Backbone.View.extend({
     el: "#center",
 
     initialize: function(params) {
+        this.params = params;
         this.model = new toolshed_model.CategoryCollection();
         this.listenTo(this.model, "sync", this.render);
         var shed = params.tool_shed.replace(/\//g, "%2f");
-        this.model.url += `?tool_shed_url=${shed}&category_id=${params.category_id}`;
+        this.model.url += `?tool_shed_url=${shed}&category_id=${params.category_id}&sort_key=${params.sort_key}&sort_order=${params.sort_order}&page=${params.page}`;
         this.model.tool_shed = shed;
         this.model.category = params.category_id;
         this.model.fetch();
     },
 
     render: function(options) {
-        this.options = _.extend(this.options, options);
+        this.options = _.defaults(this.options || {}, options);
         var category_contents_template = this.templateCategoryContents;
+        var page_navigation_template = this.templatePageNavigation;
+        var sorting = {class: {owner: 'fa-sort', description: 'fa-sort', name: 'fa-sort'},
+                       direction: {owner: 'asc', description: 'asc', name: 'asc'}};
+        sorting.class[this.params.sort_key] = (this.params.sort_order == 'desc' ? 'fa-sort-down' : 'fa-sort-up');
+        sorting.direction[this.params.sort_key] = (this.params.sort_order == 'asc' ? 'desc' : 'asc');
+        var previous_page = Math.max(this.params.page - 1, 1);
+        var pages = parseInt((this.model.models[0].get('repository_count')) % 25) + 1;
+        var next_page = Math.min(parseInt(this.params.page) + 1, pages);
+        var pagination_params = {
+            page: this.params.page,
+            next: next_page,
+            previous: previous_page,
+            pages: pages};
+        if (pages > 5) {
+            var page = parseInt(this.params.page);
+            var page_slice = Array();
+            var slice_start = Math.max(page - 2, 2);
+            var i = slice_start;
+            var slice_end = Math.min(Math.min(page + 2, slice_start + 4), pages);
+            for (i = slice_start; i <= slice_end; i++) {
+                page_slice.push(i);
+            }
+            pagination_params['page_slice'] = page_slice;
+        }
         this.$el.html(
             category_contents_template({
+                page_navigation: page_navigation_template(pagination_params),
                 category: this.model.models[0],
                 tool_shed: this.model.tool_shed,
-                queue: toolshed_util.queueLength()
+                queue: toolshed_util.queueLength(),
+                sorting: sorting,
+                page: this.params.page
             })
         );
         $("#center").css("overflow", "auto");
@@ -57,13 +85,85 @@ var ToolShedCategoryContentsView = Backbone.View.extend({
                 });
             }
         });
+        $('.fa-fw').on("click", (ev) => {
+            var parameters = {};
+            parameters.field = $(ev.target).attr('data-field');
+            parameters.direction = $(ev.target).attr('data-direction');
+            var new_route = `category/s/${this.model.tool_shed}/c/${this.model.category}/k/${parameters.field}/p/${this.params.page}/t/${parameters.direction}`
+            Backbone.history.navigate(new_route, {
+                trigger: true,
+                replace: true
+            });
+        });
+        $('.pagenav').on("click", (ev) => {
+            var page = $(ev.target).attr('data-page');
+            var new_route = `category/s/${this.model.tool_shed}/c/${this.model.category}/k/${this.params.sort_key}/p/${page}/t/${this.params.sort_order}`;
+            Backbone.history.navigate(new_route, {
+                trigger: true,
+                replace: true
+            });
+        });
     },
 
+    templatePageNavigation: _.template(
+        [
+            '<div class="navigation">',
+            "<% if (page != 1) { %>",
+            '<a data-page="1" class="pagenav fa fa-fast-backward" />',
+            '<a data-page="<%= previous %>" class="pagenav fa fa-step-backward" />',
+            "<% } else { %>",
+            '<a data-page="1" class="pagenav-inactive fa fa-fast-backward" />',
+            '<a data-page="<%= previous %>" class="pagenav-inactive fa fa-step-backward" />',
+            '<% } %>',
+            '<% if (pages > 5) { %>',
+            '<% if (page != 1) { %>',
+            '<a data-page="1" class="pagenav fa"><a>1</a>',
+            '<% if (page != 2) { %>',
+            '<a class="fa">&hellip;</a>',
+            '<% } %>',
+            '<% } else { %>',
+            '<a data-page="1" class="pagenav-inactive fa"><a>1</a>',
+            '<% } %>',
+            '<% _.each(page_slice, function(i) { %>',
+            '<% if (i == page) { %>',
+            '<a data-page="<%= i %>" class="fa"><strong><%= i %></strong></a>',
+            '<% } else { %>',
+            '<a data-page="<%= i %>" class="pagenav fa"><%= i %></a>',
+            '<% } %>',
+            '<% }); %>',
+            '<% var last_pages = [pages - 2, pages - 1, pages]; %>',
+            '<% if (last_pages.indexOf(parseInt(page)) == -1) { %>',
+            '<a class="fa">&hellip;</a>',
+            '<a data-page="<%= pages %>" class="pagenav fa"><%= pages %></a>',
+            '<% } %>',
+            '<% } else { %>',
+            '<% for (i = 1; i <= pages; i++) { %>',
+            '<% if (i == page) { %>',
+            '<a data-page="<%= i %>" class="fa"><strong><%= i %></strong></a>',
+            '<% } else { %>',
+            '<a data-page="<%= i %>" class="pagenav fa"><%= i %></a>',
+            '<% } %>',
+            '<% } %>',
+            '<% } %>',
+            '<% if (page < pages) { %>',
+            '<a data-page="<%= next %>" class="pagenav fa fa-step-forward" />',
+            '<a data-page="<%= pages %>" class="pagenav fa fa-fast-forward" />',
+            "<% } else { %>",
+            '<a data-page="<%= next %>" class="pagenav-inactive fa fa-step-forward" />',
+            '<a data-page="<%= pages %>" class="pagenav-inactive fa fa-fast-forward" />',
+            '<% } %>',
+            '</div>'
+        ].join("")
+    ),
+
     templateCategoryContents: _.template(
         [
             '<style type="text/css">',
             ".ui-autocomplete { background-color: #fff; }",
             "li.ui-menu-item { list-style-type: none; }",
+            "div.navigation { width: 100%; text-align: center; }",
+            "div.navigation a { margin-left: 0.5em; margin-right: 0.5em; display: inline; text-decoration: underline; }",
+            "a.pagenav-inactive { opacity: 0.5; }",
             "</style>",
             '<div class="unified-panel-header" id="panel_header" unselectable="on">',
             '<div class="unified-panel-header-inner">Repositories in <%= category.get("name") %><a class="ml-auto" href="#/queue">Repository Queue (<%= queue %>)</a></div>',
@@ -74,13 +174,16 @@ var ToolShedCategoryContentsView = Backbone.View.extend({
             '<input class="search-box-input" id="search_box" name="search" data-shedurl="<%= tool_shed.replace(/%2f/g, "/") %>" placeholder="Search repositories by name or id" size="60" type="text" />',
             "</span>",
             "</div>",
+            "<% if (category.get('repository_count') > 25) { %>",
+            '<%= page_navigation %>',
+            "<% } %>",
             '<div style="clear: both; margin-top: 1em;">',
-            '<table class="grid">',
+            '<table class="grid table-striped">',
             '<thead id="grid-table-header">',
             "<tr>",
-            '<th style="width: 10%;">Owner</th>',
-            '<th style="width: 15%;">Name</th>',
-            "<th>Synopsis</th>",
+            '<th style="width: 10%;"><a class="fa fa-fw <%= sorting.class.owner %>" data-direction="<%= sorting.direction.owner %>" data-field="owner">Owner</a></th>',
+            '<th style="width: 15%;"><a class="fa fa-fw <%= sorting.class.name %>" data-direction="<%= sorting.direction.name %>" data-field="name">Name</a></th>',
+            '<th><a class="fa fa-fw <%= sorting.class.description %>" data-direction="<%= sorting.direction.description %>" data-field="description">Synopsis</a></th>',
             '<th style="width: 10%;">Type</th>',
             "</tr>",
             "</thead>",

lib/galaxy/webapps/galaxy/api/toolshed.py

@@ -187,10 +187,16 @@ def category(self, trans, **kwd):
 
         :param tool_shed_url: the url of the toolshed to get repositories from
         :param category_id: the category to get repositories from
+        :param sort_key: the field by which the repositories should be sorted
+        :param sort_order: ascending or descending sort
+        :param page: the page number to return
         """
+        sort_order = kwd.get('sort_order', 'asc')
+        sort_key = kwd.get('sort_key', 'name')
+        page = kwd.get('page', 1)
         tool_shed_url = urlunquote(kwd.get('tool_shed_url', ''))
         category_id = kwd.get('category_id', '')
-        params = dict(installable=True)
+        params = dict(installable=True, sort_order=sort_order, sort_key=sort_key, page=page)
         tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(trans.app, tool_shed_url)
         url = util.build_url(tool_shed_url, pathspec=['api', 'categories', category_id, 'repositories'], params=params)
         repositories = []

lib/galaxy/webapps/tool_shed/api/categories.py

@@ -73,21 +73,33 @@ def get_repositories(self, trans, category_id, **kwd):
         Return information about the provided category and the repositories in that category.
 
         :param id: the encoded id of the Category object
+        :param sort_key: the field by which the repositories should be sorted
+        :param sort_order: ascending or descending sort
+        :param page: the page number to return
 
         Example: GET localhost:9009/api/categories/f9cad7b01a472135/repositories
         """
         installable = util.asbool(kwd.get('installable', 'false'))
+        sort_key = kwd.get('sort_key', 'name')
+        sort_order = kwd.get('sort_order', 'asc')
+        page = kwd.get('page', None)
         category = suc.get_category(self.app, category_id)
         if category is None:
             category_dict = dict(message='Unable to locate category record for id %s.' % (str(id)),
                                  status='error')
             return category_dict
         category_dict = category.to_dict(view='element',
                                          value_mapper=self.__get_value_mapper(trans))
+        category_dict['repository_count'] = suc.count_repositories_in_category(self.app, category_id)
         category_dict['url'] = web.url_for(controller='categories',
                                            action='show',
                                            id=trans.security.encode_id(category.id))
-        repositories = repository_util.get_repositories_by_category(self.app, category.id, installable=installable)
+        repositories = repository_util.get_repositories_by_category(self.app,
+                                                                    category.id,
+                                                                    installable=installable,
+                                                                    sort_order=sort_order,
+                                                                    sort_key=sort_key,
+                                                                    page=page)
         category_dict['repositories'] = repositories
         return category_dict
 

lib/galaxy/webapps/tool_shed/model/__init__.py

@@ -304,19 +304,20 @@ def to_dict(self, view='collection', value_mapper=None):
 
 
 class RepositoryMetadata(Dictifiable):
-    dict_collection_visible_keys = ['id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'missing_test_components',
+    dict_collection_visible_keys = ['id', 'repository_id', 'numeric_revision', 'changeset_revision', 'malicious', 'downloadable', 'missing_test_components',
                                     'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies',
                                     'includes_tools_for_display_in_tool_panel', 'includes_workflows']
-    dict_element_visible_keys = ['id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'missing_test_components',
+    dict_element_visible_keys = ['id', 'repository_id', 'numeric_revision', 'changeset_revision', 'malicious', 'downloadable', 'missing_test_components',
                                  'has_repository_dependencies', 'includes_datatypes', 'includes_tools', 'includes_tool_dependencies',
                                  'includes_tools_for_display_in_tool_panel', 'includes_workflows', 'repository_dependencies']
 
-    def __init__(self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False,
+    def __init__(self, id=None, repository_id=None, numeric_revision=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False,
                  downloadable=False, missing_test_components=None, tools_functionally_correct=False, test_install_error=False,
                  has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False,
                  includes_workflows=False):
         self.id = id
         self.repository_id = repository_id
+        self.numeric_revision = numeric_revision
         self.changeset_revision = changeset_revision
         self.metadata = metadata
         self.tool_versions = tool_versions

lib/galaxy/webapps/tool_shed/model/mapping.py

@@ -129,6 +129,7 @@
                                  Column("update_time", DateTime, default=now, onupdate=now),
                                  Column("repository_id", Integer, ForeignKey("repository.id"), index=True),
                                  Column("changeset_revision", TrimmedString(255), index=True),
+                                 Column("numeric_revision", Integer, index=True),
                                  Column("metadata", JSONType, nullable=True),
                                  Column("tool_versions", JSONType, nullable=True),
                                  Column("malicious", Boolean, default=False),

lib/galaxy/webapps/tool_shed/model/migrate/versions/0026_add_numeric_revision_column.py

@@ -0,0 +1,45 @@
+"""Migration script to add the numeric_revision column to the repository metadata table."""
+from __future__ import print_function
+
+import logging
+import sys
+
+from sqlalchemy import Column, Integer, MetaData, Table
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.DEBUG)
+handler = logging.StreamHandler(sys.stdout)
+format = "%(name)s %(levelname)s %(asctime)s %(message)s"
+formatter = logging.Formatter(format)
+handler.setFormatter(formatter)
+log.addHandler(handler)
+
+metadata = MetaData()
+
+
+def upgrade(migrate_engine):
+    print(__doc__)
+    metadata.bind = migrate_engine
+    metadata.reflect()
+    RepositoryMetadata_table = Table("repository_metadata", metadata, autoload=True)
+    c = Column("numeric_revision", Integer, index=True)
+    try:
+        # Create
+        c.create(RepositoryMetadata_table, index_name="ix_numeric_revision")
+        assert c is RepositoryMetadata_table.c.numeric_revision
+    except Exception:
+        log.exception("Adding numeric_revision column to the repository table failed.")
+    # Update the numeric_revision column to have the default undefined value.
+    cmd = "UPDATE repository_metadata SET numeric_revision = -1"
+    migrate_engine.execute(cmd)
+
+
+def downgrade(migrate_engine):
+    metadata.bind = migrate_engine
+    metadata.reflect()
+    # Drop numeric_revision column from repository table.
+    RepositoryMetadata_table = Table("repository_metadata", metadata, autoload=True)
+    try:
+        RepositoryMetadata_table.c.numeric_revision.drop()
+    except Exception:
+        log.exception("Dropping column numeric_revision from the repository table failed.")

lib/tool_shed/util/metadata_util.py

@@ -113,17 +113,24 @@ def get_metadata_revisions(app, repository, sort_revisions=True, reverse=False,
     """
     Return a list of changesets for the provided repository.
     """
+    sa_session = app.model.context.current
     if downloadable:
         metadata_revisions = repository.downloadable_revisions
     else:
         metadata_revisions = repository.metadata_revisions
     repo_path = repository.repo_path(app)
     changeset_tups = []
     for repository_metadata in metadata_revisions:
-        try:
-            rev = hg_util.changeset2rev(repo_path, repository_metadata.changeset_revision)
-        except Exception:
-            rev = -1
+        if repository_metadata.numeric_revision == -1 or repository_metadata.numeric_revision is None:
+            try:
+                rev = hg_util.changeset2rev(repo_path, repository_metadata.changeset_revision)
+                repository_metadata.numeric_revision = rev
+                sa_session.add(repository_metadata)
+                sa_session.flush()
+            except Exception:
+                rev = -1
+        else:
+            rev = repository_metadata.numeric_revision
         changeset_tups.append((rev, repository_metadata.changeset_revision))
     if sort_revisions:
         changeset_tups.sort(key=itemgetter(0), reverse=reverse)
@@ -218,7 +225,6 @@ def get_repository_metadata_by_changeset_revision(app, id, changeset_revision):
     all_metadata_records = sa_session.query(app.model.RepositoryMetadata) \
                                      .filter(and_(app.model.RepositoryMetadata.table.c.repository_id == app.security.decode_id(id),
                                                   app.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision)) \
-                                     .order_by(app.model.RepositoryMetadata.table.c.update_time.desc()) \
                                      .all()
     if len(all_metadata_records) > 1:
         # Delete all records older than the last one updated.