Merge branch 'release_17.01' into dev

Conflicts:
	lib/galaxy/tools/deps/container_resolvers/__init__.py
	static/scripts/bundled/libs.bundled.js
	static/scripts/bundled/libs.bundled.js.map

config/datatypes_conf.xml.sample

@@ -482,7 +482,7 @@
     <datatype extension="stockholm" type="galaxy.datatypes.msa:Stockholm_1_0" display_in_upload="true" />
     <datatype extension="xmfa" type="galaxy.datatypes.msa:MauveXmfa" display_in_upload="true" />
     <datatype extension="cel" type="galaxy.datatypes.binary:Cel" display_in_upload="true" />
-    <datatype extension="RData" type="galaxy.datatypes.binary:RData" display_in_upload="true" description="Stored data from an R session"/>
+    <datatype extension="rdata" type="galaxy.datatypes.binary:RData" display_in_upload="true" description="Stored data from an R session"/>
     <datatype extension="oxlicg" type="galaxy.datatypes.binary:OxliCountGraph" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="oxling" type="galaxy.datatypes.binary:OxliNodeGraph" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="oxlits" type="galaxy.datatypes.binary:OxliTagSet" mimetype="application/octet-stream" display_in_upload="true"/>

lib/galaxy/datatypes/binary.py

@@ -1148,7 +1148,7 @@ def display_peek(self, dataset):
 
 class RData( Binary ):
     """Generic R Data file datatype implementation"""
-    file_ext = 'RData'
+    file_ext = 'rdata'
 
     def sniff( self, filename ):
         rdata_header = b'RDX2\nX\n'

lib/galaxy/tools/__init__.py

@@ -125,6 +125,12 @@
     "maf_limit_size1",
     "maf_by_block_number1",
     "wiggle2simple1",
+    # Converters
+    "CONVERTER_fastq_to_fqtoc0",
+    "CONVERTER_gff_to_interval_index_0",
+    "CONVERTER_maf_to_fasta_0",
+    "CONVERTER_maf_to_interval_0",
+    "CONVERTER_wiggle_to_interval_0",
     # Tools improperly migrated to the tool shed (devteam)
     "lastz_wrapper_2",
     "qualityFilter",

lib/galaxy/tools/deps/container_resolvers/__init__.py

@@ -11,6 +11,7 @@
 
 
 @six.add_metaclass(ABCMeta)
+@six.python_2_unicode_compatible
 class ContainerResolver(Dictifiable, object):
     """Description of a technique for resolving container images for tool execution."""
 
@@ -49,3 +50,6 @@ def resolver_type(self):
     def _container_type_enabled(self, container_description, enabled_container_types):
         """Return a boolean indicating if the specified container type is enabled."""
         return container_description.type in enabled_container_types
+
+    def __str__(self):
+        return "%s[]" % self.__class__.__name__

lib/galaxy/tools/deps/container_resolvers/mulled.py

@@ -3,6 +3,8 @@
 import collections
 import logging
 
+import six
+
 from ..container_resolvers import (
     ContainerResolver,
 )
@@ -102,6 +104,7 @@ def cached_container_description(targets, namespace):
     return container
 
 
+@six.python_2_unicode_compatible
 class CachedMulledContainerResolver(ContainerResolver):
 
     resolver_type = "cached_mulled"
@@ -114,7 +117,11 @@ def resolve(self, enabled_container_types, tool_info):
         targets = mulled_targets(tool_info)
         return cached_container_description(targets, self.namespace)
 
+    def __str__(self):
+        return "CachedMulledContainerResolver[namespace=%s]" % self.namespace
+
 
+@six.python_2_unicode_compatible
 class MulledContainerResolver(ContainerResolver):
     """Look for mulled images matching tool dependencies."""
 
@@ -160,7 +167,11 @@ def resolve(self, enabled_container_types, tool_info):
                 type="docker",
             )
 
+    def __str__(self):
+        return "MulledContainerResolver[namespace=%s]" % self.namespace
 
+
+@six.python_2_unicode_compatible
 class BuildMulledContainerResolver(ContainerResolver):
     """Look for mulled images matching tool dependencies."""
 
@@ -195,6 +206,9 @@ def _get_involucro_context(self):
         self.enabled = ensure_installed(involucro_context, self.auto_init)
         return involucro_context
 
+    def __str__(self):
+        return "BuildContainerResolver[namespace=%s]" % self.namespace
+
 
 def mulled_targets(tool_info):
     return requirements_to_mulled_targets(tool_info.requirements)

lib/galaxy/tools/deps/requirements.py

@@ -151,6 +151,7 @@ class ToolRequirementsException(Exception):
 DEFAULT_CONTAINER_SHELL = "/bin/sh"  # Galaxy assumes bash, but containers are usually thinner.
 
 
+@six.python_2_unicode_compatible
 class ContainerDescription( object ):
 
     def __init__(
@@ -186,6 +187,9 @@ def from_dict( dict ):
             shell=shell,
         )
 
+    def __str__(self):
+        return "ContainerDescription[identifier=%s,type=%s]" % (self.identifier, self.type)
+
 
 def parse_requirements_from_dict( root_dict ):
     requirements = root_dict.get("requirements", [])

lib/galaxy/web/framework/helpers/grids.py

@@ -565,7 +565,7 @@ def get_value( self, trans, grid, item ):
                 ann_snippet += "..."
         else:
             ann_snippet = ""
-        return ann_snippet
+        return escape( ann_snippet )
 
     def get_single_filter( self, user, a_filter ):
         """ Filter by annotation and annotation owner. """

lib/galaxy/webapps/galaxy/api/users.py

@@ -3,9 +3,12 @@
 """
 
 import logging
-import datetime
+import random
 import re
+import socket
 
+from datetime import datetime
+from markupsafe import escape
 from sqlalchemy import false, true, and_, or_
 
 from galaxy import exceptions, util, web
@@ -14,6 +17,7 @@
 from galaxy.security.validate_user_input import validate_email
 from galaxy.security.validate_user_input import validate_password
 from galaxy.security.validate_user_input import validate_publicname
+from galaxy.web import url_for
 from galaxy.web import _future_expose_api as expose_api
 from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous
 from galaxy.web.base.controller import BaseAPIController
@@ -24,7 +28,7 @@
 from galaxy.web.base.controller import UsesFormDefinitionsMixin
 from galaxy.web.form_builder import AddressField
 from galaxy.tools.toolbox.filters import FilterFactory
-from galaxy.util import docstring_trim, listify
+from galaxy.util import docstring_trim, listify, hash_util
 from galaxy.util.odict import odict
 
 
@@ -231,9 +235,7 @@ def undelete( self, trans, **kwd ):
 
     # TODO: move to more basal, common resource than this
     def anon_user_api_value( self, trans ):
-        """
-        Returns data for an anonymous user, truncated to only usage and quota_percent
-        """
+        """Return data for an anonymous user, truncated to only usage and quota_percent"""
         usage = trans.app.quota_agent.get_usage( trans )
         percent = trans.app.quota_agent.get_percent( trans=trans, usage=usage )
         return {'total_disk_usage': int( usage ),
@@ -242,9 +244,7 @@ def anon_user_api_value( self, trans ):
 
     @expose_api
     def get_information(self, trans, id, **kwd):
-        '''
-        Returns user details such as public username, type, addresses, etc.
-        '''
+        """Return user details such as username, email, addresses etc."""
         user = self._get_user(trans, id)
         email = user.email
         username = user.username
@@ -315,9 +315,7 @@ def get_information(self, trans, id, **kwd):
 
     @expose_api
     def set_information(self, trans, id, payload={}, **kwd):
-        '''
-        Save a user's email address, public username, type, addresses etc.
-        '''
+        """Save a user's email, username, addresses etc."""
         user = self._get_user(trans, id)
         email = payload.get('email')
         username = payload.get('username')
@@ -327,14 +325,17 @@ def set_information(self, trans, id, payload={}, **kwd):
                 message = validate_publicname(trans, username, user)
             if message:
                 raise MessageException(message)
-            # Update user email and user's private role name which must match
             if user.email != email:
+                # Update user email and user's private role name which must match
                 private_role = trans.app.security_agent.get_private_user_role(user)
                 private_role.name = email
                 private_role.description = 'Private role for ' + email
                 user.email = email
+                trans.sa_session.add(user)
                 trans.sa_session.add(private_role)
+                trans.sa_session.flush()
                 if trans.app.config.user_activation_on:
+                    # Deactivate the user if email was changed and activation is on.
                     user.active = False
                     if self.send_verification_email(trans, user.email, user.username):
                         message = 'The login information has been updated with the changes.<br>Verification email has been sent to your new email address. Please verify it by clicking the activation link in the email.<br>Please check your spam/trash folder in case you cannot find the message.'
@@ -343,8 +344,8 @@ def set_information(self, trans, id, payload={}, **kwd):
                         if trans.app.config.error_email_to is not None:
                             message += ' Contact: %s' % trans.app.config.error_email_to
                         raise MessageException(message)
-            # Update public name
             if user.username != username:
+                # Update public name
                 user.username = username
         # Update user custom form
         user_info_form_id = payload.get('info|form_id')
@@ -393,6 +394,60 @@ def set_information(self, trans, id, payload={}, **kwd):
         trans.log_event('User information added')
         return {'message': 'User information has been saved.'}
 
+    def send_verification_email( self, trans, email, username ):
+        """
+        Send the verification email containing the activation link to the user's email.
+        """
+        if username is None:
+            username = trans.user.username
+        activation_link = self.prepare_activation_link( trans, escape( email ) )
+
+        host = trans.request.host.split( ':' )[ 0 ]
+        if host in [ 'localhost', '127.0.0.1', '0.0.0.0' ]:
+            host = socket.getfqdn()
+        body = ("Hello %s,\n\n"
+                "In order to complete the activation process for %s begun on %s at %s, please click on the following link to verify your account:\n\n"
+                "%s \n\n"
+                "By clicking on the above link and opening a Galaxy account you are also confirming that you have read and agreed to Galaxy's Terms and Conditions for use of this service (%s). This includes a quota limit of one account per user. Attempts to subvert this limit by creating multiple accounts or through any other method may result in termination of all associated accounts and data.\n\n"
+                "Please contact us if you need help with your account at: %s. You can also browse resources available at: %s. \n\n"
+                "More about the Galaxy Project can be found at galaxyproject.org\n\n"
+                "Your Galaxy Team" % (escape( username ), escape( email ),
+                                      datetime.utcnow().strftime( "%D"),
+                                      trans.request.host, activation_link,
+                                      trans.app.config.terms_url,
+                                      trans.app.config.error_email_to,
+                                      trans.app.config.instance_resource_url))
+        to = email
+        frm = trans.app.config.email_from or 'galaxy-no-reply@' + host
+        subject = 'Galaxy Account Activation'
+        try:
+            util.send_mail( frm, to, subject, body, trans.app.config )
+            return True
+        except Exception:
+            log.exception( 'Unable to send the activation email.' )
+            return False
+
+    def prepare_activation_link( self, trans, email ):
+        """
+        Prepare the account activation link for the user.
+        """
+        activation_token = self.get_activation_token( trans, email )
+        activation_link = url_for( controller='user', action='activate', activation_token=activation_token, email=email, qualified=True  )
+        return activation_link
+
+    def get_activation_token( self, trans, email ):
+        """
+        Check for the activation token. Create new activation token and store it in the database if no token found.
+        """
+        user = trans.sa_session.query( trans.app.model.User ).filter( trans.app.model.User.table.c.email == email ).first()
+        activation_token = user.activation_token
+        if activation_token is None:
+            activation_token = hash_util.new_secure_hash( str( random.getrandbits( 256 ) ) )
+            user.activation_token = activation_token
+            trans.sa_session.add( user )
+            trans.sa_session.flush()
+        return activation_token
+
     def _validate_email_publicname(self, email, username):
         ''' Validate email and username using regex '''
         if email == '' or not isinstance( email, basestring ):

lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py

@@ -1049,8 +1049,7 @@ def prepare_for_install( self, trans, **kwd ):
         repo_info_dicts = [ encoding_util.tool_shed_decode( encoded_repo_info_dict ) for encoded_repo_info_dict in encoded_repo_info_dicts ]
         dd = dependency_display.DependencyDisplayer( trans.app )
         install_repository_manager = install_manager.InstallRepositoryManager( trans.app )
-        if ( ( not includes_tools_for_display_in_tool_panel and kwd.get( 'select_shed_tool_panel_config_button', False ) ) or
-             ( includes_tools_for_display_in_tool_panel and kwd.get( 'select_tool_panel_section_button', False ) ) ):
+        if kwd.get( 'select_tool_panel_section_button', False ):
             if updating:
                 repository = repository_util.get_tool_shed_repository_by_id( trans.app, updating_repository_id )
                 decoded_updated_metadata = encoding_util.tool_shed_decode( encoded_updated_metadata )
@@ -1214,57 +1213,29 @@ def prepare_for_install( self, trans, **kwd ):
             install_resolver_dependencies_check_box = None
         encoded_repo_info_dicts = encoding_util.encoding_sep.join( encoded_repo_info_dicts )
         tool_shed_url = kwd[ 'tool_shed_url' ]
-        if includes_tools_for_display_in_tool_panel:
-            return trans.fill_template( '/admin/tool_shed_repository/select_tool_panel_section.mako',
-                                        encoded_repo_info_dicts=encoded_repo_info_dicts,
-                                        updating=updating,
-                                        updating_repository_id=updating_repository_id,
-                                        updating_to_ctx_rev=updating_to_ctx_rev,
-                                        updating_to_changeset_revision=updating_to_changeset_revision,
-                                        encoded_updated_metadata=encoded_updated_metadata,
-                                        includes_tools=includes_tools,
-                                        includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
-                                        includes_tool_dependencies=includes_tool_dependencies,
-                                        install_tool_dependencies_check_box=install_tool_dependencies_check_box,
-                                        install_resolver_dependencies_check_box=install_resolver_dependencies_check_box,
-                                        has_repository_dependencies=has_repository_dependencies,
-                                        install_repository_dependencies_check_box=install_repository_dependencies_check_box,
-                                        new_tool_panel_section_label=new_tool_panel_section_label,
-                                        containers_dict=containers_dict,
-                                        shed_tool_conf=shed_tool_conf,
-                                        shed_tool_conf_select_field=shed_tool_conf_select_field,
-                                        tool_panel_section_select_field=tool_panel_section_select_field,
-                                        tool_shed_url=tool_shed_url,
-                                        requirements_status=requirements_status,
-                                        message=message,
-                                        status=status )
-        else:
-            # If installing repositories that includes no tools and has no repository dependencies, display a page
-            # allowing the Galaxy administrator to select a shed-related tool panel configuration file whose tool_path
-            # setting will be the location the repositories will be installed.
-            return trans.fill_template( '/admin/tool_shed_repository/select_shed_tool_panel_config.mako',
-                                        encoded_repo_info_dicts=encoded_repo_info_dicts,
-                                        updating=updating,
-                                        updating_repository_id=updating_repository_id,
-                                        updating_to_ctx_rev=updating_to_ctx_rev,
-                                        updating_to_changeset_revision=updating_to_changeset_revision,
-                                        encoded_updated_metadata=encoded_updated_metadata,
-                                        includes_tools=includes_tools,
-                                        includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
-                                        includes_tool_dependencies=includes_tool_dependencies,
-                                        install_tool_dependencies_check_box=install_tool_dependencies_check_box,
-                                        install_resolver_dependencies_check_box=install_resolver_dependencies_check_box,
-                                        has_repository_dependencies=has_repository_dependencies,
-                                        install_repository_dependencies_check_box=install_repository_dependencies_check_box,
-                                        new_tool_panel_section_label=new_tool_panel_section_label,
-                                        containers_dict=containers_dict,
-                                        shed_tool_conf=shed_tool_conf,
-                                        shed_tool_conf_select_field=shed_tool_conf_select_field,
-                                        tool_panel_section_select_field=tool_panel_section_select_field,
-                                        tool_shed_url=tool_shed_url,
-                                        tool_requirements=tool_requirements,
-                                        message=message,
-                                        status=status )
+        return trans.fill_template( '/admin/tool_shed_repository/select_tool_panel_section.mako',
+                                    encoded_repo_info_dicts=encoded_repo_info_dicts,
+                                    updating=updating,
+                                    updating_repository_id=updating_repository_id,
+                                    updating_to_ctx_rev=updating_to_ctx_rev,
+                                    updating_to_changeset_revision=updating_to_changeset_revision,
+                                    encoded_updated_metadata=encoded_updated_metadata,
+                                    includes_tools=includes_tools,
+                                    includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
+                                    includes_tool_dependencies=includes_tool_dependencies,
+                                    install_tool_dependencies_check_box=install_tool_dependencies_check_box,
+                                    install_resolver_dependencies_check_box=install_resolver_dependencies_check_box,
+                                    has_repository_dependencies=has_repository_dependencies,
+                                    install_repository_dependencies_check_box=install_repository_dependencies_check_box,
+                                    new_tool_panel_section_label=new_tool_panel_section_label,
+                                    containers_dict=containers_dict,
+                                    shed_tool_conf=shed_tool_conf,
+                                    shed_tool_conf_select_field=shed_tool_conf_select_field,
+                                    tool_panel_section_select_field=tool_panel_section_select_field,
+                                    tool_shed_url=tool_shed_url,
+                                    requirements_status=requirements_status,
+                                    message=message,
+                                    status=status )
 
     @web.expose
     @web.require_admin