Merge with dev

lib/galaxy/auth/providers/pam_auth.py

@@ -0,0 +1,151 @@
+"""
+Created on 13/07/2015
+
+Author Peter van Heusden (pvh@sanbi.ac.za)
+"""
+
+import logging
+from subprocess import Popen, PIPE
+import shlex
+
+from ..providers import AuthProvider
+from galaxy.auth import _get_bool
+
+log = logging.getLogger(__name__)
+
+"""
+This module provides an AuthProvider for PAM (pluggable authentication module) authentication.
+PAM is the Pluggable Authentication Module system (http://www.linux-pam.org/)
+It relies on python-pam (https://pypi.python.org/pypi/python-pam)
+
+Configuration is via config/auth_conf.xml and the following options are supported:
+  - auto-register: True/False: automatically register an account for an unknown user. Default: False
+  - maildomain: string: all valid users fall within the specified mail domain. Default: None
+  - login-use-email: True/False: Parse the email address to get login details. Default: False
+  - login-use-username: True/False: Use the username argument for login details. Default: False
+                                    Technical note: when a user is not found in the database,
+                                    their username is the user part of a user@host email
+                                    address. After user creation, however, the username is
+                                    the user's public name.
+  - pam-service: string: The service name to use for PAM authentication. Default: galaxy
+  - use-external-helper: True/False: Run an external helper script as root with sudo to do
+                                     authentication. If False authentication is done
+                                     by the module directly. Default: False
+                                     Technical note: some PAM modules (e.g. pam_unix.so)
+                                     require to be run as root to authenticate users.
+  - authentication-helper-script: string: Absolute path to helper script to run for authentication. Default: None
+                                          There needs to be a config (in /etc/sudoers or /etc/sudoers.d)
+                                          that allows the galaxy user to run this as root with no password check
+                                          For example:
+galaxy	ALL=(root) NOPASSWD: /opt/galaxy/scripts/auth/pam_auth_helper.py
+
+
+Configuration example (for internal authentication, use email for user details):
+<authenticator>
+  <type>PAM</type>
+  <options>
+          <auto-register>True</auto-register>
+          <maildomain>example.com</maildomain>
+          <login-use-email>True</login-use-email>
+          <pam-service>ssh</pam-service>
+  </options>
+</authenticator>
+"""
+
+
+class PAM(AuthProvider):
+
+    plugin_type = 'PAM'
+
+    def authenticate(self, email, username, password, options):
+        pam_username = None
+        auto_register_username = None
+        auto_register_email = None
+        force_fail = False
+        log.debug("use username: {} use email {} email {} username {}".format(options.get('login-use-username'), options.get('login-use-email', False), email, username))
+        # check email based login first because if email exists in Galaxy DB
+        # we will be given the "public name" as username
+        if _get_bool(options, 'login-use-email', False) and email is not None:
+            if '@' in email:
+                (email_user, email_domain) = email.split('@')
+                pam_username = email_user
+                if email_domain == options.get('maildomain', None):
+                    auto_register_email = email
+                    if username is not None:
+                        auto_register_username = username
+                    else:
+                        auto_register_username = email_user
+                else:
+                    log.debug('PAM authenticate: warning: email does not match configured PAM maildomain')
+                    # no need to fail: if auto-register is not enabled, this
+                    # might still be a valid user
+            else:
+                log.debug('PAM authenticate: email must be used to login, but no valid email found')
+                force_fail = True
+        elif _get_bool(options, 'login-use-username', False):
+            # if we get here via authenticate_user then
+            # user will be "public name" and
+            # email address will be as per registered user
+            if username is not None:
+                pam_username = username
+                if email is not None:
+                    auto_register_email = email
+                elif options.get('maildomain', None) is not None:
+                    # we can register a user with this username and mail domain
+                    # if auto registration is enabled
+                    auto_register_email = '{}@{}'.format(username, options['maildomain'])
+                auto_register_username = username
+            else:
+                log.debug('PAM authenticate: username login selected but no username provided')
+                force_fail = True
+        else:
+            log.debug('PAM authenticate: could not find username for PAM')
+            force_fail = True
+
+        if force_fail:
+            return None, '', ''
+
+        pam_service = options.get('pam-service', 'galaxy')
+        use_helper = _get_bool(options, 'use-external-helper', False)
+        log.debug("PAM auth: will use external helper: {}".format(use_helper))
+        authenticated = False
+        if use_helper:
+            authentication_helper = options.get('authentication-helper-script', '/bin/false').strip()
+            log.debug("PAM auth: external helper script: {}".format(authentication_helper))
+            if not authentication_helper.startswith('/'):
+                # don't accept relative path
+                authenticated = False
+            else:
+                auth_cmd = shlex.split('/usr/bin/sudo -n {}'.format(authentication_helper))
+                log.debug("PAM auth: external helper cmd: {}".format(auth_cmd))
+                proc = Popen(auth_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+                message = '{}\n{}\n{}\n'.format(pam_service, pam_username, password)
+                (output, error) = proc.communicate(message)
+                status = proc.wait()
+                if status != 0 and error != '':
+                    log.debug("PAM auth: external authentication script had errors: status {} error {}".format(status, error))
+                if output.strip() == 'True':
+                    authenticated = True
+                else:
+                    authenticated = False
+        else:
+            try:
+                import pam
+            except ImportError:
+                log.debug('PAM authenticate: could not load pam module, PAM authentication disabled')
+                return None, '', ''
+
+            p_auth = pam.pam()
+            authenticated = p_auth.authenticate(pam_username, password, service=pam_service)
+
+        if authenticated:
+            log.debug('PAM authentication successful for {}'.format(pam_username))
+            return True, auto_register_email, auto_register_username
+        else:
+            log.debug('PAM authentication failed for {}'.format(pam_username))
+            return False, '', ''
+
+    def authenticate_user(self, user, password, options):
+        return self.authenticate(user.email, user.username, password, options)[0]
+
+__all__ = ['PAM']

lib/galaxy/tools/deps/resolvers/unlinked_tool_shed_packages.py

@@ -0,0 +1,184 @@
+"""
+Backup resolvers for when dependencies can not be loaded from the database.
+Mainly suited for testing stage.
+
+Ideally all dependencies will be stored in the database
+    when a tool is added from a Tool Shed.
+That should remain the preferred way of locating dependencies.
+
+In cases where that is not possible
+    for example during testing this resolver can act as a backup.
+This resolver looks not just for manually added dependencies
+    but also ones added from a Tool Shed.
+
+This tool is still under development so the default behaviour could change.
+It has been tested when placed in the same directory as galaxy_packages.py
+
+At the time of writing July 3 2015 this resolver has to be plugged in.
+See bottom for instructions on how to add this resolver.
+
+"""
+
+from os import listdir
+from os.path import join, exists, getmtime
+
+from .galaxy_packages import GalaxyPackageDependencyResolver
+from ..resolvers import INDETERMINATE_DEPENDENCY
+
+import logging
+log = logging.getLogger( __name__ )
+
+MANUAL = "manual"
+PREFERRED_OWNERS = MANUAL + ",iuc,devteam"
+
+
+class UnlinkedToolShedPackageDependencyResolver(GalaxyPackageDependencyResolver):
+    resolver_type = "unlinked_tool_shed_packages"
+
+    def __init__(self, dependency_manager, **kwds):
+        super(UnlinkedToolShedPackageDependencyResolver, self).__init__(dependency_manager, **kwds)
+        # Provide a list of preferred owners whose dependency to use
+        self.preferred_owners = kwds.get('preferred_owners', PREFERRED_OWNERS).split(",")
+        # Option to ignore owner and just use last modified time
+        self.select_by_owner = str(kwds.get('select_by_owner', "true")).lower() != "false"
+
+    def _find_dep_versioned( self, name, version, type='package', **kwds ):
+        try:
+            possibles = self._find_possible_depenencies(name, version, type)
+            if len(possibles) == 0:
+                log.debug("Unable to find dependency,'%s' '%s' '%s'", name, version, type)
+                return INDETERMINATE_DEPENDENCY
+            elif len(possibles) == 1:
+                # Only one candidate found so ignore any preference rules
+                return possibles[0].dependency
+            else:
+                # Pick the preferred one
+                return self._select_preferred_dependency(possibles).dependency
+        except:
+            log.exception("Unexpected error hunting for dependency '%s' '%s''%s'", name, version, type)
+            return INDETERMINATE_DEPENDENCY
+
+    # Finds all possible dependency to use
+    # Should be extended as required
+    # Returns CandidateDepenency objects with data for preference picking
+    def _find_possible_depenencies(self, name, version, type):
+        possibles = []
+        if exists(self.base_path):
+            path = join( self.base_path, name, version )
+            if exists(path):
+                # First try the way without owner/name/revision
+                package = self._galaxy_package_dep(path, version)
+                if package != INDETERMINATE_DEPENDENCY:
+                    log.debug("Found dependency '%s' '%s' '%s' at '%s'", name, version, type, path)
+                    possibles.append(CandidateDepenency(package, path))
+                # now try with an owner/name/revision
+                for owner in listdir(path):
+                    owner_path = join(path, owner)
+                    for package_name in listdir(owner_path):
+                        if package_name.startswith("package_" + name):
+                            package_path = join(owner_path, package_name)
+                            for revision in listdir(package_path):
+                                revision_path = join(package_path, revision)
+                                package = self._galaxy_package_dep(revision_path, version)
+                                if package != INDETERMINATE_DEPENDENCY:
+                                    log.debug("Found dependency '%s' '%s' '%s' at '%s'", name, version, type, revision_path)
+                                    possibles.append(CandidateDepenency(package, package_path, owner))
+        return possibles
+
+    def _select_preferred_dependency(self, possibles, by_owner=None):
+        if by_owner is None:
+            by_owner = self.select_by_owner
+        preferred = []
+        if by_owner:
+            for owner in self.preferred_owners:
+                for candidate in possibles:
+                    if candidate.owner == owner:
+                        preferred.append(candidate)
+                    if len(preferred) == 1:
+                        log.debug("Picked dependency based on owner '%s'", owner)
+                        return preferred[0]
+                    elif len(preferred) > 1:
+                        log.debug("Multiple dependency found with owner '%s'", owner)
+                        break
+        if len(preferred) == 0:
+            preferred = possibles
+        latest_modified = 0
+        for candidate in preferred:
+            modified = getmtime(candidate.path)
+            if latest_modified < modified:
+                latest_candidate = candidate
+                latest_modified = modified
+        log.debug("Picking dependency at '%s' as it was the last modified", latest_candidate.path)
+        return latest_candidate
+
+    """
+    #Currently no need has been found for expand the verionsless method
+    #This is an example of how it could be done
+    def _find_dep_default( self, name, type='package', **kwds ):
+        try:
+            possibles = TODO
+            if len(possibles) == 0:
+                log.debug("Unable to find dependency,'%s' default '%s'", name, type)
+                return INDETERMINATE_DEPENDENCY
+            elif len(possibles) == 1:
+                #Only one candidate found so ignore any preference rules
+                return possibles[0].dependency
+            else:
+                #Pick the preferred one
+                return self._select_preferred_dependency(possibles, by_owner=False).dependency
+        except:
+            log.exception("Unexpected error hunting for dependency '%s' default '%s'", name, type)
+            return INDETERMINATE_DEPENDENCY
+    """
+
+
+class CandidateDepenency():
+
+    def __init__(self, dependency, path, owner=MANUAL):
+        self.dependency = dependency
+        self.path = path
+        self.owner = owner
+
+__all__ = ['UnlinkedToolShedPackageDependencyResolver']
+
+"""
+At the time of writing July 3 2015 this resolver has to be plugged in.
+
+Adding resolver instructions:
+
+1. create a dependency_resolvers_config.xml file
+<dependency_resolvers>
+  <tool_shed_packages />
+  <galaxy_packages />
+  <galaxy_packages versionless="true" />
+  <unlinked_tool_shed_packages />
+</dependency_resolvers>
+
+
+1a. ALWAYS add <tool_shed_packages /> first!!!!
+
+1b. <galaxy_packages /> is optional as
+    this resolver will also find dependency found by that resolver
+1bi Current default is to use a dependency to find that way first!
+1bii So an alternative version of dependency_resolvers_config.xml
+<dependency_resolvers>
+  <tool_shed_packages />
+  <unlinked_tool_shed_packages />
+  <unlinked_tool_shed_packages versionless="true" />
+</dependency_resolvers>
+
+1c. See __init__ for optional config values
+1ci versionless currently is handled by the super class
+    GalaxyPackageDependencyResolver
+
+2. Add a parameter to config.ini
+dependency_resolvers_config_file = ./config/dependency_resolvers_config.xml
+
+2a. File name/path can be different
+2b. config key must be dependency_resolvers_config_file
+
+3. For planemo it may be required to specify:
+--dependency_resolvers_config_file (xml file described in 1 above)
+--tool_dependency_dir (root of dependencies typically galaxy/dependency_dir)
+See planemo test --help for more information
+"""

lib/galaxy/tools/evaluation.py

@@ -329,7 +329,8 @@ def __populate_output_dataset_wrappers(self, param_dict, output_datasets, output
                 # Assume the reason we lack this output is because a filter
                 # failed to pass; for tool writing convienence, provide a
                 # NoneDataset
-                param_dict[ out_name ] = NoneDataset( datatypes_registry=self.app.datatypes_registry, ext=output.format )
+                ext = getattr( output, "format", None )  # populate only for output datasets (not collections)
+                param_dict[ out_name ] = NoneDataset( datatypes_registry=self.app.datatypes_registry, ext=ext )
 
     def __populate_non_job_params(self, param_dict):
         # -- Add useful attributes/functions for use in creating command line.

lib/galaxy/tools/parser/xml.py

@@ -186,6 +186,7 @@ def _parse(data_elem, **kwds):
                 inherit_metadata = string_as_bool( collection_elem.get( "inherit_metadata", None ) )
             default_format_source = collection_elem.get( "format_source", None )
             default_metadata_source = collection_elem.get( "metadata_source", "" )
+            filters = collection_elem.findall( 'filter' )
 
             dataset_collectors = None
             if collection_elem.find( "discover_datasets" ) is not None:
@@ -199,6 +200,7 @@ def _parse(data_elem, **kwds):
                 name,
                 structure,
                 label=label,
+                filters=filters,
                 default_format=default_format,
                 inherit_format=inherit_format,
                 inherit_metadata=inherit_metadata,