Merge pull request #4456 from nsoranzo/logging_fixes

Logging fixes

.ci/flake8_lint_include_list.txt

@@ -94,6 +94,7 @@ lib/galaxy/web/framework/middleware/static.py
 lib/galaxy/web/framework/middleware/statsd.py
 lib/galaxy/web/framework/middleware/translogger.py
 lib/galaxy/web/framework/middleware/xforwardedhost.py
+lib/galaxy/web/framework/webapp.py
 lib/galaxy/web/__init__.py
 lib/galaxy/web/params.py
 lib/galaxy/webapps/galaxy/api/genomes.py
@@ -106,6 +107,7 @@ lib/galaxy/webapps/galaxy/api/roles.py
 lib/galaxy/webapps/galaxy/api/samples.py
 lib/galaxy/webapps/galaxy/api/tools.py
 lib/galaxy/webapps/galaxy/api/tours.py
+lib/galaxy/webapps/galaxy/api/users.py
 lib/galaxy/webapps/galaxy/api/workflows.py
 lib/galaxy/webapps/galaxy/config_watchers.py
 lib/galaxy/webapps/galaxy/controllers/async.py
@@ -188,6 +190,7 @@ scripts/build_universe_config.py
 scripts/check_galaxy.py
 scripts/check_python.py
 scripts/cleanup_datasets/
+scripts/communication/
 scripts/create_db.py
 scripts/data_libraries/
 scripts/db_shell.py
@@ -214,6 +217,7 @@ scripts/migrate_tools/
 scripts/nosetests.py
 scripts/others/
 scripts/runtime_stats.py
+scripts/secret_decoder_ring.py
 scripts/set_dataset_sizes.py
 scripts/set_user_disk_usage.py
 scripts/sync_reports_config.py

.ci/py3_sources.txt

@@ -35,9 +35,11 @@ lib/galaxy/web/framework/__init__.py
 lib/galaxy/web/framework/middleware/error.py
 lib/galaxy/web/framework/middleware/static.py
 lib/galaxy/web/framework/middleware/statsd.py
+lib/galaxy/web/framework/webapp.py
 lib/galaxy/web/__init__.py
 lib/galaxy/webapps/galaxy/api/histories.py
 lib/galaxy/webapps/galaxy/api/tours.py
+lib/galaxy/webapps/galaxy/api/users.py
 lib/galaxy/webapps/galaxy/api/workflows.py
 lib/galaxy/webapps/galaxy/controllers/external_services.py
 lib/galaxy/webapps/galaxy/controllers/search.py
@@ -64,9 +66,11 @@ scripts/check_eggs.py
 scripts/check_galaxy.py
 scripts/check_python.py
 scripts/cleanup_datasets/
+scripts/communication/
 scripts/data_libraries/build_whoosh_index.py
 scripts/db_shell.py
 scripts/drmaa_external_runner.py
+scripts/secret_decoder_ring.py
 test/
 tool_list.py
 tools/

lib/galaxy/config.py

@@ -246,7 +246,7 @@ def __init__(self, **kwargs):
         self.cleanup_job = kwargs.get("cleanup_job", "always")
         preserve_python_environment = kwargs.get("preserve_python_environment", "legacy_only")
         if preserve_python_environment not in ["legacy_only", "legacy_and_local", "always"]:
-            log.warn("preserve_python_environment set to unknown value [%s], defaulting to legacy_only")
+            log.warning("preserve_python_environment set to unknown value [%s], defaulting to legacy_only")
             preserve_python_environment = "legacy_only"
         self.preserve_python_environment = preserve_python_environment
         # Older default container cache path, I don't think anyone is using it anymore and it wasn't documented - we

lib/galaxy/containers/__init__.py

@@ -121,7 +121,7 @@ def __init__(self, conf, key, containers_config_file):
         self._key = key
         self._containers_config_file = containers_config_file
         mro = reversed(self.__class__.__mro__)
-        mro.next()
+        next(mro)
         self._conf = ContainerInterfaceConfig()
         for c in mro:
             self._conf.update(c.conf_defaults)
@@ -310,7 +310,7 @@ def parse_containers_config(containers_config_file):
             conf.update(c.get('containers', {}))
     except (OSError, IOError) as exc:
         if exc.errno == errno.ENOENT:
-            log.warning("config file '%s' does not exist, running with default config", containers_config_file)
+            log.debug("config file '%s' does not exist, running with default config", containers_config_file)
         else:
             raise
     return conf
@@ -320,10 +320,8 @@ def _get_interface_modules():
     interfaces = []
     modules = submodules(sys.modules[__name__])
     for module in modules:
-        classes = filter(
-            lambda x: inspect.isclass(x)
-                      and not x == ContainerInterface           # noqa: E131
-                      and issubclass(x, ContainerInterface),    # noqa: E131
-            [getattr(module, x) for x in dir(module)])
+        module_names = [getattr(module, _) for _ in dir(module)]
+        classes = [_ for _ in module_names if inspect.isclass(_) and
+            not _ == ContainerInterface and issubclass(_, ContainerInterface)]
         interfaces.extend(classes)
-    return dict([(x.container_type, x) for x in interfaces])
+    return dict((x.container_type, x) for x in interfaces)

lib/galaxy/jobs/runners/kubernetes.py

@@ -132,7 +132,7 @@ def __get_supplemental_group(self):
             try:
                 return int(self.runner_params["k8s_supplemental_group_id"])
             except:
-                log.warn("Supplemental group passed for Kubernetes runner needs to be an integer, value "
+                log.warning("Supplemental group passed for Kubernetes runner needs to be an integer, value "
                          + self.runner_params["k8s_supplemental_group_id"] + " passed is invalid")
                 return None
         return None
@@ -142,7 +142,7 @@ def __get_fs_group(self):
             try:
                 return int(self.runner_params["k8s_fs_group_id"])
             except:
-                log.warn("FS group passed for Kubernetes runner needs to be an integer, value "
+                log.warning("FS group passed for Kubernetes runner needs to be an integer, value "
                          + self.runner_params["k8s_fs_group_id"] + " passed is invalid")
                 return None
         return None

lib/galaxy/tools/execute.py

@@ -88,7 +88,7 @@ def worker():
         if len(param_combinations) == 0:
             template = "Attempting to map over an empty collection, this is not yet implemented. collection_info is [%s]"
             message = template % collection_info
-            log.warn(message)
+            log.warning(message)
             raise Exception(message)
         params = param_combinations[0]
         execution_tracker.create_output_collections(trans, history, params)

lib/galaxy/web/framework/webapp.py

@@ -1,41 +1,43 @@
 """
 """
 import datetime
+import hashlib
 import inspect
+import logging
 import os
-import hashlib
 import random
 import socket
 import string
 import time
-import urlparse
-from Cookie import CookieError
 from importlib import import_module
 
-from Cheetah.Template import Template
-import mako.runtime
 import mako.lookup
-from babel.support import Translations
+import mako.runtime
 from babel import Locale
+from babel.support import Translations
+from Cheetah.Template import Template
 from six import string_types
+from six.moves.http_cookies import CookieError
+from six.moves.urllib.parse import urlparse
 from sqlalchemy import and_, true
-from sqlalchemy.orm.exc import NoResultFound
 from sqlalchemy.orm import joinedload
-
-from galaxy.exceptions import MessageException
+from sqlalchemy.orm.exc import NoResultFound
 
 from galaxy import util
-from galaxy.util import asbool
-from galaxy.util import safe_str_cmp
-from galaxy.util.sanitize_html import sanitize_html
-
+from galaxy.exceptions import MessageException
 from galaxy.managers import context
-from galaxy.web.framework import url_for
-from galaxy.web.framework import base
-from galaxy.web.framework import helpers
-from galaxy.web.framework import formbuilder
+from galaxy.util import (
+    asbool,
+    safe_str_cmp
+)
+from galaxy.util.sanitize_html import sanitize_html
+from galaxy.web.framework import (
+    base,
+    formbuilder,
+    helpers,
+    url_for
+)
 
-import logging
 log = logging.getLogger(__name__)
 
 
@@ -303,7 +305,7 @@ def is_allowed_origin(origin):
             return False
 
         # boil origin header down to hostname
-        origin = urlparse.urlparse(origin_header).hostname
+        origin = urlparse(origin_header).hostname
         # check against the list of allowed strings/regexp hostnames, echo original if cleared
         if is_allowed_origin(origin):
             self.response.headers['Access-Control-Allow-Origin'] = origin_header
@@ -602,10 +604,10 @@ def get_or_create_remote_user(self, remote_user_email):
             username = remote_user_email.split('@', 1)[0].lower()
             random.seed()
             user = self.app.model.User(email=remote_user_email)
-            user.set_password_cleartext(''.join(random.sample(string.letters + string.digits, 12)))
+            user.set_password_cleartext(''.join(random.sample(string.ascii_letters + string.digits, 12)))
             user.external = True
             # Replace invalid characters in the username
-            for char in filter(lambda x: x not in string.ascii_lowercase + string.digits + '-', username):
+            for char in [x for x in username if x not in string.ascii_lowercase + string.digits + '-']:
                 username = username.replace(char, '-')
             # Find a unique username - user can change it later
             if self.sa_session.query(self.app.model.User).filter_by(username=username).first():
@@ -894,10 +896,10 @@ def check_csrf_token(self):
         session_csrf_token = self.request.params.get("session_csrf_token", None)
         problem = False
         if not session_csrf_token:
-            log.warn("No session_csrf_token set, denying request.")
+            log.warning("No session_csrf_token set, denying request.")
             problem = True
         elif session_csrf_token != self.session_csrf_token:
-            log.warn("Wrong session token found, denying request.")
+            log.warning("Wrong session token found, denying request.")
             problem = True
 
         if problem:

lib/galaxy/webapps/galaxy/api/users.py

@@ -1,37 +1,59 @@
 """
 API operations on User objects.
 """
-import logging
 import json
+import logging
 import random
 import re
 import socket
-
 from datetime import datetime
-from markupsafe import escape
-from sqlalchemy import false, true, and_, or_
 
+import six
 import yaml
-
-from galaxy import exceptions, util, web
-from galaxy.exceptions import MessageException, ObjectInvalid
+from markupsafe import escape
+from sqlalchemy import (
+    and_,
+    false,
+    or_,
+    true
+)
+
+from galaxy import (
+    exceptions,
+    util,
+    web
+)
+from galaxy.exceptions import (
+    MessageException,
+    ObjectInvalid
+)
 from galaxy.managers import users
-from galaxy.security.validate_user_input import validate_email
-from galaxy.security.validate_user_input import validate_password
-from galaxy.security.validate_user_input import validate_publicname
-from galaxy.web import url_for
-from galaxy.web import _future_expose_api as expose_api
-from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous
-from galaxy.web.base.controller import BaseAPIController
-from galaxy.web.base.controller import CreatesApiKeysMixin
-from galaxy.web.base.controller import CreatesUsersMixin
-from galaxy.web.base.controller import UsesTagsMixin
-from galaxy.web.base.controller import BaseUIController
-from galaxy.web.base.controller import UsesFormDefinitionsMixin
-from galaxy.web.form_builder import AddressField
+from galaxy.security.validate_user_input import (
+    validate_email,
+    validate_password,
+    validate_publicname
+)
 from galaxy.tools.toolbox.filters import FilterFactory
-from galaxy.util import docstring_trim, listify, hash_util
+from galaxy.util import (
+    docstring_trim,
+    hash_util,
+    listify
+)
 from galaxy.util.odict import odict
+from galaxy.web import (
+    _future_expose_api as expose_api,
+    _future_expose_api_anonymous as expose_api_anonymous,
+    url_for
+)
+from galaxy.web.base.controller import (
+    BaseAPIController,
+    BaseUIController,
+    CreatesApiKeysMixin,
+    CreatesUsersMixin,
+    UsesFormDefinitionsMixin,
+    UsesTagsMixin
+)
+from galaxy.web.form_builder import AddressField
 
 
 log = logging.getLogger(__name__)
@@ -262,7 +284,7 @@ def _get_extra_user_preferences(self, trans):
             with open(path, 'r') as stream:
                 config = yaml.load(stream)
         except:
-            log.warn('Config file (%s) could not be found or is malformed.' % path)
+            log.warning('Config file (%s) could not be found or is malformed.' % path)
             return {}
 
         return config['preferences'] if config else {}
@@ -541,7 +563,7 @@ def get_activation_token(self, trans, email):
 
     def _validate_email_publicname(self, email, username):
         ''' Validate email and username using regex '''
-        if email == '' or not isinstance(email, basestring):
+        if email == '' or not isinstance(email, six.string_types):
             return 'Please provide your email address.'
         if not re.match('^[a-z0-9\-]{3,255}$', username):
             return 'Public name must contain only lowercase letters, numbers and "-". It also has to be shorter than 255 characters but longer than 2.'
@@ -615,9 +637,8 @@ def get_permissions(self, trans, id, payload={}, **kwd):
         """
         user = self._get_user(trans, id)
         roles = user.all_roles()
-        permitted_actions = trans.app.model.Dataset.permitted_actions.items()
         inputs = []
-        for index, action in permitted_actions:
+        for index, action in trans.app.model.Dataset.permitted_actions.items():
             inputs.append({'type': 'select',
                            'multiple': True,
                            'optional': True,
@@ -634,9 +655,8 @@ def set_permissions(self, trans, id, payload={}, **kwd):
         Set the user's default permissions for the new histories
         """
         user = self._get_user(trans, id)
-        permitted_actions = trans.app.model.Dataset.permitted_actions.items()
         permissions = {}
-        for index, action in permitted_actions:
+        for index, action in trans.app.model.Dataset.permitted_actions.items():
             action_id = trans.app.security_agent.get_action(action.action).action
             permissions[action_id] = [trans.sa_session.query(trans.app.model.Role).get(x) for x in (payload.get(index) or [])]
         trans.app.security_agent.user_set_default_permissions(user, permissions)

scripts/cleanup_datasets/admin_cleanup_datasets.py

@@ -60,7 +60,7 @@
 from cleanup_datasets import CleanupDatasetsApplication  # noqa: I100
 
 log = logging.getLogger()
-log.setLevel(10)
+log.setLevel(logging.INFO)
 log.addHandler(logging.StreamHandler(sys.stdout))
 
 assert sys.version_info[:2] >= (2, 4)

scripts/cleanup_datasets/cleanup_datasets.py

@@ -24,7 +24,7 @@
 from galaxy.util import unicodify
 
 log = logging.getLogger()
-log.setLevel(10)
+log.setLevel(logging.INFO)
 log.addHandler(logging.StreamHandler(sys.stdout))
 
 assert sys.version_info[:2] >= (2, 4)