Simplify Django logging config (#2403)

* Simplify Django logging config

* Assign unique name to each Celery worker

* Rename get_query_set() to get_queryset() for compatibility with Django 1.8

* Rename _meta.module_name to _meta.model_name for compatibility with Django 1.8

refinery/config/config.json.erb

@@ -42,7 +42,6 @@
   "JAVA_ENTITY_EXPANSION_LIMIT": 1000000,
   "LANGUAGE_CODE": "en-us",
   "LIBS_DIR": "/opt",
-  "LOG_LEVEL": "DEBUG",
   "MEDIA_ROOT": "<%= @media_root %>",
   "MEDIA_URL": "/media/",
   "NEO4J": {
@@ -105,6 +104,7 @@
   "REFINERY_GOOGLE_ANALYTICS_ID": "",
   "REFINERY_INNER_NAVBAR_HEIGHT": 20,
   "REFINERY_AUXILIARY_FILE_GENERATION": "on_file_import",
+  "REFINERY_LOG_LEVEL": "DEBUG",
   "REFINERY_MAIN_LOGO": "",
   "REFINERY_REGISTRATION_CLOSED_MESSAGE": "",
   "REFINERY_REPOSITORY_MODE": false,

refinery/config/settings/base.py

@@ -232,7 +232,7 @@ def get_setting(name, settings=local_settings, default=None):
 
 LOGGING = {
     'version': 1,
-    'disable_existing_loggers': True,
+    'disable_existing_loggers': False,
     'root': {
         'level': 'DEBUG',
         'handlers': ['console'],
@@ -256,101 +256,39 @@ def get_setting(name, settings=local_settings, default=None):
             'class': 'django.utils.log.AdminEmailHandler'
         },
         'console': {
-            'level': get_setting("LOG_LEVEL"),
+            'level': get_setting("REFINERY_LOG_LEVEL"),
             'class': 'logging.StreamHandler',
             'formatter': 'default'
         },
     },
     'loggers': {
-        'django.request': {
-            'handlers': ['console', 'mail_admins'],
-            'level': 'ERROR',
-            'propagate': True,
-        },
-        'django.db.backends': {
-            'level': 'ERROR',
-            'handlers': ['console'],
-            'propagate': False,
+        'django': {
+            'level': 'WARNING',
+            'handlers': ['mail_admins'],
         },
-        'analysis_manager': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': False,
+        'boto3': {
+            'level': 'INFO',
         },
-        'annotation_server': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': False,
-        },
-        'core': {
-            'level': 'DEBUG',
-            'handlers': ['console'],
-            'propagate': False,
-        },
-        'data_set_manager': {
-            'level': 'DEBUG',
-            'handlers': ['console'],
-            'propagate': False,
+        'botocore': {
+            'level': 'INFO',
         },
         'docker': {
             'level': 'ERROR',
-            'handlers': ['console'],
-            'propagate': False,
         },
         'easyprocess': {
-            'handlers': ['console'],
             'level': 'ERROR',
-            'propagate': False,
         },
         'factory': {
-            'handlers': ['console'],
             'level': 'ERROR',
-            'propagate': False,
-        },
-        'file_server': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': False,
-        },
-        'file_store': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': False,
-        },
-        'galaxy_connector': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': False,
         },
         'httpproxy': {
             'level': 'ERROR',
-            'handlers': ['console'],
-            'propagate': False,
-        },
-        'isa_tab_parser': {
-            'level': 'DEBUG',
-            'handlers': ['console'],
-            'propagate': False,
         },
         'requests': {
-            'handlers': ['console'],
             'level': 'ERROR',
-            'propagate': False,
         },
         'selenium': {
-            'handlers': ['console'],
             'level': 'ERROR',
-            'propagate': False,
-        },
-        'visualization_manager': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': False,
-        },
-        'workflow_manager': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': False,
         },
     },
 }
@@ -375,6 +313,8 @@ def get_setting(name, settings=local_settings, default=None):
 # for system stability
 CELERYD_MAX_TASKS_PER_CHILD = get_setting("CELERYD_MAX_TASKS_PER_CHILD")
 CELERY_ROUTES = {"file_store.tasks.import_file": {"queue": "file_import"}}
+# TODO: restrict to CELERY_TASK_SERIALIZER
+CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
 
 # TODO: Does this belong here or in config.json.erb?
 CELERYBEAT_SCHEDULE = {
@@ -384,7 +324,6 @@ def get_setting(name, settings=local_settings, default=None):
     },
 }
 
-
 CHUNKED_UPLOAD_ABSTRACT_MODEL = False
 
 # === Refinery Settings ===

refinery/core/api.py

@@ -964,7 +964,7 @@ class AnalysisResource(ModelResource):
 
     class Meta:
         queryset = Analysis.objects.all()
-        resource_name = Analysis._meta.module_name
+        resource_name = Analysis._meta.model_name
         detail_uri_name = 'uuid'  # for using UUIDs instead of pk in URIs
         # required for public data set access by anonymous users
         authentication = Authentication()
@@ -1009,14 +1009,14 @@ def dehydrate(self, bundle):
         workflow_dict = None
         try:
             workflow_dict = ast.literal_eval(bundle.data['workflow_copy'])
-        except ValueError, e:
+        except ValueError as exc:
             # TODO: Remove this and just let any errors bubble up.
             # I don't think I should do that at this moment because
             # I shouldn't make production any fussier about values
             # in DB than it is right now.
             logger.warn(
                 '%s: Cannot parse workflow as python repr: %s',
-                e, bundle.data['workflow_copy']
+                exc, bundle.data['workflow_copy']
             )
         if workflow_dict:
             workflow_json = json.dumps(workflow_dict)
@@ -1026,16 +1026,13 @@ def dehydrate(self, bundle):
 
     def get_object_list(self, request, **kwargs):
         user = request.user
-        perm = 'read_%s' % DataSet._meta.module_name
-        if (user.is_authenticated()):
+        perm = 'read_%s' % DataSet._meta.model_name
+        if user.is_authenticated():
             allowed_datasets = get_objects_for_user(user, perm, DataSet)
         else:
             allowed_datasets = get_objects_for_group(
-                ExtendedGroup.objects.public_group(),
-                perm,
-                DataSet
+                ExtendedGroup.objects.public_group(), perm, DataSet
             )
-
         return Analysis.objects.filter(
             data_set__in=allowed_datasets.values_list("id", flat=True)
         )

refinery/core/models.py

@@ -485,7 +485,7 @@ def delete(self):
 
 
 class DataSetManager(models.Manager):
-    def get_query_set(self):
+    def get_queryset(self):
         return DataSetQuerySet(self.model, using=self._db)
 
 
@@ -934,7 +934,7 @@ def delete(self):
 
 
 class WorkflowManager(models.Manager):
-    def get_query_set(self):
+    def get_queryset(self):
         return WorkflowQuerySet(self.model, using=self._db)
 
 
@@ -1108,7 +1108,7 @@ def delete(self):
 
 
 class AnalysisManager(models.Manager):
-    def get_query_set(self):
+    def get_queryset(self):
         return AnalysisQuerySet(self.model, using=self._db)
 
 

refinery/supervisord.conf.erb

@@ -29,7 +29,7 @@ priority = 995
 
 [program:worker1]
 ; minimum of four processes required to avoid problems with monitoring tasks
-command = python %(here)s/manage.py celeryd -c 4 -Q celery --events
+command = python %(here)s/manage.py celeryd -c 4 -Q celery -E -n worker1.%%h
 environment = PATH="<%= @virtualenv %>/bin"
 stdout_logfile = %(here)s/log/celeryd-w1.log
 stdout_logfile_maxbytes = 5MB
@@ -47,7 +47,7 @@ priority = 995
 
 [program:worker2]
 ; limits concurrency of file imports to one to avoid overloading system IO
-command = python %(here)s/manage.py celeryd -c 1 -Q file_import --events
+command = python %(here)s/manage.py celeryd -c 1 -Q file_import -E -n worker2.%%h
 environment = PATH="<%= @virtualenv %>/bin"
 stdout_logfile = %(here)s/log/celeryd-w2.log
 stdout_logfile_maxbytes = 5MB

refinery/tool_manager/models.py

@@ -267,7 +267,7 @@ def delete_input_files_and_file_relationships(sender, instance, *args,
 
 
 class ToolManager(models.Manager):
-    def get_query_set(self):
+    def get_queryset(self):
         return VisualizationTool.objects.all() | WorkflowTool.objects.all()