Merge pull request #5479 from dannon/slow_request_timing_sentry

Slow request timing in sentry

config/galaxy.yml.sample

@@ -998,6 +998,12 @@ galaxy:
   # <project_name> -> Settings -> API Keys.
   #sentry_dsn: null
 
+  # Sentry slow request logging.  Requests slower than the threshold indicated
+  # below will be sent as events to the configured Sentry server (above).  A
+  # value of '0' is disabled.  For example, you would set this to .005 to log
+  # all queries taking longer than 5 milliseconds.
+  #sentry_sloreq_threshold: 0
+
   # Log to statsd Statsd is an external statistics aggregator
   # (https://github.com/etsy/statsd) Enabling the following options will
   # cause galaxy to log request timing and other statistics to the

lib/galaxy/web/framework/middleware/sentry.py

@@ -5,6 +5,7 @@
 :copyright: (c) 2010-2012 by the Sentry Team, see AUTHORS for more details.
 :license: BSD, see LICENSE for more details.
 """
+import time
 
 try:
     from raven import Client
@@ -25,10 +26,11 @@ class Sentry(object):
     uncaught exceptions and send them to Sentry.
     """
 
-    def __init__(self, application, dsn):
+    def __init__(self, application, dsn, sloreq):
         assert Client is not None, RAVEN_IMPORT_MESSAGE
         self.application = application
         self.client = None
+        self.sloreq_threshold = sloreq
 
         def postfork_sentry_client():
             self.client = Client(dsn)
@@ -37,7 +39,11 @@ def postfork_sentry_client():
 
     def __call__(self, environ, start_response):
         try:
+            start_time = time.time()
             iterable = self.application(environ, start_response)
+            dt = (time.time() - start_time)
+            if self.sloreq_threshold and dt > self.sloreq_threshold:
+                self.handle_slow_request(environ, dt)
         except Exception:
             self.handle_exception(environ)
             raise
@@ -57,6 +63,38 @@ def __call__(self, environ, start_response):
                 except Exception:
                     self.handle_exception(environ)
 
+    def handle_slow_request(self, environ, dt):
+        headers = dict(get_headers(environ))
+        if 'Authorization' in headers:
+            headers['Authorization'] = 'redacted'
+        if 'Cookie' in headers:
+            headers['Cookie'] = 'redacted'
+        cak = environ.get('controller_action_key', None) or environ.get('PATH_INFO', "NOPATH").strip('/').replace('/', '.')
+        event_id = self.client.captureMessage(
+            "SLOREQ: %s" % cak,
+            data={
+                'sentry.interfaces.Http': {
+                    'method': environ.get('REQUEST_METHOD'),
+                    'url': get_current_url(environ, strip_querystring=True),
+                    'query_string': environ.get('QUERY_STRING'),
+                    'headers': headers,
+                    'env': dict(get_environ(environ)),
+                }
+            },
+            extra={
+                'request_id': environ.get('request_id', 'Unknown'),
+                'request_duration_millis': dt * 1000
+            },
+            tags={
+                'type': 'sloreq',
+                'action_key': cak
+            }
+
+        )
+        # Galaxy: store event_id in environment so we can show it to the user
+        environ['sentry_event_id'] = event_id
+        return event_id
+
     def handle_exception(self, environ):
         headers = dict(get_headers(environ))
         # Authorization header for REMOTE_USER sites consists of a base64() of

lib/galaxy/webapps/galaxy/buildapp.py

@@ -995,9 +995,10 @@ def wrap_in_middleware(app, global_conf, application_stack, **local_conf):
     # If sentry logging is enabled, log here before propogating up to
     # the error middleware
     sentry_dsn = conf.get('sentry_dsn', None)
+    sentry_sloreq = float(conf.get('sentry_sloreq_threshold', 0))
     if sentry_dsn:
         from galaxy.web.framework.middleware.sentry import Sentry
-        app = wrap_if_allowed(app, stack, Sentry, args=(sentry_dsn,))
+        app = wrap_if_allowed(app, stack, Sentry, args=(sentry_dsn, sentry_sloreq))
     # Various debug middleware that can only be turned on if the debug
     # flag is set, either because they are insecure or greatly hurt
     # performance