Adding some STATSD logging

superset/config.py

@@ -17,6 +17,10 @@
 from dateutil import tz
 from flask_appbuilder.security.manager import AUTH_DB
 
+from superset.stats_logger import DummyStatsLogger
+
+stats = DummyStatsLogger()
+
 BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 DATA_DIR = os.path.join(os.path.expanduser('~'), '.superset')
 if not os.path.exists(DATA_DIR):

superset/migrations/versions/a65458420354_add_result_backend_time_logging.py

@@ -0,0 +1,26 @@
+"""add_result_backend_time_logging
+
+Revision ID: a65458420354
+Revises: 2fcdcb35e487
+Create Date: 2017-04-25 10:00:58.053120
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'a65458420354'
+down_revision = '2fcdcb35e487'
+
+
+def upgrade():
+    op.add_column(
+        'query',
+        sa.Column(
+            'end_result_backend_time',
+            sa.Numeric(precision=20, scale=6),
+            nullable=True))
+
+
+def downgrade():
+    op.drop_column('query', 'end_result_backend_time')

superset/migrations/versions/a6c18f869a4e_query_start_running_time.py

@@ -13,7 +13,6 @@
 down_revision = '979c03af3341'
 
 
-
 def upgrade():
     op.add_column(
         'query',

superset/models/sql_lab.py

@@ -64,9 +64,11 @@ class Query(Model):
 
     # Using Numeric in place of DateTime for sub-second precision
     # stored as seconds since epoch, allowing for milliseconds
-    start_time = Column(Numeric(precision=3))
-    start_running_time = Column(Numeric(precision=3))
-    end_time = Column(Numeric(precision=3))
+    start_time = Column(Numeric(precision=20, scale=6))
+    start_running_time = Column(Numeric(precision=20, scale=6))
+    end_time = Column(Numeric(precision=20, scale=6))
+    end_result_backend_time = Column(Numeric(precision=20, scale=6))
+
     changed_on = Column(
         DateTime,
         default=datetime.utcnow,

superset/sql_lab.py

@@ -186,6 +186,7 @@ def handle_error(msg):
         logging.info("Storing results in results backend, key: {}".format(key))
         results_backend.set(key, utils.zlib_compress(payload))
         query.results_key = key
+        query.end_result_backend_time = utils.now_as_float()
 
     session.merge(query)
     session.commit()

superset/stats_logger.py

@@ -0,0 +1,58 @@
+
+class BaseStatsLogger(object):
+    """Base class for logging realtime events"""
+
+    def __init__(self, prefix='superset'):
+        self.prefix = prefix
+
+    def key(self, key):
+        if self.prefix:
+            return self.prefix + key
+        return key
+
+
+    def incr(self, key):
+        """Increment a counter"""
+        raise NotImplementedError()
+
+    def decr(self, key):
+        """Decrement a counter"""
+        raise NotImplementedError()
+
+    def gauge(self, key):
+        """Setup a gauge"""
+        raise NotImplementedError()
+
+
+class DummyStatsLogger(BaseStatsLogger):
+
+    def incr(self, key):
+        pass
+
+    def decr(self, key):
+        pass
+
+    def gauge(self, key):
+        pass
+
+
+try:
+    from statsd import StatsClient
+
+    class StatsdStatsLogger(BaseStatsLogger):
+        def __init__(self, host, port, prefix='superset'):
+            self.client = StatsClient(
+                  host=host,
+                  port=port,
+                  prefix=prefix)
+
+        def incr(self, key):
+            self.client.incr(key)
+
+        def decr(self, key):
+            self.client.decr(key)
+
+        def gauge(self, key):
+            self.client.gauge(key)
+except Exception as e:
+    pass