Dttm format

caravel/migrations/versions/422116f431b7_add_d_timestamp_format_for_tables.py

@@ -0,0 +1,30 @@
+"""add d timestamp format for tables
+
+Revision ID: 422116f431b7
+Revises: 956a063c52b3
+Create Date: 2016-05-20 19:11:57.218062
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '422116f431b7'
+down_revision = '956a063c52b3'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+	try:
+		op.add_column('datasources', sa.Column('timestamp_format', sa.String(length=256), nullable=True))
+		op.add_column('tables', sa.Column('timestamp_format', sa.String(length=256), nullable=True))
+	except Exception:
+		pass
+
+
+def downgrade():
+	try:
+		op.drop_column('tables', 'timestamp_format')
+		op.drop_column('datasources', 'timestamp_format')
+	except Exception:
+		pass

caravel/models.py

@@ -428,9 +428,11 @@ def grains(self):
             if self.sqlalchemy_uri.startswith(db_type):
                 return grains
 
-    def dttm_converter(self, dttm):
+    def dttm_converter(self, dttm, tf=None):
         """Returns a string that the database flavor understands as a date"""
-        default = "'{}'".format(dttm.strftime('%Y-%m-%d %H:%M:%S.%f'))
+        if tf is None or tf == '':
+            tf = '%Y-%m-%d %H:%M:%S.%f'
+        default = "'{}'".format(dttm.strftime(tf))
         iso = dttm.isoformat()
         d = {
             'mssql': "CONVERT(DATETIME, '{}', 126)".format(iso), #untested
@@ -508,6 +510,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
     offset = Column(Integer, default=0)
     cache_timeout = Column(Integer)
     schema = Column(String(255))
+    timestamp_format = Column(String(256))
 
     baselink = "tablemodelview"
 
@@ -657,18 +660,18 @@ def query(  # sqla
                 select_exprs += [timestamp_grain]
                 groupby_exprs += [timestamp_grain]
 
-            tf = '%Y-%m-%d %H:%M:%S.%f'
+            tf = self.timestamp_format
             time_filter = [
-                timestamp >= text(self.database.dttm_converter(from_dttm)),
-                timestamp <= text(self.database.dttm_converter(to_dttm)),
+                timestamp >= text(self.database.dttm_converter(from_dttm, tf)),
+                timestamp <= text(self.database.dttm_converter(to_dttm, tf)),
             ]
             inner_time_filter = copy(time_filter)
             if inner_from_dttm:
                 inner_time_filter[0] = timestamp >= text(
-                    self.database.dttm_converter(inner_from_dttm))
+                    self.database.dttm_converter(inner_from_dttm, tf))
             if inner_to_dttm:
                 inner_time_filter[1] = timestamp <= text(
-                    self.database.dttm_converter(inner_to_dttm))
+                    self.database.dttm_converter(inner_to_dttm, tf))
         else:
             inner_time_filter = []
 
@@ -961,6 +964,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
         'DruidCluster', backref='datasources', foreign_keys=[cluster_name])
     offset = Column(Integer, default=0)
     cache_timeout = Column(Integer)
+    timestamp_format = Column(String(256))
 
     @property
     def metrics_combo(self):

caravel/views.py

@@ -287,10 +287,11 @@ class TableModelView(CaravelModelView, DeleteMixin):  # noqa
         'table_link', 'database', 'sql_link', 'is_featured',
         'changed_by_', 'changed_on_']
     add_columns = [
-        'table_name', 'database', 'schema',
+        'table_name', 'database', 'schema', 'timestamp_format',
         'default_endpoint', 'offset', 'cache_timeout']
     edit_columns = [
-        'table_name', 'is_featured', 'database', 'schema', 'description', 'owner',
+        'table_name', 'is_featured', 'database', 'schema',
+        'timestamp_format', 'description', 'owner',
         'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout']
     related_views = [TableColumnInlineView, SqlMetricInlineView]
     base_order = ('changed_on', 'desc')
@@ -299,6 +300,8 @@ class TableModelView(CaravelModelView, DeleteMixin):  # noqa
         'schema': (
             "Schema, as used only in some databases like Postgres, Redshift "
             "and DB2"),
+        'timestamp_format': (
+            "Use default(%Y-%m-%d %H:%M:%S.%f) if empty"),
         'description': Markup(
             "Supports <a href='https://daringfireball.net/projects/markdown/'>"
             "markdown</a>"),

caravel/viz.py

@@ -143,14 +143,16 @@ def get_df(self, query_obj=None):
         self.results = None
 
         # The datasource here can be different backend but the interface is common
+        timestamp_format = self.datasource.timestamp_format
         self.results = self.datasource.query(**query_obj)
         self.query = self.results.query
         df = self.results.df
         if df is None or df.empty:
             raise Exception("No data, review your incantations!")
         else:
             if 'timestamp' in df.columns:
-                df.timestamp = pd.to_datetime(df.timestamp, utc=False)
+                df.timestamp = pd.to_datetime(
+                    df.timestamp, utc=False, format=timestamp_format)
                 if self.datasource.offset:
                     df.timestamp += timedelta(hours=self.datasource.offset)
         df = df.fillna(0)