Merge 54dc668 into ab23ef7

fireant/slicer/queries.py

@@ -116,7 +116,10 @@ def _query_data(self, table, joins=None,
         if getattr(settings, 'debug', False):
             print("Executing query:\n----START----\n{query}\n-----END-----".format(query=querystring))
 
-        dataframe = settings.database.fetch_dataframe(querystring).set_index(
+        dataframe = settings.database.fetch_dataframe(querystring)
+        dataframe.columns = [col.decode('utf-8') if isinstance(col, bytes) else col
+                             for col in dataframe.columns]
+        dataframe = dataframe.set_index(
             # Removed the reference keys for now
             list(dimensions.keys())  # + ['{1}_{0}'.format(*ref) for ref in references.items()]
         ).sort_index()

fireant/slicer/transformers/datatables.py

@@ -11,10 +11,12 @@ class TableIndex(object):
 
 
 def format_data_point(value):
-    if value is np.nan:
-        value = None
+    if isinstance(value, str):
+        return value
     if isinstance(value, pd.Timestamp):
-        return value.isoformat()
+        return value.strftime('%Y-%m-%dT%H:%M:%S')
+    if np.isnan(value):
+        return None
     if isinstance(value, np.int64):
         # Cannot transform np.int64 to json
         return int(value)

fireant/slicer/transformers/highcharts.py

@@ -6,10 +6,14 @@
 
 
 def format_data_point(value):
+    if isinstance(value, str):
+        return value
     if isinstance(value, pd.Timestamp):
         return int(value.asm8) // int(1e6)
+    if np.isnan(value):
+        return None
     if isinstance(value, np.int64):
-        # Cannot transform np.int64 to json
+        # Cannot serialize np.int64 to json
         return int(value)
     return value
 
@@ -195,7 +199,9 @@ def __init__(self, chart_type=column):
     def _make_series_item(self, idx, item, dim_ordinal, display_schema, y_axis, reference):
         return {
             'name': self._format_label(idx, dim_ordinal, display_schema, reference),
-            'data': [format_data_point(x) for x in item if x is not np.nan],
+            'data': [format_data_point(x)
+                     for x in item
+                     if not np.isnan(x)],
             'yAxis': y_axis
         }
 
@@ -218,7 +224,7 @@ def _prepare_data_frame(self, data_frame, dim_ordinal, dimensions):
         # Replaces invalid values and unstacks the data frame for line charts.
 
         # Force all fields to be float (Safer for highcharts)
-        data_frame = data_frame.astype(np.float).replace([np.inf, -np.inf], np.nan)
+        data_frame = data_frame.replace([np.inf, -np.inf], np.nan)
 
         # Unstack multi-indices
         if 1 < len(dimensions):

fireant/tests/database/test_databases.py

@@ -15,7 +15,19 @@ def test_fetch(self, mock_connect):
 
         result = Database().fetch('SELECT 1')
 
-        self.assertEqual('OK', result)
+        self.assertEqual(mock_cursor.fetchall.return_value, result)
         mock_cursor_func.assert_called_once_with()
         mock_cursor.execute.assert_called_once_with('SELECT 1')
         mock_cursor.fetchall.assert_called_once_with()
+
+    @patch('pandas.read_sql', name='mock_read_sql')
+    @patch('fireant.database.Database.connect', name='mock_connect')
+    def test_fetch_dataframe(self, mock_connect, mock_read_sql):
+        query = 'SELECT 1'
+        mock_read_sql.return_value = 'OK'
+
+        result = Database().fetch_dataframe(query)
+
+        self.assertEqual(mock_read_sql.return_value, result)
+
+        mock_read_sql.assert_called_once_with(query, mock_connect().__enter__())

fireant/tests/slicer/transformers/test_datatables.py

@@ -1,8 +1,12 @@
 # coding: utf-8
+from datetime import date
+from unittest import TestCase
 
 import numpy as np
+import pandas as pd
 
 from fireant.slicer.transformers import TableIndex, DataTablesTransformer
+from fireant.slicer.transformers import datatables
 from fireant.tests.slicer.transformers.base import BaseTransformerTests
 
 
@@ -414,3 +418,24 @@ def test_rollup_cont_cat_cat_dim_multi_metric(self):
                         # skip the rest of the level if the previous level is rolled up
                         if label1 is None:
                             break
+
+
+class DatatablesUtilityTests(TestCase):
+    def test_nan_data_point(self):
+        # Needs to be cast to python int
+        result = datatables.format_data_point(np.nan)
+        self.assertIsNone(result)
+
+    def test_str_data_point(self):
+        result = datatables.format_data_point('abc')
+        self.assertEqual('abc', result)
+
+    def test_int64_data_point(self):
+        # Needs to be cast to python int
+        result = datatables.format_data_point(np.int64(1))
+        self.assertEqual(int(1), result)
+
+    def test_datetime_data_point(self):
+        # Needs to be converted to milliseconds
+        result = datatables.format_data_point(pd.Timestamp(date(2000, 1, 1)))
+        self.assertEqual('2000-01-01T00:00:00', result)

fireant/tests/slicer/transformers/test_highcharts.py

@@ -1,7 +1,12 @@
 # coding: utf-8
+from datetime import date
+from unittest import TestCase
 
+import numpy as np
+import pandas as pd
 
 from fireant.slicer.transformers import HighchartsTransformer, TransformationException, HighchartsColumnTransformer
+from fireant.slicer.transformers import highcharts
 from fireant.tests.slicer.transformers.base import BaseTransformerTests
 
 
@@ -353,3 +358,19 @@ def test_require_no_more_than_one_dimension_with_multi_metrics(self):
 
 class HighChartsBarTransformerTests(HighChartsColumnTransformerTests):
     type = HighchartsColumnTransformer.bar
+
+
+class HighchartsUtilityTests(TestCase):
+    def test_str_data_point(self):
+        result = highcharts.format_data_point('abc')
+        self.assertEqual('abc', result)
+
+    def test_int64_data_point(self):
+        # Needs to be cast to python int
+        result = highcharts.format_data_point(np.int64(1))
+        self.assertEqual(int(1), result)
+
+    def test_datetime_data_point(self):
+        # Needs to be converted to milliseconds
+        result = highcharts.format_data_point(pd.Timestamp(date(2000, 1, 1)))
+        self.assertEqual(946684800000, result)