Merge pull request #14 from kayak/rollup_fix

Fixes for Rollup operation in highcharts

docs/2_slicer.rst

@@ -318,7 +318,7 @@ For each |FeatureReference|, there are the following variations:
 
 .. code-block:: python
 
-    from fireant.slicer import WoW, DeltaMoM, DeltaQoQ
+    from fireant.slicer.references import WoW, DeltaMoM, DeltaQoQ
 
     slicer.notebook.column_index_table(
         metrics=['clicks', 'conversions'],
@@ -328,4 +328,30 @@ For each |FeatureReference|, there are the following variations:
 
 .. note::
 
-    For any reference, the comparison is made for the same days of the week.
+    For any reference, the comparison is made for the same days of the week.
+
+
+Post-Processing Operations
+--------------------------
+
+Operations include extra computations that modify the final results.
+
+Totals
+""""""
+
+Totals adds ``ROLLUP`` to the SQL query to load the data and aggregated across dimensions.  It requires one or more dimension keys as parameters for the dimensions that should be totaled.  The below example will add an extra line with the total clicks and conversions for each date in addition to the three lines for each device type, desktop, mobile and tablet.
+
+.. code-block:: python
+
+    from fireant.slicer.operations import Totals
+
+    slicer.notebook.line_chart(
+        metrics=['clicks', 'conversions'],
+        dimensions=['date', 'device'],
+        operations=[Totals('device')],
+    )
+
+L1 and L2 Loss
+""""""""""""""
+
+Coming soon

fireant/slicer/__init__.py

@@ -1,7 +1,5 @@
 # coding: utf-8
 from .filters import EqualityFilter, ContainsFilter, RangeFilter, WildcardFilter
 from .managers import SlicerException
-from .operations import Rollup
-from .references import WoW, MoM, QoQ, YoY, Delta, DeltaPercentage
 from .schemas import (Slicer, Metric, Dimension, CategoricalDimension, ContinuousDimension, NumericInterval,
                       UniqueDimension, DatetimeDimension, DatetimeInterval, DimensionValue, EqualityOperator, Join)

fireant/slicer/operations.py

@@ -13,9 +13,9 @@ def schemas(self, slicer):
         raise NotImplementedError
 
 
-class Rollup(Operation):
-    def __init__(self, dimension_keys):
-        super(Rollup, self).__init__('rollup')
+class Totals(Operation):
+    def __init__(self, *dimension_keys):
+        super(Totals, self).__init__('rollup')
         self.dimension_keys = dimension_keys
 
     def schemas(self, slicer):

fireant/slicer/transformers/datatables.py

@@ -1,16 +1,17 @@
 # coding: utf-8
 import numpy as np
 import pandas as pd
+from future.types.newstr import newstr
 
 from .base import Transformer
 
 
 def _format_data_point(value):
-    if isinstance(value, str):
-        return value
+    if isinstance(value, (str, newstr)):
+        return str(value)
     if isinstance(value, pd.Timestamp):
         return value.strftime('%Y-%m-%dT%H:%M:%S')
-    if np.isnan(value):
+    if value is None or np.isnan(value):
         return None
     if isinstance(value, np.int64):
         # Cannot transform np.int64 to json
@@ -73,24 +74,28 @@ def _render_index_levels(self, idx, dim_ordinal, display_schema):
         for dimension in row_dimensions:
             key = dimension['label']
 
-            if not isinstance(idx, tuple):
-                value = _format_data_point(idx)
-
-            elif 1 < len(dimension['id_fields']) or 'label_field' in dimension:
+            if 'label_field' in dimension:
                 fields = dimension['id_fields'] + [dimension.get('label_field')]
 
                 value = {id_field: idx[dim_ordinal[id_field]]
-                         for id_field in filter(None, fields)}
+                         for id_field in fields
+                         if id_field is not None}
 
-            else:
+                yield key, value
+                continue
+
+            if isinstance(idx, tuple):
                 id_field = dimension['id_fields'][0]
                 value = _format_data_point(idx[dim_ordinal[id_field]])
 
-                if value is None:
-                    value = 'Total'
+            else:
+                value = _format_data_point(idx)
+
+            if 'label_options' in dimension:
+                value = dimension['label_options'].get(value, value)
 
-                if 'label_options' in dimension:
-                    value = dimension['label_options'].get(value, value)
+            if value is None:
+                value = 'Total'
 
             yield key, value
 

fireant/slicer/transformers/highcharts.py

@@ -85,20 +85,18 @@ def _reorder_index_levels(self, data_frame, display_schema):
         return reordered
 
     def _make_series(self, data_frame, dim_ordinal, display_schema, reference=None):
-        # This value represents how many iterations over data_frame items per yAxis we have.  It's the product of
-        # non-metric levels of the data_frame's columns. We want metrics to share the same yAxis for all dimensions.
-        yaxis_span = (np.product([len(l) for l in data_frame.columns.levels[1:]])
-                      if isinstance(data_frame.columns, pd.MultiIndex)
-                      else 1)
+        metrics = list(data_frame.columns.levels[0]
+                       if isinstance(data_frame.columns, pd.MultiIndex)
+                       else data_frame.columns)
 
-        return [self._make_series_item(idx, item, dim_ordinal, display_schema, int(i // yaxis_span), reference)
-                for i, (idx, item) in enumerate(data_frame.iteritems())]
+        return [self._make_series_item(idx, item, dim_ordinal, display_schema, metrics, reference)
+                for idx, item in data_frame.iteritems()]
 
-    def _make_series_item(self, idx, item, dim_ordinal, display_schema, y_axis, reference):
+    def _make_series_item(self, idx, item, dim_ordinal, display_schema, metrics, reference):
         return {
             'name': self._format_label(idx, dim_ordinal, display_schema, reference),
             'data': self._format_data(item),
-            'yAxis': y_axis,
+            'yAxis': metrics.index(idx[0] if isinstance(idx, tuple) else idx),
             'dashStyle': 'Dot' if reference else 'Solid'
         }
 
@@ -187,13 +185,15 @@ def _unstack_levels(self, dimensions, dim_ordinal):
 
 
 class HighchartsColumnTransformer(HighchartsLineTransformer):
-    def _make_series_item(self, idx, item, dim_ordinal, display_schema, y_axis, reference):
+    chart_type = 'column'
+
+    def _make_series_item(self, idx, item, dim_ordinal, display_schema, metrics, reference):
         return {
             'name': self._format_label(idx, dim_ordinal, display_schema, reference),
             'data': [_format_data_point(x)
                      for x in item
                      if not np.isnan(x)],
-            'yAxis': y_axis
+            'yAxis': metrics.index(idx[0] if isinstance(idx, tuple) else idx),
         }
 
     def xaxis_options(self, data_frame, dim_ordinal, display_schema):
@@ -232,6 +232,9 @@ def _make_categories(data_frame, dim_ordinal, display_schema):
         category_dimension = display_schema['dimensions'][0]
         if 'label_options' in category_dimension:
             return [category_dimension['label_options'].get(dim, dim)
+                    # Pandas gives both NaN or None in the index depending on whether a level was unstacked
+                    if dim and not (isinstance(dim, float) and np.isnan(dim))
+                    else 'Totals'
                     for dim in data_frame.index]
 
         if 'label_field' in category_dimension:

fireant/tests/slicer/test_slicer_api.py

@@ -4,6 +4,8 @@
 
 from fireant import settings
 from fireant.slicer import *
+from fireant.slicer.operations import Totals
+from fireant.slicer.references import *
 from fireant.tests.database.mock_database import TestDatabase
 from pypika import functions as fn, Tables, Case
 
@@ -702,7 +704,7 @@ def test_rollup_operation(self):
         query_schema = self.test_slicer.manager.query_schema(
             metrics=['foo'],
             dimensions=['date', 'locale', 'account'],
-            operations=[Rollup(['locale', 'account'])],
+            operations=[Totals('locale', 'account')],
         )
 
         self.assertTrue({'table', 'metrics', 'dimensions', 'rollup'}.issubset(query_schema.keys()))