All sorts of improvements

TODO.md

@@ -1,8 +1,8 @@
 # TODO
 * in/notin filters autocomplete
-* Highstock, sort legend based on y value: ![stackoverflow](http://stackoverflow.com/questions/6867607/want-to-sort-highcharts-tooltip-results)
 * compare time ranges
 * Add verbose_name and label method to metrics and columns
 * CSV
 * Save / bookmark / url shortener
 * on save, process metadata / generate metrics
+* Allow for post aggregations (ratios!)

app/highchart.py

@@ -24,6 +24,7 @@ def __init__(
             logy=False,
             xlim=None,
             ylim=None,
+            sort_legend_y=False,
             grid=False,
             zoom=None):
         self.df = df
@@ -42,6 +43,7 @@ def __init__(
         self.polar = polar
         self.grid = grid
         self.stacked = stacked
+        self.sort_legend_y = sort_legend_y
 
         chart['chart'] = {}
         chart['chart']["type"] = chart_type
@@ -61,6 +63,11 @@ def __init__(
 
         if tooltip:
             chart['tooltip'] = tooltip
+        if sort_legend_y:
+            if 'tooltip' not in chart:
+                chart['tooltip'] = {
+                    'formatter': "{{TOOLTIP_FORMATTER}}"
+                }
         if self.zoom:
             chart["zoomType"] = self.zoom
 
@@ -70,6 +77,39 @@ def __init__(
 
         self.chart = chart
 
+    @property
+    def tooltip_formatter(self):
+        if self.compare == 'percent':
+            tf = """
+            function() {
+               var s = '<b>' + Highcharts.dateFormat('%Y-%m-%d %H:%M:%S', new Date(this.x))+'</b><br/>';
+               var sortedPoints = this.points.sort(function(a, b){
+                     return ((a.point.change > b.point.change) ? -1 : ((a.point.change < b.point.change) ? 1 : 0));
+               });
+               $.each(sortedPoints , function(i, point) {
+               s += '<span style="color:'+ point.series.color +'">\u25CF</span> ' + point.series.name + ': ' + f(point.y) + ' (<b>' + Highcharts.numberFormat(point.point.change, 2) + '%</b>)' + '<br/>';
+               });
+
+               return s;
+            }
+            """
+        else:
+            tf = """
+            function() {
+               var s = '<b>' + Highcharts.dateFormat('%Y-%m-%d %H:%M:%S', new Date(this.x))+'</b><br/>';
+               var sortedPoints = this.points.sort(function(a, b){
+                     return ((a.y > b.y) ? -1 : ((a.y < b.y) ? 1 : 0));
+               });
+               $.each(sortedPoints , function(i, point) {
+               s += '<span style="color:'+ point.series.color +'">\u25CF</span> ' + point.series.name + ': ' + f(point.y) + '<br/>';
+               });
+
+               return s;
+            }
+            """
+        return tf
+
+
     def serialize_series(self):
         df = self.df
         chart = self.chart
@@ -114,14 +154,6 @@ def serialize_xaxis(self):
         if self.xlim:
             x_axis["min"] = self.xlim[0]
             x_axis["max"] = self.xlim[1]
-        '''
-        if "rot" in kwargs:
-            x_axis["labels"] = {"rotation": kwargs["rot"]}
-        if "fontsize" in kwargs:
-            x_axis.setdefault("labels", {})["style"] = {"fontSize": kwargs["fontsize"]}
-        if "xticks" in kwargs:
-            x_axis["tickPositions"] = kwargs["xticks"]
-        '''
         self.chart['xAxis'] = x_axis
 
     def serialize_yaxis(self):
@@ -135,14 +167,6 @@ def serialize_yaxis(self):
         if self.ylim:
             yAxis["min"] = self.ylim[0]
             yAxis["max"] = self.ylim[1]
-        '''
-        if "rot" in kwargs:
-            yAxis["labels"] = {"rotation": kwargs["rot"]}
-        if "fontsize" in kwargs:
-            yAxis.setdefault("labels", {})["style"] = {"fontSize": kwargs["fontsize"]}
-        if "yticks" in kwargs:
-            yAxis["tickPositions"] = kwargs["yticks"]
-        '''
         chart["yAxis"] = [yAxis]
         if self.secondary_y:
             yAxis2 = copy.deepcopy(yAxis)
@@ -153,6 +177,7 @@ def serialize_yaxis(self):
     @property
     def javascript_cmd(self):
         js = dumps(self.chart)
+        js = js.replace('"{{TOOLTIP_FORMATTER}}"', self.tooltip_formatter).replace("\n", " ")
         if self.stockchart:
             return "new Highcharts.StockChart(%s);" % js
         return "new Highcharts.Chart(%s);" %js

app/templates/panoramix/datasource.html

@@ -23,7 +23,12 @@
   <div class="col-md-3">
     <h3>
       {{ datasource.datasource_name }}
-      <a href="/datasourcemodelview/edit/{{ datasource.id }}"><span class="glyphicon glyphicon-edit"></span></a>
+      {% if datasource.description %}
+        <i class="fa fa-info-circle" data-toggle="tooltip" data-placement="bottom" title="{{ datasource.description }}"></i>
+      {% endif %}
+      <a href="/datasourcemodelview/edit/{{ datasource.id }}">
+        <i class="fa fa-edit"></i>
+      </a>
     </h3>
 
     <hr>
@@ -59,6 +64,7 @@ <h4>Filters</h4>
       </button>
       <hr>
       <button type="button" class="btn btn-primary" id="druidify">Druidify!</button>
+      <button type="button" class="btn btn-default" id="bookmark">Bookmark</button>
       <hr style="margin-bottom: 0px;">
       <img src="{{ url_for("static", filename="panoramix.png") }}" width=250>
     </form><br>
@@ -87,8 +93,10 @@ <h3>Latest Segment Metadata</h3>
 
 {% block tail_js %}
 {{ super() }}
+<script src="{{ url_for("static", filename="d3.min.js") }}"></script>
 <script>
 $( document ).ready(function() {
+  f = d3.format(".4s");
   function getParam(name) {
       name = name.replace(/[\[]/, "\\[").replace(/[\]]/, "\\]");
       var regex = new RegExp("[\\?&]" + name + "=([^&#]*)"),
@@ -98,6 +106,7 @@ <h3>Latest Segment Metadata</h3>
 
   $(".select2").select2();
   $("form").slideDown("slow");
+  $('[data-toggle="tooltip"]').tooltip();
 
   function set_filters(){
     for (var i=1; i<10; i++){
@@ -125,6 +134,7 @@ <h3>Latest Segment Metadata</h3>
       });
   }
   $("#plus").click(add_filter);
+  $("#bookmark").click(function () {alert("Not implemented yet...");})
   add_filter();
   $("#druidify").click(function () {
     var i = 1;

app/views.py

@@ -8,7 +8,7 @@
 from app import appbuilder, db, models, viz, utils
 from flask.ext.appbuilder.security.decorators import has_access, permission_name
 import config
-from wtforms.fields import Field
+from pydruid.client import doublesum
 
 
 class ColumnInlineView(CompactCRUDMixin, ModelView):
@@ -44,7 +44,7 @@ class DatasourceModelView(ModelView):
         'datasource_name', 'description', 'owner', 'is_featured', 'is_hidden',
         'default_endpoint']
     page_size = 100
-    order_columns = ['datasource_name']
+    base_order = ('datasource_name', 'asc')
 
 
 appbuilder.add_view(
@@ -102,6 +102,21 @@ def refresh_datasources(self):
         flash("Refreshed metadata from Druid!", 'info')
         return redirect("/datasourcemodelview/list/")
 
+    @expose("/autocomplete/<datasource>/<column>/")
+    def autocomplete(self, datasource, column):
+        client = utils.get_pydruid_client()
+        top = client.topn(
+            datasource=datasource,
+            granularity='all',
+            intervals='2013-10-04/2020-10-10',
+            aggregations={"count": doublesum("count")},
+            dimension=column,
+            metric='count',
+            threshold=1000,
+        )
+        values = sorted([d[column] for d in top[0]['result']])
+        return json.dumps(values)
+
 appbuilder.add_view_no_menu(Panoramix)
 appbuilder.add_link(
     "Refresh Metadata",

app/viz.py

@@ -221,27 +221,33 @@ class TimeSeriesViz(HighchartsViz):
     verbose_name = "Time Series - Line Chart"
     chart_type = "spline"
     stockchart = True
+    sort_legend_y = True
 
     def render(self):
         metrics = self.metrics
         df = self.df
         df = df.pivot_table(
             index="timestamp",
             columns=self.groupby,
-            values=metrics)
+            values=metrics,)
 
         rolling_periods = request.args.get("rolling_periods")
         rolling_type = request.args.get("rolling_type")
         if rolling_periods and rolling_type:
             if rolling_type == 'mean':
                 df = pd.rolling_mean(df, int(rolling_periods))
+            elif rolling_type == 'std':
+                df = pd.rolling_std(df, int(rolling_periods))
+            elif rolling_type == 'sum':
+                df = pd.rolling_sum(df, int(rolling_periods))
 
         chart = Highchart(
             df,
             compare=self.compare,
             chart_type=self.chart_type,
             stacked=self.stacked,
             stockchart=self.stockchart,
+            sort_legend_y=self.sort_legend_y,
             **CHART_ARGS)
         return super(TimeSeriesViz, self).render(chart_js=chart.javascript_cmd)