From 9ce4f684734c9c3000aa9fa30f5e953971410f6a Mon Sep 17 00:00:00 2001
From: Maxime <maxime.beauchemin@airbnb.com>
Date: Thu, 23 Jul 2015 05:20:10 +0000
Subject: [PATCH 1/2] New Charts

---
 TODO.md          |   5 +-
 app.db           | Bin 78848 -> 79872 bytes
 app/highchart.py | 156 +++++++++++++++++++++++++++++++++++++++++++++++
 app/viz.py       |  32 +++++-----
 requirements.txt |   1 -
 5 files changed, 176 insertions(+), 18 deletions(-)
 create mode 100644 app/highchart.py

diff --git a/TODO.md b/TODO.md
index f6a28803fd803..6bbbfc32a7f9e 100644
--- a/TODO.md
+++ b/TODO.md
@@ -1,5 +1,8 @@
 # TODO
+* in/notin filters autocomplete
+* Highstock, sort legend based on y value: ![stackoverflow](http://stackoverflow.com/questions/6867607/want-to-sort-highcharts-tooltip-results)
 * compare time ranges
-* Label
+* Add verbose_name and label method to metrics and columns
 * CSV
 * Save / bookmark / url shortener
+* on save, process metadata / generate metrics
diff --git a/app.db b/app.db
index f0bf7ecb2de2cd533b5beba7407a282c854ff14d..5c4db1afcfce9dbcc9c4894efe7b7036c4fd6565 100644
GIT binary patch
delta 498
zcmY*TK}!Nb7~ORZElBWCmk8!i8zH2w6(kJerJ%qLUAj%<?pQ-RGuzB4*(D>o7vewY
z4}?L7euRIaJax>Amk1gxB~Rb`zW2WOUh5;?`i$Q%&GaqHsyu*_-G4fH+)Fvmz#OcK
z^W_YkK{uW2JF)UYcX*I`c4lqRN!Y!(;rH9=`0fj^?Kf|3-t2ZPVZ{>Nbn@Fy-8L3e
z<+8P$Ox~343Y~X*{kl-}c8dAUjqTiaZZlu=H9C)|8c*O0t}*2bV8F1J^b&{~C{tPG
z)ini}P})cs6C5yNXkP&XMO{u&0cC_MD!AInMwknU8NJK~SYvf$v_8p>^RlG|y+XPf
z5g1ZV+&@#O>I@-<8aS*I_f%qD53!`W3G<l%H(f=!PhcTqdgwFN7QK~2L%JblR6}pe
zH5DT)33R0hjqmAi0wb7DE)kJb1PZ#fP$OT2C<je#R3(8Ff~odvZ}s@M4dR%QkxWD=
c4=EpEa$&I13vq4&)WpV{vKha5|K2?K12L<whyVZp

delta 102
zcmV-s0Ga=Q@C1P91dtmA;Q#;tO_3l!0pYP=q(=+_1NHy_002P)`vdl~5isckv-wL4
zv;ztN)|CeV&kINv3_(Ev9S8^j0GAOW0T`DpHUSZrT`B<x4mUI~IW{?FV{~6@aA|Y5
IAOQj3GEt5ldH?_b

diff --git a/app/highchart.py b/app/highchart.py
new file mode 100644
index 0000000000000..a872bb9274ff5
--- /dev/null
+++ b/app/highchart.py
@@ -0,0 +1,156 @@
+import pandas
+import copy
+from pandas.io.json import dumps
+
+
+class Highchart(object):
+    def __init__(
+            self, df,
+            chart_type="spline",
+            target_div="#chart",
+            polar=False,
+            width=None,
+            height=None,
+            show_legend=True,
+            stockchart=True,
+            title=None,
+            tooltip=None,
+            sort_columns=False,
+            secondary_y=None,
+            mark_right=False,
+            compare=False,
+            stacked=False,
+            logx=False,
+            logy=False,
+            xlim=None,
+            ylim=None,
+            grid=False,
+            zoom=None):
+        self.df = df
+        self.chart_type = chart_type
+        self.chart = chart = {}
+        self.stockchart = stockchart
+        self.sort_columns = sort_columns
+        self.secondary_y = secondary_y or []
+        self.mark_right = mark_right
+        self.compare = compare
+        self.logx = logx
+        self.logy = logy
+        self.xlim = xlim
+        self.ylim = ylim
+        self.zoom = zoom
+        self.polar = polar
+        self.grid = grid
+
+        chart['chart'] = {}
+        chart['chart']["type"] = chart_type
+        chart['chart']['renderTo'] = target_div
+
+        if width:
+            chart['chart']["width"] = width
+        if height:
+            chart['chart']["height"] = height
+
+        chart['chart']['polar'] = polar
+
+        chart["legend"] = {
+            "enabled": show_legend
+        }
+        if title:
+            chart["title"] = {"text": title}
+        if tooltip:
+            chart['tooltip'] = tooltip
+        if self.zoom:
+            chart["zoomType"] = self.zoom
+
+        self.serialize_series()
+        self.serialize_xaxis()
+        self.serialize_yaxis()
+
+        self.chart = chart
+
+    def serialize_series(self):
+        df = self.df
+        chart = self.chart
+        if self.sort_columns:
+            df = df.sort_index()
+        series = df.to_dict('series')
+        chart["series"] = []
+        for name, data in series.items():
+            if df[name].dtype.kind not in "biufc":
+                continue
+            sec = name in self.secondary_y
+            d = {
+                "name": name if not sec or self.mark_right else name + " (right)",
+                "yAxis": int(sec),
+                "data": zip(df.index, data.tolist())
+            }
+            if self.polar:
+                d['data'] = [v for k, v in d['data']]
+            if self.compare:
+                d['compare'] = self.compare  # either `value` or `percent`
+            if self.chart_type in ("area", "bar") and self.stacked:
+                d["stacking"] = 'normal'
+            #if kwargs.get("style"):
+            #    d["dashStyle"] = pd2hc_linestyle(kwargs["style"].get(name, "-"))
+            chart["series"].append(d)
+
+    def serialize_xaxis(self):
+        df = self.df
+        x_axis = {}
+        if df.index.name:
+            x_axis["title"] = {"text": df.index.name}
+        if df.index.dtype.kind in "M":
+            x_axis["type"] = "datetime"
+        if df.index.dtype.kind == 'O':
+            chart['xAxis']['categories'] = sorted(list(df.index)) if self.sort_columns else list(df.index)
+        if self.grid:
+            x_axis["gridLineWidth"] = 1
+            x_axis["gridLineDashStyle"] = "Dot"
+        if self.logx:
+            x_axis["type"] = 'logarithmic'
+        if self.xlim:
+            x_axis["min"] = self.xlim[0]
+            x_axis["max"] = self.xlim[1]
+        '''
+        if "rot" in kwargs:
+            x_axis["labels"] = {"rotation": kwargs["rot"]}
+        if "fontsize" in kwargs:
+            x_axis.setdefault("labels", {})["style"] = {"fontSize": kwargs["fontsize"]}
+        if "xticks" in kwargs:
+            x_axis["tickPositions"] = kwargs["xticks"]
+        '''
+        self.x_axis = x_axis
+
+    def serialize_yaxis(self):
+        yAxis = {}
+        chart = self.chart
+        if self.grid:
+            yAxis["gridLineWidth"] = 1
+            yAxis["gridLineDashStyle"] = "Dot"
+        if self.logy:
+            yAxis["type"] = 'logarithmic'
+        if self.ylim:
+            yAxis["min"] = self.ylim[0]
+            yAxis["max"] = self.ylim[1]
+        '''
+        if "rot" in kwargs:
+            yAxis["labels"] = {"rotation": kwargs["rot"]}
+        if "fontsize" in kwargs:
+            yAxis.setdefault("labels", {})["style"] = {"fontSize": kwargs["fontsize"]}
+        if "yticks" in kwargs:
+            yAxis["tickPositions"] = kwargs["yticks"]
+        '''
+        chart["yAxis"] = [yAxis]
+        if self.secondary_y:
+            yAxis2 = copy.deepcopy(yAxis)
+            yAxis2["opposite"] = True
+            chart["yAxis"].append(yAxis2)
+
+
+    @property
+    def javascript_cmd(self):
+        js = dumps(self.chart)
+        if self.stockchart:
+            return "new Highcharts.StockChart({});".format(js)
+        return "new Highcharts.Chart({});".format(js)
diff --git a/app/viz.py b/app/viz.py
index f529c2a73fc4e..433ec3c0c9203 100644
--- a/app/viz.py
+++ b/app/viz.py
@@ -1,19 +1,18 @@
 from pydruid.utils.filters import Dimension, Filter
 from datetime import datetime
-from flask import render_template, flash, request
+from flask import flash, request
 import pandas as pd
-from pandas_highcharts.core import serialize
-from pydruid.utils import aggregators as agg
 from collections import OrderedDict
 from app import utils
+from app.highchart import Highchart
 from wtforms import Form, SelectMultipleField, SelectField, TextField
 import config
 
 
 CHART_ARGS = {
-    'figsize': (None, 700),
+    'height': 700,
     'title': None,
-    'render_to': 'chart',
+    'target_div': 'chart',
 }
 
 class OmgWtForm(Form):
@@ -220,8 +219,8 @@ class HighchartsViz(BaseViz):
 
 class TimeSeriesViz(HighchartsViz):
     verbose_name = "Time Series - Line Chart"
-    chart_kind = "spline"
-    chart_type = 'stock'
+    chart_type = "spline"
+    highstock = True
 
     def render(self):
         metrics = self.metrics
@@ -237,12 +236,13 @@ def render(self):
             if rolling_type == 'mean':
                 df = pd.rolling_mean(df, int(rolling_periods))
 
-        chart_js = serialize(
-            df, kind=self.chart_kind,
-            viz=self,
+        chart = Highchart(
+            df,
             compare=self.compare,
-            chart_type=self.chart_type, stacked=self.stacked, **CHART_ARGS)
-        return super(TimeSeriesViz, self).render(chart_js=chart_js)
+            chart_type=self.chart_type,
+            stacked=self.stacked,
+            **CHART_ARGS)
+        return super(TimeSeriesViz, self).render(chart_js=chart.javascript_cmd)
 
     def form_class(self):
         return form_factory(self.datasource, request.args,
@@ -297,23 +297,23 @@ class TimeSeriesCompareViz(TimeSeriesViz):
 class TimeSeriesAreaViz(TimeSeriesViz):
     verbose_name = "Time Series - Stacked Area Chart"
     stacked=True
-    chart_kind = "area"
+    chart_type = "area"
 
 
 class TimeSeriesBarViz(TimeSeriesViz):
     verbose_name = "Time Series - Bar Chart"
-    chart_kind = "bar"
+    chart_type = "bar"
 
 
 class TimeSeriesStackedBarViz(TimeSeriesViz):
     verbose_name = "Time Series - Stacked Bar Chart"
-    chart_kind = "bar"
+    chart_type = "bar"
     stacked = True
 
 
 class DistributionBarViz(HighchartsViz):
     verbose_name = "Distribution - Bar Chart"
-    chart_kind = "bar"
+    chart_type = "bar"
 
     def query_obj(self):
         d = super(DistributionBarViz, self).query_obj()
diff --git a/requirements.txt b/requirements.txt
index 40c6da8e83af0..e629f4fbbc02c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,6 @@ flask
 flask-alembic
 flask-appbuilder
 pandas
-pandas-highcharts
 parsedatetime
 pydruid
 python-dateutil

From d438520660cf16dd4376c20a6660c8e47b2331fc Mon Sep 17 00:00:00 2001
From: Maxime <maxime.beauchemin@airbnb.com>
Date: Thu, 23 Jul 2015 06:11:51 +0000
Subject: [PATCH 2/2] Implementing my own highcharts wrapper

---
 app/highchart.py                            | 18 ++++++++-------
 app/templates/panoramix/viz_highcharts.html |  2 +-
 app/viz.py                                  | 25 ++++++++++++---------
 3 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/app/highchart.py b/app/highchart.py
index a872bb9274ff5..00df150106d49 100644
--- a/app/highchart.py
+++ b/app/highchart.py
@@ -12,7 +12,7 @@ def __init__(
             width=None,
             height=None,
             show_legend=True,
-            stockchart=True,
+            stockchart=False,
             title=None,
             tooltip=None,
             sort_columns=False,
@@ -41,6 +41,7 @@ def __init__(
         self.zoom = zoom
         self.polar = polar
         self.grid = grid
+        self.stacked = stacked
 
         chart['chart'] = {}
         chart['chart']["type"] = chart_type
@@ -56,8 +57,8 @@ def __init__(
         chart["legend"] = {
             "enabled": show_legend
         }
-        if title:
-            chart["title"] = {"text": title}
+        chart["title"] = {"text": title}
+
         if tooltip:
             chart['tooltip'] = tooltip
         if self.zoom:
@@ -89,7 +90,7 @@ def serialize_series(self):
                 d['data'] = [v for k, v in d['data']]
             if self.compare:
                 d['compare'] = self.compare  # either `value` or `percent`
-            if self.chart_type in ("area", "bar") and self.stacked:
+            if self.chart_type in ("area", "column", "bar") and self.stacked:
                 d["stacking"] = 'normal'
             #if kwargs.get("style"):
             #    d["dashStyle"] = pd2hc_linestyle(kwargs["style"].get(name, "-"))
@@ -103,7 +104,8 @@ def serialize_xaxis(self):
         if df.index.dtype.kind in "M":
             x_axis["type"] = "datetime"
         if df.index.dtype.kind == 'O':
-            chart['xAxis']['categories'] = sorted(list(df.index)) if self.sort_columns else list(df.index)
+            x_axis['categories'] = sorted(list(df.index)) if self.sort_columns else list(df.index)
+            print list(df.index)
         if self.grid:
             x_axis["gridLineWidth"] = 1
             x_axis["gridLineDashStyle"] = "Dot"
@@ -120,7 +122,7 @@ def serialize_xaxis(self):
         if "xticks" in kwargs:
             x_axis["tickPositions"] = kwargs["xticks"]
         '''
-        self.x_axis = x_axis
+        self.chart['xAxis'] = x_axis
 
     def serialize_yaxis(self):
         yAxis = {}
@@ -152,5 +154,5 @@ def serialize_yaxis(self):
     def javascript_cmd(self):
         js = dumps(self.chart)
         if self.stockchart:
-            return "new Highcharts.StockChart({});".format(js)
-        return "new Highcharts.Chart({});".format(js)
+            return "new Highcharts.StockChart(%s);" % js
+        return "new Highcharts.Chart(%s);" %js
diff --git a/app/templates/panoramix/viz_highcharts.html b/app/templates/panoramix/viz_highcharts.html
index c3940726b9e2c..efb4d19276a79 100644
--- a/app/templates/panoramix/viz_highcharts.html
+++ b/app/templates/panoramix/viz_highcharts.html
@@ -17,7 +17,7 @@
 
 {% block tail %}
 {{ super() }}
-{% if viz.chart_type == "stock" %}
+{% if viz.stockchart %}
 <script src="{{ url_for("static", filename="highstock.js") }}"></script>
 {% else %}
 <script src="{{ url_for("static", filename="highcharts.js") }}"></script>
diff --git a/app/viz.py b/app/viz.py
index 433ec3c0c9203..4cc1ccb0a1fcd 100644
--- a/app/viz.py
+++ b/app/viz.py
@@ -220,7 +220,7 @@ class HighchartsViz(BaseViz):
 class TimeSeriesViz(HighchartsViz):
     verbose_name = "Time Series - Line Chart"
     chart_type = "spline"
-    highstock = True
+    stockchart = True
 
     def render(self):
         metrics = self.metrics
@@ -241,6 +241,7 @@ def render(self):
             compare=self.compare,
             chart_type=self.chart_type,
             stacked=self.stacked,
+            stockchart=self.stockchart,
             **CHART_ARGS)
         return super(TimeSeriesViz, self).render(chart_js=chart.javascript_cmd)
 
@@ -302,18 +303,18 @@ class TimeSeriesAreaViz(TimeSeriesViz):
 
 class TimeSeriesBarViz(TimeSeriesViz):
     verbose_name = "Time Series - Bar Chart"
-    chart_type = "bar"
+    chart_type = "column"
 
 
 class TimeSeriesStackedBarViz(TimeSeriesViz):
     verbose_name = "Time Series - Stacked Bar Chart"
-    chart_type = "bar"
+    chart_type = "column"
     stacked = True
 
 
 class DistributionBarViz(HighchartsViz):
     verbose_name = "Distribution - Bar Chart"
-    chart_type = "bar"
+    chart_type = "column"
 
     def query_obj(self):
         d = super(DistributionBarViz, self).query_obj()
@@ -326,14 +327,15 @@ def render(self):
             index=self.groupby,
             values=self.metrics)
         df = df.sort(self.metrics[0], ascending=False)
-        chart_js = serialize(
-            df, kind=self.chart_kind, **CHART_ARGS)
-        return super(DistributionBarViz, self).render(chart_js=chart_js)
+        chart = Highchart(
+            df, chart_type=self.chart_type, **CHART_ARGS)
+        return super(DistributionBarViz, self).render(
+            chart_js=chart.javascript_cmd)
 
 
 class DistributionPieViz(HighchartsViz):
     verbose_name = "Distribution - Pie Chart"
-    chart_kind = "pie"
+    chart_type = "pie"
 
     def query_obj(self):
         d = super(DistributionPieViz, self).query_obj()
@@ -346,9 +348,10 @@ def render(self):
             index=self.groupby,
             values=[self.metrics[0]])
         df = df.sort(self.metrics[0], ascending=False)
-        chart_js = serialize(
-            df, kind=self.chart_kind, **CHART_ARGS)
-        return super(DistributionPieViz, self).render(chart_js=chart_js)
+        chart = Highchart(
+            df, chart_type=self.chart_type, **CHART_ARGS)
+        return super(DistributionPieViz, self).render(
+            chart_js=chart.javascript_cmd)
 
 viz_types = OrderedDict([
     ['table', TableViz],