Merge pull request apache#1 in DATA-PLATFORM/caravel from annotations…

… to dev

* commit 'e43b9ae513a0576ada62c88e1eadba503097fe85':
  Annotation support in timeseries bar chart

caravel/assets/javascripts/explore/explore.jsx

@@ -336,6 +336,51 @@ function initExploreView() {
   prepSaveDialog();
 }
 
+function getAnnotationFilters(annotationSource) {
+  let sqlaTableId = annotationSource;
+  if (!sqlaTableId) {
+    sqlaTableId = px.getParam('annotation_source');
+    if (!sqlaTableId || sqlaTableId === 'None') {
+      return;
+    }
+  }
+
+  const annotationFilterSelect = $('#annotation_filter');
+  const url = $(location).attr('protocol') + '//' +
+    $(location).attr('host') + '/caravel/annotations/' + sqlaTableId;
+
+  $.ajax({
+    method: 'GET',
+    url,
+    dataType: 'json',
+    contentType: 'application/json; charset=utf-8',
+  }).done(function (data) {
+    $(annotationFilterSelect).select2('destroy');
+    $(annotationFilterSelect).empty();
+
+    for (const key in data) {
+      const annotationText = data[key];
+      annotationFilterSelect.append($('<option></option>')
+        .attr('value', annotationText)
+        .html(annotationText));
+    }
+
+    $(annotationFilterSelect).select2();
+  }).fail(function () {
+    $(annotationFilterSelect).select2('destroy');
+    $(annotationFilterSelect).empty();
+    $(annotationFilterSelect).select2();
+  });
+}
+
+function initAnnotationForm() {
+  const annotationSource = $('#annotation_source');
+  annotationSource.change(function () {
+    getAnnotationFilters(annotationSource.val());
+  });
+  getAnnotationFilters(null);
+}
+
 function initComponents() {
   const queryAndSaveBtnsEl = document.getElementById('js-query-and-save-btns');
   ReactDOM.render(
@@ -354,6 +399,8 @@ function initComponents() {
     />,
     exploreActionsEl
   );
+
+  initAnnotationForm();
 }
 
 $(document).ready(function () {

caravel/assets/package.json

@@ -47,6 +47,7 @@
     "classnames": "^2.2.5",
     "d3": "^3.5.14",
     "d3-cloud": "^1.2.1",
+    "d3-format": "^1.0.2",
     "d3-sankey": "^0.2.1",
     "d3-tip": "^0.6.7",
     "datamaps": "^0.4.4",

caravel/assets/visualizations/nvd3_vis.css

@@ -31,3 +31,28 @@ text.nv-axislabel {
 .bar svg.nvd3-svg {
   width: auto;
 }
+
+div.annotation_tooltip {
+  position: absolute;
+  min-width: 100px;
+  max-width: 200px;
+  min-height: 28px;
+  padding: 7px;
+  font: 11px sans-serif;
+  font-weight: bold;
+  background: white;
+  border: 1px;
+  border-color: black;
+  border-style: solid;
+  border-radius: 4px;
+  pointer-events: none;
+}
+
+span.annotation_title {
+  font: 11px sans-serif;
+  font-weight: bold;
+}
+
+span.annotation_description {
+  font: 11px sans-serif;
+}

caravel/assets/visualizations/nvd3_vis.js

@@ -11,8 +11,8 @@ require('./nvd3_vis.css');
 const minBarWidth = 15;
 const animationTime = 1000;
 
-const addTotalBarValues = function (chart, data, stacked) {
-  const svg = d3.select('svg');
+const addTotalBarValues = function (containerId, chart, data, stacked) {
+  const svg = d3.select('#' + containerId + ' svg');
   const format = d3.format('.3s');
   const countSeriesDisplayed = data.length;
 
@@ -53,6 +53,158 @@ const addTotalBarValues = function (chart, data, stacked) {
     });
 };
 
+function strToRGB(str) {
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    hash = str.charCodeAt(i) + ((hash << 5) - hash);
+  }
+
+  const c = (hash & 0x00FFFFFF)
+    .toString(16)
+    .toUpperCase();
+
+  return '0000'.substring(0, 6 - c.length) + c;
+}
+
+const addBarAnnotations = function (containerId, chart, data, numberFormat) {
+  const svg = d3.select('#' + containerId + ' svg');
+  const targetAnnotations = svg.select('g.nv-barsWrap').append('g');
+
+
+  const barWidth = parseFloat(d3.select('#' + containerId + ' g.nv-group rect')
+    .attr('width'));
+
+  const div = d3.select('body')
+    .append('div')
+    .attr('class', 'annotation_tooltip')
+    .style('opacity', 0);
+
+  // Map of "timestamp-value" -> "text"
+  // to keep track of overlapping annotations
+  const annotationTitleValues = {};
+
+  data.forEach(
+    function (annotation) {
+      const key = annotation.timestamp + '-' + annotation.value;
+      if (key in annotationTitleValues) {
+        annotationTitleValues[key] = annotationTitleValues[key]
+          + '<br/><br/>'
+          + '<span class="annotation_title">'
+          + annotation.title
+          + '</span>';
+      } else {
+        annotationTitleValues[key] = '<span class="annotation_title">'
+          + annotation.title
+          + '</span>';
+      }
+
+      if (annotation.description) {
+        annotationTitleValues[key] = annotationTitleValues[key]
+          +'<br/><span class="annotation_description">'
+          + annotation.description
+          + '</span>';
+      }
+
+      const annotationColor = strToRGB(annotationTitleValues[key]);
+      const xAxisPosition = chart.xAxis.scale()(annotation.timestamp);
+      if (isNaN(xAxisPosition)) {
+        return;
+      }
+
+      targetAnnotations.append('svg:rect')
+        .attr('x', xAxisPosition + barWidth * 0.1)
+        .attr('y', chart.yAxis.scale()(
+          annotation.value) - 1.5)
+        .attr('width', barWidth * 0.8)
+        .attr('height', 3)
+        .style('fill', annotationColor)
+        .style('stroke', annotationColor)
+        .on('mouseover', function () {
+          d3.event.stopPropagation();
+          const rect = d3.select(this);
+          rect.style('fill', annotationColor);
+          rect.style('stroke', annotationColor);
+          rect.attr('opacity', 0.5);
+          div.transition()
+            .duration(200)
+            .style('opacity', 0.8);
+          div.html('<span>' +
+                    annotationTitleValues[key] +
+                  '</span><br/><br/>' +
+                  '<span style="font-weight:normal;">' +
+                      formatDate(annotation.timestamp) +
+                  '</span><br/>' +
+                  '<span>' +
+                    'Value: ' +
+                      d3.format(numberFormat)(annotation.value) +
+                  '</span>')
+            .style('left', (d3.event.pageX) + 25 + 'px')
+            .style('top', (d3.event.pageY - 30) + 'px');
+        })
+        .on('mouseout', function () {
+          d3.event.stopPropagation();
+          const rect = d3.select(this);
+          rect.style('fill', annotationColor);
+          rect.style('stroke', annotationColor);
+          rect.attr('opacity', 1);
+          div.transition()
+            .duration(200)
+            .style('opacity', 0);
+        });
+    }
+  );
+};
+
+// const addVerticalLineAnnotations = function (chart, data) {
+//   const svg = d3.select('svg');
+//   svg.select('g.nv-linesWrap').append('g')
+//     .attr('class', 'vertical-lines');
+//
+//   let annotationData = [];
+//   let numAnnotations = Object.keys(data['annotation_ts']).length;
+//   for (let i=0; i < numAnnotations; i++) {
+//     annotationData.push({
+//       'date': data['annotation_ts'][i],
+//       'label': data['annotation_val'][i]
+//     })
+//   }
+//
+//   const vertLines = d3.select('.vertical-lines')
+//     .selectAll('.vertical-line').data(annotationData);
+//
+//   var vertG = vertLines.enter()
+//         .append('g')
+//         .attr('class', 'vertical-line');
+//
+//   vertG.append('svg:line');
+//   vertG.append('text');
+//
+//   vertLines.exit().remove();
+//
+//   vertLines.selectAll('line')
+//     .attr('x1', function (d) {
+//        return chart.xAxis.scale()(d.date);
+//     })
+//     .attr('x2', function (d) {
+//       return chart.xAxis.scale()(d.date);
+//     })
+//     .attr('y1', chart.yAxis.scale().range()[0] )
+//     .attr('y2', chart.yAxis.scale().range()[1] )
+//     .style('stroke', 'blue');
+//
+//   vertLines.selectAll('text')
+//     .text( function(d) { return d.label })
+//     .attr('dy', '1em')
+//     .attr('transform', function (d) {
+//        return  'translate(' +
+//         chart.xAxis.scale()(d.date) +
+//         ',' +
+//         chart.yAxis.scale()(2) +
+//         ') rotate(-90)'
+//     })
+//     .style('font-size','80%')
+// };
+
 function nvd3Vis(slice) {
   let chart;
   let colorKey = 'key';
@@ -122,6 +274,12 @@ function nvd3Vis(slice) {
             chart.xAxis
             .showMaxMin(fd.x_axis_showminmax)
             .staggerLabels(false);
+
+            // if (fd.enable_annotations) {
+            //   setTimeout(function () {
+            //     addVerticalLineAnnotations(chart, payload.annotations);
+            //   }, animationTime);
+            // }
             break;
 
           case 'bar':
@@ -140,9 +298,39 @@ function nvd3Vis(slice) {
             stacked = fd.bar_stacked;
             chart.stacked(stacked);
 
+            if (fd.enable_annotations) {
+              const chartData = payload.data[0].values;
+              const latestDataDate = chartData[chartData.length - 1].x;
+
+              const dateValues = {};
+              chartData.forEach(function (barData) {
+                dateValues[barData.x] = true;
+              });
+
+              let yMax = 0;
+              payload.annotations.forEach(function (annotation) {
+                const annotationTimestamp = annotation.timestamp;
+                if (!(annotationTimestamp in dateValues)) {
+                  if (annotationTimestamp > latestDataDate) {
+                    chartData.push({ x: annotationTimestamp, y: 0 });
+                  }
+                }
+
+                yMax = yMax > annotation.value ?
+                  yMax : annotation.value;
+              });
+              chart.forceY([0, yMax]);
+
+              setTimeout(function () {
+                addBarAnnotations(slice.containerId,
+                  chart, payload.annotations, fd.y_axis_format);
+              }, animationTime);
+            }
+
             if (fd.show_bar_value) {
               setTimeout(function () {
-                addTotalBarValues(chart, payload.data, stacked);
+                addTotalBarValues(slice.containerId,
+                  chart, payload.data, stacked);
               }, animationTime);
             }
             break;

caravel/forms.py

@@ -15,7 +15,7 @@
     BooleanField, IntegerField, HiddenField, DecimalField)
 from wtforms import validators, widgets
 
-from caravel import app
+from caravel import app, db
 
 config = app.config
 
@@ -961,6 +961,24 @@ def __init__(self, viz):
                 ],
                 "description": _("The color for points and clusters in RGB")
             }),
+            'enable_annotations': (BetterBooleanField, {
+                "label": _("Enable Annotations"),
+                "default": False,
+                "description": _("Enable annotations on this graph. Must choose "
+                                 "a source for annotation data.")
+            }),
+            'annotation_source': (SelectField, {
+                "label": _("Annotation Source"),
+                "choices": self.get_annotation_source_choices(),
+                "description": _("Source of annotation data"),
+            }),
+            'annotation_filter': (SelectMultipleSortableField, {
+                "label": _("Annotation Filter"),
+                "choices": [(text, text) for text
+                            in viz.get_annotation_filter_choices(
+                                viz.orig_form_data.get('annotation_source')
+                            )]
+            }),
         }
 
         # Override default arguments with form overrides
@@ -973,6 +991,17 @@ def __init__(self, viz):
             for field_name, v in field_data.items()
         }
 
+    @staticmethod
+    def get_annotation_source_choices():
+        from caravel import models
+
+        choices = [('None', '')]
+        choices.extend([(unicode(table.id), table.full_name)
+                       for table
+                       in db.session.query(models.SqlaTable)
+                       .filter_by(annotation=True)])
+        return choices
+
     @staticmethod
     def choicify(l):
         return [("{}".format(obj), "{}".format(obj)) for obj in l]

caravel/migrations/versions/16f55059a4d6_.py

@@ -0,0 +1,22 @@
+"""merge 319 and ef8
+
+Revision ID: 16f55059a4d6
+Revises: ('3196bd55582b', 'ef8843b41dac')
+Create Date: 2016-10-03 13:08:23.208002
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '16f55059a4d6'
+down_revision = ('3196bd55582b', 'ef8843b41dac')
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass