diff --git a/caravel/assets/javascripts/explore/explore.jsx b/caravel/assets/javascripts/explore/explore.jsx
index 79aff8fe3b902..aa72bfdd3aac1 100644
--- a/caravel/assets/javascripts/explore/explore.jsx
+++ b/caravel/assets/javascripts/explore/explore.jsx
@@ -336,6 +336,51 @@ function initExploreView() {
prepSaveDialog();
}
+function getAnnotationFilters(annotationSource) {
+ let sqlaTableId = annotationSource;
+ if (!sqlaTableId) {
+ sqlaTableId = px.getParam('annotation_source');
+ if (!sqlaTableId || sqlaTableId === 'None') {
+ return;
+ }
+ }
+
+ const annotationFilterSelect = $('#annotation_filter');
+ const url = $(location).attr('protocol') + '//' +
+ $(location).attr('host') + '/caravel/annotations/' + sqlaTableId;
+
+ $.ajax({
+ method: 'GET',
+ url,
+ dataType: 'json',
+ contentType: 'application/json; charset=utf-8',
+ }).done(function (data) {
+ $(annotationFilterSelect).select2('destroy');
+ $(annotationFilterSelect).empty();
+
+ for (const key in data) {
+ const annotationText = data[key];
+ annotationFilterSelect.append($('')
+ .attr('value', annotationText)
+ .html(annotationText));
+ }
+
+ $(annotationFilterSelect).select2();
+ }).fail(function () {
+ $(annotationFilterSelect).select2('destroy');
+ $(annotationFilterSelect).empty();
+ $(annotationFilterSelect).select2();
+ });
+}
+
+function initAnnotationForm() {
+ const annotationSource = $('#annotation_source');
+ annotationSource.change(function () {
+ getAnnotationFilters(annotationSource.val());
+ });
+ getAnnotationFilters(null);
+}
+
function initComponents() {
const queryAndSaveBtnsEl = document.getElementById('js-query-and-save-btns');
ReactDOM.render(
@@ -354,6 +399,8 @@ function initComponents() {
/>,
exploreActionsEl
);
+
+ initAnnotationForm();
}
$(document).ready(function () {
diff --git a/caravel/assets/package.json b/caravel/assets/package.json
index fa678959aa726..a5103b6e0b7a5 100644
--- a/caravel/assets/package.json
+++ b/caravel/assets/package.json
@@ -47,6 +47,7 @@
"classnames": "^2.2.5",
"d3": "^3.5.14",
"d3-cloud": "^1.2.1",
+ "d3-format": "^1.0.2",
"d3-sankey": "^0.2.1",
"d3-tip": "^0.6.7",
"datamaps": "^0.4.4",
diff --git a/caravel/assets/visualizations/nvd3_vis.css b/caravel/assets/visualizations/nvd3_vis.css
index 20dbd65a74ca6..d11ebf6893cf0 100644
--- a/caravel/assets/visualizations/nvd3_vis.css
+++ b/caravel/assets/visualizations/nvd3_vis.css
@@ -31,3 +31,28 @@ text.nv-axislabel {
.bar svg.nvd3-svg {
width: auto;
}
+
+div.annotation_tooltip {
+ position: absolute;
+ min-width: 100px;
+ max-width: 200px;
+ min-height: 28px;
+ padding: 7px;
+ font: 11px sans-serif;
+ font-weight: bold;
+ background: white;
+ border: 1px;
+ border-color: black;
+ border-style: solid;
+ border-radius: 4px;
+ pointer-events: none;
+}
+
+span.annotation_title {
+ font: 11px sans-serif;
+ font-weight: bold;
+}
+
+span.annotation_description {
+ font: 11px sans-serif;
+}
diff --git a/caravel/assets/visualizations/nvd3_vis.js b/caravel/assets/visualizations/nvd3_vis.js
index 7d8dbfa25a548..f0b2acd6b2c83 100644
--- a/caravel/assets/visualizations/nvd3_vis.js
+++ b/caravel/assets/visualizations/nvd3_vis.js
@@ -11,8 +11,8 @@ require('./nvd3_vis.css');
const minBarWidth = 15;
const animationTime = 1000;
-const addTotalBarValues = function (chart, data, stacked) {
- const svg = d3.select('svg');
+const addTotalBarValues = function (containerId, chart, data, stacked) {
+ const svg = d3.select('#' + containerId + ' svg');
const format = d3.format('.3s');
const countSeriesDisplayed = data.length;
@@ -53,6 +53,158 @@ const addTotalBarValues = function (chart, data, stacked) {
});
};
+function strToRGB(str) {
+ let hash = 0;
+ for (let i = 0; i < str.length; i++) {
+ hash = str.charCodeAt(i) + ((hash << 5) - hash);
+ }
+
+ const c = (hash & 0x00FFFFFF)
+ .toString(16)
+ .toUpperCase();
+
+ return '0000'.substring(0, 6 - c.length) + c;
+}
+
+const addBarAnnotations = function (containerId, chart, data, numberFormat) {
+ const svg = d3.select('#' + containerId + ' svg');
+ const targetAnnotations = svg.select('g.nv-barsWrap').append('g');
+
+
+ const barWidth = parseFloat(d3.select('#' + containerId + ' g.nv-group rect')
+ .attr('width'));
+
+ const div = d3.select('body')
+ .append('div')
+ .attr('class', 'annotation_tooltip')
+ .style('opacity', 0);
+
+ // Map of "timestamp-value" -> "text"
+ // to keep track of overlapping annotations
+ const annotationTitleValues = {};
+
+ data.forEach(
+ function (annotation) {
+ const key = annotation.timestamp + '-' + annotation.value;
+ if (key in annotationTitleValues) {
+ annotationTitleValues[key] = annotationTitleValues[key]
+ + '
'
+ + ''
+ + annotation.title
+ + '';
+ } else {
+ annotationTitleValues[key] = ''
+ + annotation.title
+ + '';
+ }
+
+ if (annotation.description) {
+ annotationTitleValues[key] = annotationTitleValues[key]
+ +'
'
+ + annotation.description
+ + '';
+ }
+
+ const annotationColor = strToRGB(annotationTitleValues[key]);
+ const xAxisPosition = chart.xAxis.scale()(annotation.timestamp);
+ if (isNaN(xAxisPosition)) {
+ return;
+ }
+
+ targetAnnotations.append('svg:rect')
+ .attr('x', xAxisPosition + barWidth * 0.1)
+ .attr('y', chart.yAxis.scale()(
+ annotation.value) - 1.5)
+ .attr('width', barWidth * 0.8)
+ .attr('height', 3)
+ .style('fill', annotationColor)
+ .style('stroke', annotationColor)
+ .on('mouseover', function () {
+ d3.event.stopPropagation();
+ const rect = d3.select(this);
+ rect.style('fill', annotationColor);
+ rect.style('stroke', annotationColor);
+ rect.attr('opacity', 0.5);
+ div.transition()
+ .duration(200)
+ .style('opacity', 0.8);
+ div.html('' +
+ annotationTitleValues[key] +
+ '
' +
+ '' +
+ formatDate(annotation.timestamp) +
+ '
' +
+ '' +
+ 'Value: ' +
+ d3.format(numberFormat)(annotation.value) +
+ '')
+ .style('left', (d3.event.pageX) + 25 + 'px')
+ .style('top', (d3.event.pageY - 30) + 'px');
+ })
+ .on('mouseout', function () {
+ d3.event.stopPropagation();
+ const rect = d3.select(this);
+ rect.style('fill', annotationColor);
+ rect.style('stroke', annotationColor);
+ rect.attr('opacity', 1);
+ div.transition()
+ .duration(200)
+ .style('opacity', 0);
+ });
+ }
+ );
+};
+
+// const addVerticalLineAnnotations = function (chart, data) {
+// const svg = d3.select('svg');
+// svg.select('g.nv-linesWrap').append('g')
+// .attr('class', 'vertical-lines');
+//
+// let annotationData = [];
+// let numAnnotations = Object.keys(data['annotation_ts']).length;
+// for (let i=0; i < numAnnotations; i++) {
+// annotationData.push({
+// 'date': data['annotation_ts'][i],
+// 'label': data['annotation_val'][i]
+// })
+// }
+//
+// const vertLines = d3.select('.vertical-lines')
+// .selectAll('.vertical-line').data(annotationData);
+//
+// var vertG = vertLines.enter()
+// .append('g')
+// .attr('class', 'vertical-line');
+//
+// vertG.append('svg:line');
+// vertG.append('text');
+//
+// vertLines.exit().remove();
+//
+// vertLines.selectAll('line')
+// .attr('x1', function (d) {
+// return chart.xAxis.scale()(d.date);
+// })
+// .attr('x2', function (d) {
+// return chart.xAxis.scale()(d.date);
+// })
+// .attr('y1', chart.yAxis.scale().range()[0] )
+// .attr('y2', chart.yAxis.scale().range()[1] )
+// .style('stroke', 'blue');
+//
+// vertLines.selectAll('text')
+// .text( function(d) { return d.label })
+// .attr('dy', '1em')
+// .attr('transform', function (d) {
+// return 'translate(' +
+// chart.xAxis.scale()(d.date) +
+// ',' +
+// chart.yAxis.scale()(2) +
+// ') rotate(-90)'
+// })
+// .style('font-size','80%')
+// };
+
function nvd3Vis(slice) {
let chart;
let colorKey = 'key';
@@ -122,6 +274,12 @@ function nvd3Vis(slice) {
chart.xAxis
.showMaxMin(fd.x_axis_showminmax)
.staggerLabels(false);
+
+ // if (fd.enable_annotations) {
+ // setTimeout(function () {
+ // addVerticalLineAnnotations(chart, payload.annotations);
+ // }, animationTime);
+ // }
break;
case 'bar':
@@ -140,9 +298,39 @@ function nvd3Vis(slice) {
stacked = fd.bar_stacked;
chart.stacked(stacked);
+ if (fd.enable_annotations) {
+ const chartData = payload.data[0].values;
+ const latestDataDate = chartData[chartData.length - 1].x;
+
+ const dateValues = {};
+ chartData.forEach(function (barData) {
+ dateValues[barData.x] = true;
+ });
+
+ let yMax = 0;
+ payload.annotations.forEach(function (annotation) {
+ const annotationTimestamp = annotation.timestamp;
+ if (!(annotationTimestamp in dateValues)) {
+ if (annotationTimestamp > latestDataDate) {
+ chartData.push({ x: annotationTimestamp, y: 0 });
+ }
+ }
+
+ yMax = yMax > annotation.value ?
+ yMax : annotation.value;
+ });
+ chart.forceY([0, yMax]);
+
+ setTimeout(function () {
+ addBarAnnotations(slice.containerId,
+ chart, payload.annotations, fd.y_axis_format);
+ }, animationTime);
+ }
+
if (fd.show_bar_value) {
setTimeout(function () {
- addTotalBarValues(chart, payload.data, stacked);
+ addTotalBarValues(slice.containerId,
+ chart, payload.data, stacked);
}, animationTime);
}
break;
diff --git a/caravel/forms.py b/caravel/forms.py
index 638d9f5344176..d53092daf72b7 100755
--- a/caravel/forms.py
+++ b/caravel/forms.py
@@ -15,7 +15,7 @@
BooleanField, IntegerField, HiddenField, DecimalField)
from wtforms import validators, widgets
-from caravel import app
+from caravel import app, db
config = app.config
@@ -961,6 +961,24 @@ def __init__(self, viz):
],
"description": _("The color for points and clusters in RGB")
}),
+ 'enable_annotations': (BetterBooleanField, {
+ "label": _("Enable Annotations"),
+ "default": False,
+ "description": _("Enable annotations on this graph. Must choose "
+ "a source for annotation data.")
+ }),
+ 'annotation_source': (SelectField, {
+ "label": _("Annotation Source"),
+ "choices": self.get_annotation_source_choices(),
+ "description": _("Source of annotation data"),
+ }),
+ 'annotation_filter': (SelectMultipleSortableField, {
+ "label": _("Annotation Filter"),
+ "choices": [(text, text) for text
+ in viz.get_annotation_filter_choices(
+ viz.orig_form_data.get('annotation_source')
+ )]
+ }),
}
# Override default arguments with form overrides
@@ -973,6 +991,17 @@ def __init__(self, viz):
for field_name, v in field_data.items()
}
+ @staticmethod
+ def get_annotation_source_choices():
+ from caravel import models
+
+ choices = [('None', '')]
+ choices.extend([(unicode(table.id), table.full_name)
+ for table
+ in db.session.query(models.SqlaTable)
+ .filter_by(annotation=True)])
+ return choices
+
@staticmethod
def choicify(l):
return [("{}".format(obj), "{}".format(obj)) for obj in l]
diff --git a/caravel/migrations/versions/16f55059a4d6_.py b/caravel/migrations/versions/16f55059a4d6_.py
new file mode 100644
index 0000000000000..75c89f5e739a2
--- /dev/null
+++ b/caravel/migrations/versions/16f55059a4d6_.py
@@ -0,0 +1,22 @@
+"""merge 319 and ef8
+
+Revision ID: 16f55059a4d6
+Revises: ('3196bd55582b', 'ef8843b41dac')
+Create Date: 2016-10-03 13:08:23.208002
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '16f55059a4d6'
+down_revision = ('3196bd55582b', 'ef8843b41dac')
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ pass
+
+
+def downgrade():
+ pass
diff --git a/caravel/migrations/versions/3196bd55582b_annotations.py b/caravel/migrations/versions/3196bd55582b_annotations.py
new file mode 100644
index 0000000000000..90a65f2deee57
--- /dev/null
+++ b/caravel/migrations/versions/3196bd55582b_annotations.py
@@ -0,0 +1,30 @@
+"""Annotation Support
+
+Revision ID: 3196bd55582b
+Revises: 3b626e2a6783
+Create Date: 2016-09-28 17:08:46.950505
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '3196bd55582b'
+down_revision = '3b626e2a6783'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ op.add_column('tables', sa.Column('annotation', sa.Boolean(), server_default='0'))
+ op.add_column('table_columns', sa.Column('annotation_time', sa.Boolean(), server_default='0'))
+ op.add_column('table_columns', sa.Column('annotation_value', sa.Boolean(), server_default='0'))
+ op.add_column('table_columns', sa.Column('annotation_title', sa.Boolean(), server_default='0'))
+ op.add_column('table_columns', sa.Column('annotation_desc', sa.Boolean(), server_default='0'))
+
+
+def downgrade():
+ op.drop_column('tables', 'annotation')
+ op.drop_column('table_columns', 'annotation_time')
+ op.drop_column('table_columns', 'annotation_value')
+ op.drop_column('table_columns', 'annotation_title')
+ op.drop_column('table_columns', 'annotation_desc')
diff --git a/caravel/models.py b/caravel/models.py
index ec82efa53144d..fbf18704fb80c 100644
--- a/caravel/models.py
+++ b/caravel/models.py
@@ -677,6 +677,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
default_endpoint = Column(Text)
database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
is_featured = Column(Boolean, default=False)
+ annotation = Column(Boolean, default=False)
user_id = Column(Integer, ForeignKey('ab_user.id'))
owner = relationship('User', backref='tables', foreign_keys=[user_id])
database = relationship(
@@ -767,6 +768,44 @@ def get_col(self, col_name):
if col_name == col.column_name:
return col
+ def values_for_column(self,
+ column_name,
+ from_dttm=None,
+ to_dttm=None,
+ limit=500):
+ """Runs query against sqla to retrieve some sample values for the given column."""
+ granularity = self.main_dttm_col
+
+ cols = {col.column_name: col for col in self.columns}
+ target_col = cols[column_name]
+
+ tbl = table(self.table_name)
+ qry = select([target_col.sqla_col])
+ qry = qry.select_from(tbl)
+ qry = qry.distinct(column_name)
+ qry = qry.limit(limit)
+
+ if granularity:
+ dttm_col = cols[granularity]
+ timestamp = dttm_col.sqla_col.label('timestamp')
+ time_filter = []
+ if from_dttm:
+ time_filter.append(timestamp >= text(dttm_col.dttm_sql_literal(from_dttm)))
+ if to_dttm:
+ time_filter.append(timestamp <= text(dttm_col.dttm_sql_literal(to_dttm)))
+ qry = qry.where(and_(*time_filter))
+
+ engine = self.database.get_sqla_engine()
+ sql = "{}".format(
+ qry.compile(
+ engine, compile_kwargs={"literal_binds": True}, ),
+ )
+
+ return pd.read_sql_query(
+ sql=sql,
+ con=engine
+ )
+
def query( # sqla
self, groupby, metrics,
granularity,
@@ -949,6 +988,7 @@ def _(element, compiler, **kw):
con=engine
)
sql = sqlparse.format(sql, reindent=True)
+
return QueryResult(
df=df, duration=datetime.now() - qry_start_dttm, query=sql)
@@ -1051,6 +1091,37 @@ def fetch_metadata(self):
if not self.main_dttm_col:
self.main_dttm_col = any_date_col
+ def get_annotations(self, from_dttm, to_dttm):
+ time_column = None
+ value_column = None
+ for column in self.table_columns:
+ if column.annotation_time:
+ time_column = column
+ if column.annotation_value:
+ value_column = column
+
+ if not time_column or not value_column:
+ return None
+
+ tbl = table(self.table_name)
+ select_exprs = [
+ time_column.sqla_col.label('annotation_ts'),
+ value_column.sqla_col.label('annotation_val')
+ ]
+ qry = select(select_exprs)
+ qry = qry.select_from(tbl)
+
+ engine = self.database.get_sqla_engine()
+ sql = "{}".format(
+ qry.compile(
+ engine, compile_kwargs={"literal_binds": True}, ),
+ )
+
+ return pd.read_sql_query(
+ sql=sql,
+ con=engine
+ )
+
class SqlMetric(Model, AuditMixinNullable):
@@ -1106,6 +1177,10 @@ class TableColumn(Model, AuditMixinNullable):
description = Column(Text, default='')
python_date_format = Column(String(255))
database_expression = Column(String(255))
+ annotation_time = Column(Boolean, default=False)
+ annotation_value = Column(Boolean, default=False)
+ annotation_title = Column(Boolean, default=False)
+ annotation_desc = Column(Boolean, default=False)
num_types = ('DOUBLE', 'FLOAT', 'INT', 'BIGINT', 'LONG')
date_types = ('DATE', 'TIME')
@@ -1271,7 +1346,8 @@ def perm(self):
@property
def link(self):
name = escape(self.datasource_name)
- return Markup('{name}').format(**locals())
+ url = self.url
+ return Markup('{name}').format(url=url, name=name)
@property
def full_name(self):
diff --git a/caravel/utils.py b/caravel/utils.py
index 168656986f0aa..b062ef5736562 100644
--- a/caravel/utils.py
+++ b/caravel/utils.py
@@ -20,6 +20,7 @@
from flask import flash, Markup
from flask_appbuilder.security.sqla import models as ab_models
from markdown import markdown as md
+from semantic.numbers import NumberService
from sqlalchemy.types import TypeDecorator, TEXT
from pydruid.utils.having import Having
@@ -167,6 +168,15 @@ def parse_human_datetime(s):
return dttm
+def parse_natural_language_number(n):
+ """
+ :param n: natural language number e.g. Two hundred and six
+ :return: double representation of the words w.g. 206
+ """
+ service = NumberService()
+ return service.parse(n)
+
+
def dttm_from_timtuple(d):
return datetime(
d.tm_year, d.tm_mon, d.tm_mday, d.tm_hour, d.tm_min, d.tm_sec)
diff --git a/caravel/views.py b/caravel/views.py
index baba51a9388c4..9d3f3866502f0 100755
--- a/caravel/views.py
+++ b/caravel/views.py
@@ -286,11 +286,13 @@ class TableColumnInlineView(CompactCRUDMixin, CaravelModelView): # noqa
edit_columns = [
'column_name', 'verbose_name', 'description', 'groupby', 'filterable',
'table', 'count_distinct', 'sum', 'min', 'max', 'expression',
- 'is_dttm', 'python_date_format', 'database_expression']
+ 'is_dttm', 'python_date_format', 'database_expression',
+ 'annotation_time', 'annotation_value']
add_columns = edit_columns
list_columns = [
'column_name', 'type', 'groupby', 'filterable', 'count_distinct',
- 'sum', 'min', 'max', 'is_dttm']
+ 'sum', 'min', 'max', 'is_dttm',
+ 'annotation_time', 'annotation_value', 'annotation_title', 'annotation_desc']
page_size = 500
description_columns = {
'is_dttm': (_(
@@ -333,7 +335,11 @@ class TableColumnInlineView(CompactCRUDMixin, CaravelModelView): # noqa
'expression': _("Expression"),
'is_dttm': _("Is temporal"),
'python_date_format': _("Datetime Format"),
- 'database_expression': _("Database Expression")
+ 'database_expression': _("Database Expression"),
+ 'annotation_time': _("Annotation Time"),
+ 'annotation_value': _("Annotation Value"),
+ 'annotation_title': _("Annotation Title"),
+ 'annotation_desc': _("Annotation Description"),
}
appbuilder.add_view_no_menu(TableColumnInlineView)
@@ -558,10 +564,10 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa
'changed_by_', 'changed_on_']
order_columns = [
'link', 'database', 'is_featured', 'changed_on_']
- add_columns = ['table_name', 'database', 'schema']
+ add_columns = ['table_name', 'database', 'schema', 'annotation']
edit_columns = [
- 'table_name', 'sql', 'is_featured', 'database', 'schema',
- 'description', 'owner',
+ 'table_name', 'sql', 'is_featured', 'annotation',
+ 'database', 'schema','description', 'owner',
'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout']
related_views = [TableColumnInlineView, SqlMetricInlineView]
base_order = ('changed_on', 'desc')
@@ -587,6 +593,7 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa
'database': _("Database"),
'changed_on_': _("Last Changed"),
'is_featured': _("Is Featured"),
+ 'annotation': _("Annotation"),
'schema': _("Schema"),
'default_endpoint': _("Default Endpoint"),
'offset': _("Offset"),
@@ -1942,6 +1949,31 @@ def sqlanvil(self):
"""SQL Editor"""
return self.render_template('caravel/sqllab.html')
+ @has_access_api
+ @expose("/annotations/")
+ def annotation_filter(self, table_id):
+ sqla_table = db.session.query(
+ models.SqlaTable).filter_by(id=table_id).first()
+
+ # Find title column
+ title_column = None
+ for column in sqla_table.table_columns:
+ if column.annotation_title:
+ title_column = column.column_name
+
+ if not title_column:
+ return Response(
+ json.dumps({'error': "No annotation text column found."}),
+ status=403,
+ mimetype="application/json")
+
+ df = sqla_table.values_for_column(column_name=title_column)
+ return Response(
+ df[title_column].to_json(),
+ status=200,
+ mimetype="application/json")
+
+
appbuilder.add_view_no_menu(Caravel)
if config['DRUID_IS_ACTIVE']:
diff --git a/caravel/viz.py b/caravel/viz.py
index 524a462acf479..92d09c8c2c1e1 100755
--- a/caravel/viz.py
+++ b/caravel/viz.py
@@ -11,6 +11,7 @@
import copy
import hashlib
import logging
+import re
import uuid
import zlib
@@ -28,9 +29,9 @@
from werkzeug.urls import Href
from dateutil import relativedelta as rdelta
-from caravel import app, utils, cache
+from caravel import app, db, utils, cache
from caravel.forms import FormFactory
-from caravel.utils import flasher
+from caravel.utils import flasher, parse_natural_language_number
config = app.config
@@ -288,6 +289,10 @@ def cache_timeout(self):
return config.get("CACHE_DEFAULT_TIMEOUT")
def get_json(self, force=False):
+ payload = self.get_payload(force)
+ return self.json_dumps(payload)
+
+ def get_payload(self, force=False):
"""Handles caching around the json payload retrieval"""
cache_key = self.cache_key
payload = None
@@ -339,7 +344,7 @@ def get_json(self, force=False):
logging.exception(e)
cache.delete(cache_key)
payload['is_cached'] = is_cached
- return self.json_dumps(payload)
+ return payload
def json_dumps(self, obj):
"""Used by get_json, can be overridden to use specific switches"""
@@ -349,7 +354,7 @@ def json_dumps(self, obj):
def data(self):
"""This is the data object serialized to the js layer"""
content = {
- 'csv_endpoint': self.csv_endpoint,
+ 'csv_end*point': self.csv_endpoint,
'form_data': self.form_data,
'json_endpoint': self.json_endpoint,
'standalone_endpoint': self.standalone_endpoint,
@@ -371,6 +376,9 @@ def get_csv(self):
def get_data(self):
return []
+ def get_annotation_filter_choices(self, annotation_source):
+ return []
+
@property
def json_endpoint(self):
return self.get_url(json="true")
@@ -1142,6 +1150,23 @@ def get_data(self):
chart_data = sorted(chart_data, key=lambda x: x['key'])
return chart_data
+ def get_annotations(self):
+ from caravel import models
+ datasource = (db.session.query(models.SqlaTable)
+ .filter_by(id=self.form_data.get('annotation_source'))
+ .first())
+ return datasource.get_annotations(None, None)
+
+ def get_json(self):
+ payload = super(NVD3TimeSeriesViz, self).get_payload()
+
+ if self.form_data.get('enable_annotations'):
+ annotations = self.get_annotations()
+ if annotations is not None:
+ payload['annotations'] = annotations.to_dict()
+
+ return self.json_dumps(payload)
+
class NVD3TimeSeriesBarViz(NVD3TimeSeriesViz):
@@ -1150,18 +1175,178 @@ class NVD3TimeSeriesBarViz(NVD3TimeSeriesViz):
viz_type = "bar"
sort_series = True
verbose_name = _("Time Series - Bar Chart")
- fieldsets = [NVD3TimeSeriesViz.fieldsets[0]] + [{
- 'label': _('Chart Options'),
- 'fields': (
- ('show_brush', 'show_legend', 'show_bar_value'),
- ('rich_tooltip', 'y_axis_zero'),
- ('y_log_scale', 'contribution'),
- ('x_axis_format', 'y_axis_format'),
- ('line_interpolation', 'bar_stacked'),
- ('x_axis_showminmax', 'bottom_margin'),
- ('x_axis_label', 'y_axis_label'),
- ('reduce_x_ticks', 'show_controls'),
- ), }] + [NVD3TimeSeriesViz.fieldsets[2]]
+ fieldsets = [NVD3TimeSeriesViz.fieldsets[0]] + [
+ {
+ 'label': _('Chart Options'),
+ 'fields': (
+ ('show_brush', 'show_legend', 'show_bar_value'),
+ ('rich_tooltip', 'y_axis_zero'),
+ ('y_log_scale', 'contribution'),
+ ('x_axis_showminmax', 'bar_stacked'),
+ ('reduce_x_ticks', 'show_controls'),
+ ('x_axis_format', 'y_axis_format'),
+ ('line_interpolation', 'bottom_margin'),
+ ('x_axis_label', 'y_axis_label'),
+ ),
+ }, {
+ 'label': _('Annotation Options'),
+ 'fields': (
+ ('enable_annotations'),
+ ('annotation_source'),
+ ('annotation_filter')
+ ),
+ }
+ ] + [NVD3TimeSeriesViz.fieldsets[2]]
+
+ TIMEGROUPER_MAPPING = {
+ 'second': 'S',
+ 'minute': 'T',
+ 'hour': 'H',
+ 'day': 'D',
+ 'week': 'W',
+ 'month': 'M',
+ 'year': 'A',
+ }
+
+ def _get_timegrouper_freq(self):
+ from caravel.models import DruidDatasource
+ if isinstance(self.datasource, DruidDatasource):
+ gran = self.form_data.get('granularity').strip().lower()
+ duration_val, duration_unit = gran.rstrip('s').split()
+ duration_val = parse_natural_language_number(duration_val)
+ freq = str(int(duration_val)) + self.TIMEGROUPER_MAPPING[duration_unit]
+
+ return freq
+
+ def get_bar_annotations(self):
+ annotation_source = self.form_data.get('annotation_source')
+ annotation_filters = self.form_data.get('annotation_filter')
+ if not annotation_source or str(annotation_source) == 'None':
+ raise Exception("Annotations are enabled but no "
+ "annotation source is selected.")
+
+ from caravel.models import SqlaTable
+ datasource = (db.session.query(SqlaTable)
+ .filter_by(id=self.form_data.get('annotation_source'))
+ .first())
+
+ if not datasource:
+ raise Exception("Annotation datasource does not exist.")
+
+ time_column, value_column = None, None
+ title_column, desc_column = None, None
+ for column in datasource.table_columns:
+ if column.annotation_time:
+ time_column = column.column_name
+ if column.annotation_value:
+ value_column = column.column_name
+ if column.annotation_title:
+ title_column = column.column_name
+ if column.annotation_desc:
+ desc_column = column.column_name
+
+ if time_column and value_column and title_column:
+ break
+
+ if not time_column or not value_column or not title_column:
+ raise Exception("Time, Text, and Value columns must "
+ "be selected in annotation table source.")
+
+ query_obj = self.query_obj()
+ query_obj['granularity'] = time_column
+ query_obj['groupby'] = None
+ query_obj['metrics'] = []
+ query_obj['filter'] = []
+ query_obj['columns'] = [value_column, title_column]
+
+ if desc_column:
+ query_obj['columns'].append(desc_column)
+
+ if annotation_filters:
+ cols = {col.column_name: col for col in datasource.columns}
+ in_clause = cols[title_column].sqla_col.in_(annotation_filters)
+ engine = datasource.database.get_sqla_engine()
+ query_obj['extras']['where'] = '{}'.format(in_clause.compile(
+ engine, compile_kwargs={"literal_binds": True}, ))
+
+ freq = self._get_timegrouper_freq() # Translated Granularity
+
+ annotations = datasource.query(**query_obj).df
+ if not annotations.empty:
+ annotations = annotations.set_index('timestamp')
+ grouby_columns = [pd.TimeGrouper(freq=freq), title_column]
+ if desc_column:
+ grouby_columns.append(desc_column)
+ annotations = annotations.groupby(grouby_columns).sum()
+ annotations = annotations.reset_index()
+ if desc_column:
+ annotations.columns = ['timestamp', 'title', 'description', 'value']
+ else:
+ annotations.columns = ['timestamp', 'title', 'value']
+
+ return annotations
+
+ def get_json(self):
+ payload = super(NVD3TimeSeriesBarViz, self).get_payload()
+
+ if self.form_data.get('enable_annotations'):
+ annotations = self.get_bar_annotations()
+ payload['annotations'] = annotations.to_dict(orient='records')
+
+ return self.json_dumps(payload)
+
+ def get_annotation_filter_choices(self, annotation_source):
+ """
+ Retrieves values for a column to be used by the filter dropdown.
+ :param annotation_source: SQLA Table ID
+ :return: JSON containing the some values for a column
+ """
+
+ form_data = self.orig_form_data
+
+ if (not annotation_source
+ or not form_data.get('enable_annotations')
+ or form_data.get('enable_annotations') == u'false'):
+ return []
+
+ from caravel.models import SqlaTable
+ datasource = (db.session.query(SqlaTable)
+ .filter_by(id=annotation_source)
+ .first())
+
+ if not datasource:
+ Exception("Annotations are Enabled. "
+ "Please select an Annotation Source.")
+
+ title_column = None
+ for column in datasource.table_columns:
+ if column.annotation_title:
+ title_column = column.column_name
+
+ if not title_column:
+ raise Exception("Please define a title column "
+ "in the selected annotations source.")
+
+ form_data = self.orig_form_data
+
+ since = form_data.get("since", "1 year ago")
+ from_dttm = utils.parse_human_datetime(since)
+ now = datetime.now()
+ if from_dttm > now:
+ from_dttm = now - (from_dttm - now)
+ until = form_data.get("until", "now")
+ to_dttm = utils.parse_human_datetime(until)
+ if from_dttm > to_dttm:
+ flasher("The date range doesn't seem right.", "danger")
+ from_dttm = to_dttm # Making them identical to not raise
+
+ kwargs = dict(
+ column_name=title_column,
+ from_dttm=from_dttm,
+ to_dttm=to_dttm,
+ )
+ df = datasource.values_for_column(**kwargs)
+ return df[title_column]
class NVD3CompareTimeSeriesViz(NVD3TimeSeriesViz):
diff --git a/setup.py b/setup.py
index 0a83a422a6def..2bb5c9924a18d 100644
--- a/setup.py
+++ b/setup.py
@@ -37,6 +37,7 @@
'PyHive>=0.2.1',
'python-dateutil==2.5.3',
'requests==2.10.0',
+ 'semantic==1.0.3',
'simplejson==3.8.2',
'six==1.10.0',
'sqlalchemy==1.0.13',