[Geo] Added DeckGL Arc Layer and Refactor on BaseDeckGL class (#4134)

* Added DeckGL.arc layer

* added color controls

* added stroke_width control

* added process spatial key methods

* change exception to ValueError

* put location into tuple

* reference global spatial keys array

* linting

* refactor on process_spatial_data_obj

* rm whitespace

* refactor arc.get_data

* Revert "refactor arc.get_data"

This reverts commit 8d01b2a.

* add spatial controls array

* refactor on spatial keys again :)

* return altered df

* Working refactor with deckGL Arcs

* working arcs refactor :)

* refactored all other deckGL viz types

superset/assets/javascripts/explore/stores/controls.jsx

@@ -519,6 +519,26 @@ export const controls = {
     }),
   },
 
+  start_spatial: {
+    type: 'SpatialControl',
+    label: t('Start Longitude & Latitude'),
+    validators: [v.nonEmpty],
+    description: t('Point to your spatial columns'),
+    mapStateToProps: state => ({
+      choices: (state.datasource) ? state.datasource.all_cols : [],
+    }),
+  },
+
+  end_spatial: {
+    type: 'SpatialControl',
+    label: t('End Longitude & Latitude'),
+    validators: [v.nonEmpty],
+    description: t('Point to your spatial columns'),
+    mapStateToProps: state => ({
+      choices: (state.datasource) ? state.datasource.all_cols : [],
+    }),
+  },
+
   longitude: {
     type: 'SelectControl',
     label: t('Longitude'),
@@ -560,6 +580,15 @@ export const controls = {
     choices: formatSelectOptions([0, 100, 200, 300, 500]),
   },
 
+  stroke_width: {
+    type: 'SelectControl',
+    freeForm: true,
+    label: t('Stroke Width'),
+    validators: [v.integer],
+    default: null,
+    choices: formatSelectOptions([1, 2, 3, 4, 5]),
+  },
+
   all_columns_x: {
     type: 'SelectControl',
     label: 'X',

superset/assets/javascripts/explore/stores/visTypes.js

@@ -517,6 +517,34 @@ export const visTypes = {
     ],
   },
 
+  deck_arc: {
+    label: t('Deck.gl - Arc'),
+    requiresTime: true,
+    controlPanelSections: [
+      {
+        label: t('Query'),
+        expanded: true,
+        controlSetRows: [
+          ['start_spatial', 'end_spatial'],
+          ['row_limit', null],
+        ],
+      },
+      {
+        label: t('Map'),
+        controlSetRows: [
+          ['mapbox_style', 'viewport'],
+        ],
+      },
+      {
+        label: t('Arc'),
+        controlSetRows: [
+          ['color_picker', null],
+          ['stroke_width', null],
+        ],
+      },
+    ],
+  },
+
   deck_scatter: {
     label: t('Deck.gl - Scatter plot'),
     requiresTime: true,

superset/assets/visualizations/deckgl/layers/arc.jsx

@@ -0,0 +1,16 @@
+import { ArcLayer } from 'deck.gl';
+
+export default function arcLayer(formData, payload) {
+  const fd = formData;
+  const fc = fd.color_picker;
+  const data = payload.data.arcs.map(d => ({
+    ...d,
+    color: [fc.r, fc.g, fc.b, 255 * fc.a],
+  }));
+
+  return new ArcLayer({
+    id: `path-layer-${fd.slice_id}`,
+    data,
+    strokeWidth: (fd.stroke_width) ? fd.stroke_width : 3,
+  });
+}

superset/assets/visualizations/deckgl/layers/index.js

@@ -5,6 +5,7 @@ import deck_path from './path';
 import deck_hex from './hex';
 import deck_scatter from './scatter';
 import deck_geojson from './geojson';
+import deck_arc from './arc';
 
 const layerGenerators = {
   deck_grid,
@@ -13,5 +14,6 @@ const layerGenerators = {
   deck_hex,
   deck_scatter,
   deck_geojson,
+  deck_arc,
 };
 export default layerGenerators;

superset/assets/visualizations/main.js

@@ -46,6 +46,7 @@ export const VIZ_TYPES = {
   deck_path: 'deck_path',
   deck_geojson: 'deck_geojson',
   deck_multi: 'deck_multi',
+  deck_arc: 'deck_arc',
 };
 
 const vizMap = {
@@ -92,6 +93,7 @@ const vizMap = {
   [VIZ_TYPES.deck_hex]: deckglFactory,
   [VIZ_TYPES.deck_path]: deckglFactory,
   [VIZ_TYPES.deck_geojson]: deckglFactory,
+  [VIZ_TYPES.deck_arc]: deckglFactory,
   [VIZ_TYPES.deck_multi]: require('./deckgl/multi.jsx'),
 };
 export default vizMap;

superset/viz.py

@@ -1806,6 +1806,7 @@ class BaseDeckGLViz(BaseViz):
 
     is_timeseries = False
     credits = '<a href="https://uber.github.io/deck.gl/">deck.gl</a>'
+    spatial_control_keys = []
 
     def get_metrics(self):
         self.metric = self.form_data.get('size')
@@ -1817,26 +1818,48 @@ def get_properties(self, d):
         }
 
     def get_position(self, d):
-        return [
-            d.get('lon'),
-            d.get('lat'),
-        ]
+        raise Exception('Not implemented in child class!')
+
+    def process_spatial_query_obj(self, key, group_by):
+        spatial = self.form_data.get(key)
+        if spatial is None:
+            raise ValueError(_('Bad spatial key'))
+
+        if spatial.get('type') == 'latlong':
+            group_by += [spatial.get('lonCol')]
+            group_by += [spatial.get('latCol')]
+        elif spatial.get('type') == 'delimited':
+            group_by += [spatial.get('lonlatCol')]
+        elif spatial.get('type') == 'geohash':
+            group_by += [spatial.get('geohashCol')]
+
+    def process_spatial_data_obj(self, key, df):
+        spatial = self.form_data.get(key)
+        if spatial is None:
+            raise ValueError(_('Bad spatial key'))
+
+        if spatial.get('type') == 'latlong':
+            df[key] = list(zip(df[spatial.get('lonCol')], df[spatial.get('latCol')]))
+        elif spatial.get('type') == 'delimited':
+            df[key] = (df[spatial.get('lonlatCol')].str.split(spatial.get('delimiter')))
+            if spatial.get('reverseCheckbox'):
+                df[key] = [list(reversed(item))for item in df[key]]
+            del df[spatial.get('lonlatCol')]
+        elif spatial.get('type') == 'geohash':
+            latlong = df[spatial.get('geohashCol')].map(geohash.decode)
+            df[key] = list(zip(latlong.apply(lambda x: x[0]),
+                               latlong.apply(lambda x: x[1])))
+            del df[spatial.get('geohashCol')]
+
+        return df
 
     def query_obj(self):
         d = super(BaseDeckGLViz, self).query_obj()
         fd = self.form_data
-
         gb = []
 
-        spatial = fd.get('spatial')
-        if spatial:
-            if spatial.get('type') == 'latlong':
-                gb += [spatial.get('lonCol')]
-                gb += [spatial.get('latCol')]
-            elif spatial.get('type') == 'delimited':
-                gb += [spatial.get('lonlatCol')]
-            elif spatial.get('type') == 'geohash':
-                gb += [spatial.get('geohashCol')]
+        for key in self.spatial_control_keys:
+            self.process_spatial_query_obj(key, gb)
 
         if fd.get('dimension'):
             gb += [fd.get('dimension')]
@@ -1849,36 +1872,16 @@ def query_obj(self):
             d['metrics'] = self.get_metrics()
         else:
             d['columns'] = gb
+
         return d
 
     def get_js_columns(self, d):
         cols = self.form_data.get('js_columns') or []
         return {col: d.get(col) for col in cols}
 
     def get_data(self, df):
-        fd = self.form_data
-        spatial = fd.get('spatial')
-        if spatial:
-            if spatial.get('type') == 'latlong':
-                df = df.rename(columns={
-                    spatial.get('lonCol'): 'lon',
-                    spatial.get('latCol'): 'lat'})
-            elif spatial.get('type') == 'delimited':
-                cols = ['lon', 'lat']
-                if spatial.get('reverseCheckbox'):
-                    cols.reverse()
-                df[cols] = (
-                    df[spatial.get('lonlatCol')]
-                    .str
-                    .split(spatial.get('delimiter'), expand=True)
-                    .astype(np.float64)
-                )
-                del df[spatial.get('lonlatCol')]
-            elif spatial.get('type') == 'geohash':
-                latlong = df[spatial.get('geohashCol')].map(geohash.decode)
-                df['lat'] = latlong.apply(lambda x: x[0])
-                df['lon'] = latlong.apply(lambda x: x[1])
-                del df['geohash']
+        for key in self.spatial_control_keys:
+            df = self.process_spatial_data_obj(key, df)
 
         features = []
         for d in df.to_dict(orient='records'):
@@ -1899,13 +1902,17 @@ class DeckScatterViz(BaseDeckGLViz):
 
     viz_type = 'deck_scatter'
     verbose_name = _('Deck.gl - Scatter plot')
+    spatial_control_keys = ['spatial']
 
     def query_obj(self):
         fd = self.form_data
         self.point_radius_fixed = (
             fd.get('point_radius_fixed') or {'type': 'fix', 'value': 500})
         return super(DeckScatterViz, self).query_obj()
 
+    def get_position(self, d):
+        return d['spatial']
+
     def get_metrics(self):
         self.metric = None
         if self.point_radius_fixed.get('type') == 'metric':
@@ -1935,6 +1942,10 @@ class DeckScreengrid(BaseDeckGLViz):
 
     viz_type = 'deck_screengrid'
     verbose_name = _('Deck.gl - Screen Grid')
+    spatial_control_keys = ['spatial']
+
+    def get_position(self, d):
+        return d['spatial']
 
 
 class DeckGrid(BaseDeckGLViz):
@@ -1943,6 +1954,10 @@ class DeckGrid(BaseDeckGLViz):
 
     viz_type = 'deck_grid'
     verbose_name = _('Deck.gl - 3D Grid')
+    spatial_control_keys = ['spatial']
+
+    def get_position(self, d):
+        return d['spatial']
 
 
 class DeckPathViz(BaseDeckGLViz):
@@ -1955,6 +1970,10 @@ class DeckPathViz(BaseDeckGLViz):
         'json': json.loads,
         'polyline': polyline.decode,
     }
+    spatial_control_keys = ['spatial']
+
+    def get_position(self, d):
+        return d['spatial']
 
     def query_obj(self):
         d = super(DeckPathViz, self).query_obj()
@@ -1982,6 +2001,10 @@ class DeckHex(BaseDeckGLViz):
 
     viz_type = 'deck_hex'
     verbose_name = _('Deck.gl - 3D HEX')
+    spatial_control_keys = ['spatial']
+
+    def get_position(self, d):
+        return d['spatial']
 
 
 class DeckGeoJson(BaseDeckGLViz):
@@ -2011,6 +2034,32 @@ def get_data(self, df):
         }
 
 
+class DeckArc(BaseDeckGLViz):
+
+    """deck.gl's Arc Layer"""
+
+    viz_type = 'deck_arc'
+    verbose_name = _('Deck.gl - Arc')
+    spatial_control_keys = ['start_spatial', 'end_spatial']
+
+    def get_position(self, d):
+        deck_map = {
+            'start_spatial': 'sourcePosition',
+            'end_spatial': 'targetPosition',
+        }
+
+        return {deck_map[key]: d[key] for key in self.spatial_control_keys}
+
+    def get_data(self, df):
+        d = super(DeckArc, self).get_data(df)
+        arcs = d['features']
+
+        return {
+            'arcs': [arc['position'] for arc in arcs],
+            'mapboxApiKey': config.get('MAPBOX_API_KEY'),
+        }
+
+
 class EventFlowViz(BaseViz):
 
     """A visualization to explore patterns in event sequences"""