Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove private commits related to ZIP code and FSA (Canada ZIP) #73

Merged
merged 1 commit into from
Oct 28, 2019
Merged

Remove private commits related to ZIP code and FSA (Canada ZIP) #73

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 0 additions & 50 deletions superset/assets/src/explore/components/controls/SpatialControl.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@ const spatialTypes = {
latlong: 'latlong',
delimited: 'delimited',
geohash: 'geohash',
zipcode: 'zipcode',
fsa: 'fsa',
};

const propTypes = {
Expand Down Expand Up @@ -65,8 +63,6 @@ export default class SpatialControl extends React.Component {
lonlatCol: v.lonlatCol || defaultCol,
reverseCheckbox: v.reverseCheckbox || false,
geohashCol: v.geohashCol || defaultCol,
zipcodeCol: v.zipcodeCol || defaultCol,
fsaCol: v.fsaCol || defaultCol,
value: null,
errors: [],
};
Expand Down Expand Up @@ -101,18 +97,6 @@ export default class SpatialControl extends React.Component {
if (!value.geohashCol) {
errors.push(errMsg);
}
} else if (type === spatialTypes.zipcode) {
value.zipcodeCol = this.state.zipcodeCol;
value.reverseCheckbox = this.state.reverseCheckbox;
if (!value.zipcodeCol) {
errors.push(errMsg);
}
} else if (type === spatialTypes.fsa) {
value.fsaCol = this.state.fsaCol;
value.reverseCheckbox = this.state.reverseCheckbox;
if (!value.fsaCol) {
errors.push(errMsg);
}
}
this.setState({ value, errors });
this.props.onChange(value, errors);
Expand All @@ -136,10 +120,6 @@ export default class SpatialControl extends React.Component {
return `${this.state.lonlatCol}`;
} else if (this.state.type === spatialTypes.geohash) {
return `${this.state.geohashCol}`;
} else if (this.state.type === spatialTypes.zipcode) {
return `${this.state.zipcodeCol}`;
} else if (this.state.type === spatialTypes.fsa) {
return `${this.state.fsaCol}`;
}
return null;
}
Expand Down Expand Up @@ -219,36 +199,6 @@ export default class SpatialControl extends React.Component {
</Col>
</Row>
</PopoverSection>
<PopoverSection
title={t('ZIP code')}
isSelected={this.state.type === spatialTypes.zipcode}
onSelect={this.setType.bind(this, spatialTypes.zipcode)}
>
<Row>
<Col md={6}>
Column
{this.renderSelect('zipcodeCol', spatialTypes.zipcode)}
</Col>
<Col md={6}>
{this.renderReverseCheckbox()}
</Col>
</Row>
</PopoverSection>
<PopoverSection
title={t('FSA')}
isSelected={this.state.type === spatialTypes.fsa}
onSelect={this.setType.bind(this, spatialTypes.fsa)}
>
<Row>
<Col md={6}>
Column
{this.renderSelect('fsaCol', spatialTypes.fsa)}
</Col>
<Col md={6}>
{this.renderReverseCheckbox()}
</Col>
</Row>
</PopoverSection>
<div className="clearfix">
<Button
bsSize="small"
Expand Down
2 changes: 0 additions & 2 deletions superset/assets/src/explore/controls.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2213,8 +2213,6 @@ export const controls = {
['polyline', 'Polyline'],
['json', 'JSON'],
['geohash', 'geohash (square)'],
['zipcode', 'ZIP code'],
['fsa', 'FSA'],
],
},

Expand Down
9 changes: 3 additions & 6 deletions superset/assets/src/visualizations/deckgl/layers/Polygon/Polygon.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,7 @@ import sandboxedEval from '../../../../modules/sandbox';

const DOUBLE_CLICK_TRESHOLD = 250; // milliseconds

function getPoints(features, lineType) {
if (lineType === 'zipcode' || lineType === 'fsa') {
return features.map(d => d.polygon).flat().flat();
}
function getPoints(features) {
return features.map(d => d.polygon).flat();
}

Expand Down Expand Up @@ -97,7 +94,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip, selected, o
}
return baseColor;
};
const tooltipContentGenerator = (fd.line_column && fd.metric && ['geohash', 'zipcode', 'fsa'].indexOf(fd.line_type) >= 0)
const tooltipContentGenerator = (fd.line_column && fd.metric && fd.line_type === 'geohash')
? setTooltipContent(fd)
: undefined;
return new PolygonLayer({
Expand Down Expand Up @@ -170,7 +167,7 @@ class DeckGLPolygon extends React.Component {
} = getPlaySliderParams(timestamps, granularity);

const viewport = props.formData.autozoom
? fitViewport(props.viewport, getPoints(features, props.payload.form_data.line_type))
? fitViewport(props.viewport, getPoints(features))
: props.viewport;

return {
Expand Down
139 changes: 2 additions & 137 deletions superset/viz.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
from itertools import product
import logging
import math
import os
import pickle as pkl
import re
from typing import Any, Dict, List, Optional
Expand All @@ -46,7 +45,6 @@
from pandas.tseries.frequencies import to_offset
import polyline
import simplejson as json
import sqlalchemy

from superset import app, cache, get_css_manifest_files
from superset.exceptions import NullValueException, SpatialException
Expand Down Expand Up @@ -2132,10 +2130,6 @@ def get_spatial_columns(self, key):
return [spatial.get("lonlatCol")]
elif spatial.get("type") == "geohash":
return [spatial.get("geohashCol")]
elif spatial.get("type") == "zipcode":
return [spatial.get("zipcodeCol")]
elif spatial.get("type") == "fsa":
return [spatial.get("fsaCol")]

@staticmethod
def parse_coordinates(s):
Expand All @@ -2152,18 +2146,6 @@ def reverse_geohash_decode(geohash_code):
lat, lng = geohash.decode(geohash_code)
return (lng, lat)

@staticmethod
def reverse_zipcode_decode(zipcode):
raise NotImplementedError(
"No mapping from ZIP code to single latitude/longitude"
)

@staticmethod
def reverse_fsa_decode(fsa):
raise NotImplementedError(
"No mapping from FSA code to single latitude/longitude"
)

@staticmethod
def reverse_latlong(df, key):
df[key] = [tuple(reversed(o)) for o in df[key] if isinstance(o, (list, tuple))]
Expand All @@ -2187,12 +2169,6 @@ def process_spatial_data_obj(self, key, df):
elif spatial.get("type") == "geohash":
df[key] = df[spatial.get("geohashCol")].map(self.reverse_geohash_decode)
del df[spatial.get("geohashCol")]
elif spatial.get("type") == "zipcode":
df[key] = df[spatial.get("zipcodeCol")].map(self.reverse_zipcode_decode)
del df[spatial.get("zipcodeCol")]
elif spatial.get("type") == "fsa":
df[key] = df[spatial.get("fsaCol")].map(self.reverse_fsa_decode)
del df[spatial.get("fsaCol")]

if spatial.get("reverseCheckbox"):
self.reverse_latlong(df, key)
Expand Down Expand Up @@ -2382,106 +2358,6 @@ def geohash_to_json(geohash_code):
]


def zipcode_deser(zipcodes):
geojson = zipcodes_to_json(zipcodes)

def deser(zipcode):
if str(zipcode) in geojson:
return geojson[str(zipcode)]["coordinates"][0]

return deser


def fsa_deser(fsas):
geojson = fsas_to_json(fsas)

def deser(fsa):
if fsa in geojson:
return geojson[fsa][0][0]

return deser


def zipcodes_to_json(zipcodes):
user = os.environ.get("CREDENTIALS_LYFTPG_USER", "")
password = os.environ.get("CREDENTIALS_LYFTPG_PASSWORD", "")
url = (
"postgresql+psycopg2://"
"{user}:{password}"
"@analytics-platform-vpc.c067nfzisc99.us-east-1.rds.amazonaws.com:5432"
"/platform".format(user=user, password=password)
)

out = {}
missing = set()
for zipcode in zipcodes:
cache_key = "zipcode_geojson_{}".format(zipcode)
geojson = cache and cache.get(cache_key)
if geojson:
out[zipcode] = geojson
else:
missing.add(str(zipcode))

if not missing:
return out

# fetch missing geojson from
in_clause = ", ".join(["%s"] * len(missing))
query = "SELECT zipcode, geojson FROM zip_codes WHERE zipcode IN ({0})".format(
in_clause
)
conn = sqlalchemy.create_engine(url, client_encoding="utf8")
results = conn.execute(query, tuple(missing)).fetchall()

for zipcode, geojson in results:
out[zipcode] = geojson
if cache and len(results) < 10000: # avoid storing too much
cache_key = "zipcode_geojson_{}".format(zipcode)
try:
cache.set(cache_key, geojson, timeout=86400)
except Exception:
pass

return out


def fsas_to_json(fsas):
url = "presto://prestoproxy.lyft.net:8443/hive"

out = {}
missing = set()
for fsa in fsas:
cache_key = "fsa_geojson_{}".format(fsa)
geojson = cache and cache.get(cache_key)
if geojson:
out[fsa] = geojson
else:
missing.add(fsa)

if not missing:
return out

# fetch missing geojson from lyftpg
in_clause = ", ".join(["%s"] * len(missing))
query = "SELECT fsa, geo_json FROM jbridgem.test_geo_json_fsa WHERE fsa IN ({0})".format(
in_clause
)
conn = sqlalchemy.create_engine(url, connect_args={"protocol": "https"})
results = conn.execute(query, tuple(missing)).fetchall()

for fsa, geojson in results:
geojson = json.loads(geojson)
out[fsa] = geojson
if cache and len(results) < 10000: # avoid storing too much
cache_key = "fsa_geojson_{}".format(fsa)
try:
cache.set(cache_key, geojson, timeout=86400)
except Exception:
pass

return out


class DeckPathViz(BaseDeckGLViz):

"""deck.gl's PathLayer"""
Expand All @@ -2494,8 +2370,6 @@ class DeckPathViz(BaseDeckGLViz):
"json": json.loads,
"polyline": polyline.decode,
"geohash": geohash_to_json,
"zipcode": None, # per request
"fsa": None, # per request
}

def query_obj(self):
Expand All @@ -2521,23 +2395,14 @@ def get_properties(self, d):
if fd.get("reverse_long_lat"):
path = [(o[1], o[0]) for o in path]
d[self.deck_viz_key] = path
if line_type not in ["geohash", "zipcode", "fsa"]:
if line_type != "geohash":
del d[line_column]
d["__timestamp"] = d.get(DTTM_ALIAS) or d.get("__time")
return d

def get_data(self, df):
self.metric_label = utils.get_metric_name(self.metric)
fd = self.form_data
line_type = fd.get("line_type")
if line_type == "zipcode":
zipcodes = df[fd["line_column"]].unique()
self.deser_map["zipcode"] = zipcode_deser(zipcodes)
elif line_type == "fsa":
fsas = df[fd["line_column"]].unique()
self.deser_map["fsa"] = fsa_deser(fsas)

return super(DeckPathViz, self).get_data(df)
return super().get_data(df)


class DeckPolygon(DeckPathViz):
Expand Down