Fixing date processing

panoramix/data/__init__.py

@@ -87,13 +87,14 @@ def load_birth_names():
     session = db.session
     with gzip.open(os.path.join(DATA_FOLDER, 'birth_names.json.gz')) as f:
         pdf = pd.read_json(f)
-    pdf.ds = pd.to_datetime(pdf.ds)
+    pdf.ds = pd.to_datetime(pdf.ds, unit='ms')
     pdf.to_sql(
         'birth_names',
         db.engine,
         if_exists='replace',
         chunksize=500,
         dtype={
+            'ds': DateTime,
             'gender': String(16),
             'state': String(10),
             'name': String(255),

panoramix/models.py

@@ -1,10 +1,18 @@
-from datetime import timedelta
+from copy import deepcopy, copy
+from collections import namedtuple
+from datetime import timedelta, datetime
+import json
+from six import string_types
+import sqlparse
+import requests
+import textwrap
+
 from dateutil.parser import parse
 from flask import flash
 from flask.ext.appbuilder import Model
 from flask.ext.appbuilder.models.mixins import AuditMixin
+import pandas as pd
 from pandas import read_sql_query
-from pandas.io.json import dumps
 from pydruid import client
 from pydruid.utils.filters import Dimension, Filter
 import sqlalchemy as sqla
@@ -16,15 +24,7 @@
 from sqlalchemy.sql.elements import ColumnClause
 from sqlalchemy_utils import EncryptedType
 
-from copy import deepcopy, copy
-from collections import namedtuple
-from datetime import datetime
-import json
-import sqlparse
-import requests
-import textwrap
 
-from six import string_types
 
 from panoramix import app, db, get_session, utils
 from panoramix.viz import viz_types

panoramix/static/panoramix.js

@@ -22,11 +22,9 @@ var px = (function() {
       jsonEndpoint: function() {
         var parser = document.createElement('a');
         parser.href = data.json_endpoint;
-        console.log(parser);
         // Shallow copy
         if (dashboard !== undefined){
           qrystr = parser.search + "&extra_filters=" + JSON.stringify(dashboard.filters);
-          console.log(qrystr);
         }
         else {
           qrystr = '?' + $('#query').serialize();
@@ -67,6 +65,9 @@ var px = (function() {
         $('#timer').removeClass('btn-danger btn-success');
         $('#timer').addClass('btn-warning');
         viz.render();
+        $('#json').click(function(){window.location=slice.json_endpoint});
+        $('#standalone').click(function(){window.location=slice.standalone_endpoint});
+        $('#csv').click(function(){window.location=slice.csv_endpoint});
       },
       resize: function() {
         token.find("img.loading").show();

panoramix/utils.py

@@ -212,3 +212,13 @@ def datetime_f(dttm):
         elif now_iso[:4] == dttm[:4]:
             dttm = dttm[5:]
     return Markup("<nobr>{}</nobr>".format(dttm))
+
+
+def json_iso_dttm_ser(obj):
+    """
+    json serializer that deals with dates
+    usage: json.dumps(object, default=utils.json_ser)
+    """
+    if isinstance(obj, datetime):
+        obj = obj.isoformat()
+    return obj

panoramix/viz.py

@@ -5,7 +5,7 @@
 
 from flask import flash, request, Markup
 from markdown import markdown
-from pandas.io.json import dumps
+from pandas.io.json import dumps, to_json
 from werkzeug.datastructures import ImmutableMultiDict
 from werkzeug.urls import Href
 import numpy as np
@@ -212,6 +212,9 @@ def get_json(self):
             'data': json.loads(self.get_json_data()),
             'query': self.query,
             'form_data': self.form_data,
+            'json_endpoint': self.json_endpoint,
+            'csv_endpoint': self.csv_endpoint,
+            'standalone_endpoint': self.standalone_endpoint,
         }
         return json.dumps(payload)
 
@@ -308,10 +311,13 @@ def get_df(self):
 
     def get_json_data(self):
         df = self.get_df()
-        return dumps(dict(
-            records=df.to_dict(orient="records"),
-            columns=df.columns,
-        ))
+        return json.dumps(
+            dict(
+                records=df.to_dict(orient="records"),
+                columns=list(df.columns),
+            ),
+            default=utils.json_iso_dttm_ser,
+        )
 
 
 class PivotTableViz(BaseViz):