From 02b3fb1d634f1e6d287060af902024c82b548c3b Mon Sep 17 00:00:00 2001 From: Bogdan Date: Fri, 10 Apr 2020 14:46:46 -0700 Subject: [PATCH] Add visualization flow to the CTA queries (#9370) * Visualize after running Ctas query (#4) * Visualize after running Ctas query. Stub for table viz Work on the viz Add JS functionality for Ctas viz Add test for the table viz flow Fix test Add js test for the ctas viz * Fix tests * Resolve comments * Leverate tmpSchema in the query object * Fix i18n string Co-authored-by: bogdan kyryliuk --- .../src/SqlLab/actions/sqlLab.js | 20 +++ .../components/ExploreCtasResultsButton.jsx | 131 ++++++++++++++++++ .../src/SqlLab/components/ResultSet.jsx | 45 ++++-- .../sqllab/ExploreCtasResultsButton_spec.jsx | 78 +++++++++++ .../components/ExploreCtasResultsButton.jsx | 131 ++++++++++++++++++ superset/connectors/sqla/models.py | 1 + superset/connectors/sqla/views.py | 35 +---- superset/models/sql_lab.py | 1 + superset/security/manager.py | 1 + superset/views/base.py | 45 +++++- superset/views/core.py | 44 ++++++ tests/sqllab_tests.py | 16 +++ 12 files changed, 505 insertions(+), 43 deletions(-) create mode 100644 superset-frontend/src/SqlLab/components/ExploreCtasResultsButton.jsx create mode 100644 superset/assets/spec/javascripts/sqllab/ExploreCtasResultsButton_spec.jsx create mode 100644 superset/assets/src/SqlLab/components/ExploreCtasResultsButton.jsx diff --git a/superset-frontend/src/SqlLab/actions/sqlLab.js b/superset-frontend/src/SqlLab/actions/sqlLab.js index 9dad773219fef..eb514b2c5032c 100644 --- a/superset-frontend/src/SqlLab/actions/sqlLab.js +++ b/superset-frontend/src/SqlLab/actions/sqlLab.js @@ -1249,3 +1249,23 @@ export function createDatasource(vizOptions) { }); }; } + +export function createCtasDatasource(vizOptions) { + return dispatch => { + dispatch(createDatasourceStarted()); + return SupersetClient.post({ + endpoint: '/superset/get_or_create_table/', + postPayload: { data: vizOptions }, + }) + .then(({ json }) => { + dispatch(createDatasourceSuccess(json)); + + return json; + }) + .catch(() => { + const errorMsg = t('An error occurred while creating the data source'); + dispatch(createDatasourceFailed(errorMsg)); + return Promise.reject(new Error(errorMsg)); + }); + }; +} diff --git a/superset-frontend/src/SqlLab/components/ExploreCtasResultsButton.jsx b/superset-frontend/src/SqlLab/components/ExploreCtasResultsButton.jsx new file mode 100644 index 0000000000000..b90d351b31945 --- /dev/null +++ b/superset-frontend/src/SqlLab/components/ExploreCtasResultsButton.jsx @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { bindActionCreators } from 'redux'; +import { connect } from 'react-redux'; +import Dialog from 'react-bootstrap-dialog'; +import { t } from '@superset-ui/translation'; + +import { exportChart } from '../../explore/exploreUtils'; +import * as actions from '../actions/sqlLab'; +import InfoTooltipWithTrigger from '../../components/InfoTooltipWithTrigger'; +import Button from '../../components/Button'; + +const propTypes = { + actions: PropTypes.object.isRequired, + table: PropTypes.string.isRequired, + schema: PropTypes.string, + dbId: PropTypes.number.isRequired, + errorMessage: PropTypes.string, + templateParams: PropTypes.string, +}; +const defaultProps = { + vizRequest: {}, +}; + +class ExploreCtasResultsButton extends React.PureComponent { + constructor(props) { + super(props); + this.visualize = this.visualize.bind(this); + this.onClick = this.onClick.bind(this); + } + onClick() { + this.visualize(); + } + + buildVizOptions() { + return { + datasourceName: this.props.table, + schema: this.props.schema, + dbId: this.props.dbId, + templateParams: this.props.templateParams, + }; + } + visualize() { + this.props.actions + .createCtasDatasource(this.buildVizOptions()) + .then(data => { + const formData = { + datasource: `${data.table_id}__table`, + metrics: ['count'], + groupby: [], + viz_type: 'table', + since: '100 years ago', + all_columns: [], + row_limit: 1000, + }; + this.props.actions.addInfoToast( + t('Creating a data source and creating a new tab'), + ); + + // open new window for data visualization + exportChart(formData); + }) + .catch(() => { + this.props.actions.addDangerToast( + this.props.errorMessage || t('An error occurred'), + ); + }); + } + render() { + return ( + <> + + { + this.dialog = el; + }} + /> + + ); + } +} +ExploreCtasResultsButton.propTypes = propTypes; +ExploreCtasResultsButton.defaultProps = defaultProps; + +function mapStateToProps({ sqlLab, common }) { + return { + errorMessage: sqlLab.errorMessage, + timeout: common.conf ? common.conf.SUPERSET_WEBSERVER_TIMEOUT : null, + }; +} + +function mapDispatchToProps(dispatch) { + return { + actions: bindActionCreators(actions, dispatch), + }; +} + +export { ExploreCtasResultsButton }; +export default connect( + mapStateToProps, + mapDispatchToProps, +)(ExploreCtasResultsButton); diff --git a/superset-frontend/src/SqlLab/components/ResultSet.jsx b/superset-frontend/src/SqlLab/components/ResultSet.jsx index b7655d5fd8f53..cbe704e524ba8 100644 --- a/superset-frontend/src/SqlLab/components/ResultSet.jsx +++ b/superset-frontend/src/SqlLab/components/ResultSet.jsx @@ -23,6 +23,7 @@ import shortid from 'shortid'; import { t } from '@superset-ui/translation'; import Loading from '../../components/Loading'; +import ExploreCtasResultsButton from './ExploreCtasResultsButton'; import ExploreResultsButton from './ExploreResultsButton'; import HighlightedSql from './HighlightedSql'; import FilterableTable from '../../components/FilterableTable/FilterableTable'; @@ -101,13 +102,13 @@ export default class ResultSet extends React.PureComponent { clearQueryResults(query) { this.props.actions.clearQueryResults(query); } - popSelectStar() { + popSelectStar(tmpSchema, tmpTable) { const qe = { id: shortid.generate(), - title: this.props.query.tempTable, + title: tmpTable, autorun: false, dbId: this.props.query.dbId, - sql: `SELECT * FROM ${this.props.query.tempTable}`, + sql: `SELECT * FROM ${tmpSchema}.${tmpTable}`, }; this.props.actions.addQueryEditor(qe); } @@ -216,18 +217,38 @@ export default class ResultSet extends React.PureComponent { ); } else if (query.state === 'success' && query.ctas) { + // Async queries + let tmpSchema = query.tempSchema; + let tmpTable = query.tempTableName; + // Sync queries, query.results.query contains the source of truth for them. + if (query.results && query.results.query) { + tmpTable = query.results.query.tempTable; + tmpSchema = query.results.query.tempSchema; + } return (
- {t('Table')} [{query.tempTable}] {t('was created')}{' '} -   - + {t('Table')} [ + + {tmpSchema}.{tmpTable} + + ] {t('was created')}   + + + +
); diff --git a/superset/assets/spec/javascripts/sqllab/ExploreCtasResultsButton_spec.jsx b/superset/assets/spec/javascripts/sqllab/ExploreCtasResultsButton_spec.jsx new file mode 100644 index 0000000000000..98c8c10cf7d9b --- /dev/null +++ b/superset/assets/spec/javascripts/sqllab/ExploreCtasResultsButton_spec.jsx @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import configureStore from 'redux-mock-store'; +import thunk from 'redux-thunk'; + +import { shallow } from 'enzyme'; +import sinon from 'sinon'; + +import sqlLabReducer from '../../../src/SqlLab/reducers/index'; +import ExploreCtasResultsButton from '../../../src/SqlLab/components/ExploreCtasResultsButton'; +import Button from '../../../src/components/Button'; + +describe('ExploreCtasResultsButton', () => { + const middlewares = [thunk]; + const mockStore = configureStore(middlewares); + const initialState = { + sqlLab: { + ...sqlLabReducer(undefined, {}), + }, + common: { + conf: { SUPERSET_WEBSERVER_TIMEOUT: 45 }, + }, + }; + const store = mockStore(initialState); + const mockedProps = { + table: 'dummy_table', + schema: 'dummy_schema', + dbId: 123, + }; + const getExploreCtasResultsButtonWrapper = (props = mockedProps) => + shallow(, { + context: { store }, + }).dive(); + + it('renders', () => { + expect(React.isValidElement()).toBe(true); + }); + + it('renders with props', () => { + expect(React.isValidElement()).toBe(true); + }); + + it('renders a Button', () => { + const wrapper = getExploreCtasResultsButtonWrapper(); + expect(wrapper.find(Button)).toHaveLength(1); + }); + + describe('datasourceName', () => { + it('should build viz options', () => { + const wrapper = getExploreCtasResultsButtonWrapper(); + const spy = sinon.spy(wrapper.instance(), 'buildVizOptions'); + wrapper.instance().buildVizOptions(); + expect(spy.returnValues[0]).toEqual({ + schema: 'dummy_schema', + dbId: 123, + templateParams: undefined, + datasourceName: 'dummy_table', + }); + }); + }); +}); diff --git a/superset/assets/src/SqlLab/components/ExploreCtasResultsButton.jsx b/superset/assets/src/SqlLab/components/ExploreCtasResultsButton.jsx new file mode 100644 index 0000000000000..b90d351b31945 --- /dev/null +++ b/superset/assets/src/SqlLab/components/ExploreCtasResultsButton.jsx @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { bindActionCreators } from 'redux'; +import { connect } from 'react-redux'; +import Dialog from 'react-bootstrap-dialog'; +import { t } from '@superset-ui/translation'; + +import { exportChart } from '../../explore/exploreUtils'; +import * as actions from '../actions/sqlLab'; +import InfoTooltipWithTrigger from '../../components/InfoTooltipWithTrigger'; +import Button from '../../components/Button'; + +const propTypes = { + actions: PropTypes.object.isRequired, + table: PropTypes.string.isRequired, + schema: PropTypes.string, + dbId: PropTypes.number.isRequired, + errorMessage: PropTypes.string, + templateParams: PropTypes.string, +}; +const defaultProps = { + vizRequest: {}, +}; + +class ExploreCtasResultsButton extends React.PureComponent { + constructor(props) { + super(props); + this.visualize = this.visualize.bind(this); + this.onClick = this.onClick.bind(this); + } + onClick() { + this.visualize(); + } + + buildVizOptions() { + return { + datasourceName: this.props.table, + schema: this.props.schema, + dbId: this.props.dbId, + templateParams: this.props.templateParams, + }; + } + visualize() { + this.props.actions + .createCtasDatasource(this.buildVizOptions()) + .then(data => { + const formData = { + datasource: `${data.table_id}__table`, + metrics: ['count'], + groupby: [], + viz_type: 'table', + since: '100 years ago', + all_columns: [], + row_limit: 1000, + }; + this.props.actions.addInfoToast( + t('Creating a data source and creating a new tab'), + ); + + // open new window for data visualization + exportChart(formData); + }) + .catch(() => { + this.props.actions.addDangerToast( + this.props.errorMessage || t('An error occurred'), + ); + }); + } + render() { + return ( + <> + + { + this.dialog = el; + }} + /> + + ); + } +} +ExploreCtasResultsButton.propTypes = propTypes; +ExploreCtasResultsButton.defaultProps = defaultProps; + +function mapStateToProps({ sqlLab, common }) { + return { + errorMessage: sqlLab.errorMessage, + timeout: common.conf ? common.conf.SUPERSET_WEBSERVER_TIMEOUT : null, + }; +} + +function mapDispatchToProps(dispatch) { + return { + actions: bindActionCreators(actions, dispatch), + }; +} + +export { ExploreCtasResultsButton }; +export default connect( + mapStateToProps, + mapDispatchToProps, +)(ExploreCtasResultsButton); diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index 4d1b9f24d45fb..642997a10554d 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -1132,6 +1132,7 @@ def fetch_metadata(self, commit=True) -> None: if not self.main_dttm_col: self.main_dttm_col = any_date_col self.add_missing_metrics(metrics) + db.session.merge(self) if commit: db.session.commit() diff --git a/superset/connectors/sqla/views.py b/superset/connectors/sqla/views.py index b59bf3c8e75ff..38f78d2202dc0 100644 --- a/superset/connectors/sqla/views.py +++ b/superset/connectors/sqla/views.py @@ -29,16 +29,17 @@ from wtforms.ext.sqlalchemy.fields import QuerySelectField from wtforms.validators import Regexp -from superset import app, appbuilder, db, security_manager +from superset import app, db, security_manager from superset.connectors.base.views import DatasourceModelView from superset.constants import RouteMethod from superset.utils import core as utils from superset.views.base import ( + create_table_permissions, DatasourceFilter, DeleteMixin, - get_datasource_exist_error_msg, ListWidgetWithCheckboxes, SupersetModelView, + validate_sqlatable, YamlExportMixin, ) @@ -375,37 +376,11 @@ class TableModelView(DatasourceModelView, DeleteMixin, YamlExportMixin): } def pre_add(self, table): - with db.session.no_autoflush: - table_query = db.session.query(models.SqlaTable).filter( - models.SqlaTable.table_name == table.table_name, - models.SqlaTable.schema == table.schema, - models.SqlaTable.database_id == table.database.id, - ) - if db.session.query(table_query.exists()).scalar(): - raise Exception(get_datasource_exist_error_msg(table.full_name)) - - # Fail before adding if the table can't be found - try: - table.get_sqla_table_object() - except Exception as ex: - logger.exception(f"Got an error in pre_add for {table.name}") - raise Exception( - _( - "Table [{}] could not be found, " - "please double check your " - "database connection, schema, and " - "table name, error: {}" - ).format(table.name, str(ex)) - ) + validate_sqlatable(table) def post_add(self, table, flash_message=True): table.fetch_metadata() - security_manager.add_permission_view_menu("datasource_access", table.get_perm()) - if table.schema: - security_manager.add_permission_view_menu( - "schema_access", table.schema_perm - ) - + create_table_permissions(table) if flash_message: flash( _( diff --git a/superset/models/sql_lab.py b/superset/models/sql_lab.py index 8f0254df538c9..654bc2fb4c2b8 100644 --- a/superset/models/sql_lab.py +++ b/superset/models/sql_lab.py @@ -120,6 +120,7 @@ def to_dict(self): "startDttm": self.start_time, "state": self.status.lower(), "tab": self.tab_name, + "tempSchema": self.tmp_schema_name, "tempTable": self.tmp_table_name, "userId": self.user_id, "user": user_label(self.user), diff --git a/superset/security/manager.py b/superset/security/manager.py index fe39b1924d602..2d720965ae5a5 100644 --- a/superset/security/manager.py +++ b/superset/security/manager.py @@ -772,6 +772,7 @@ def _is_sql_lab_pvm(self, pvm: PermissionModelView) -> bool: "can_csv", "can_search_queries", "can_sqllab_viz", + "can_sqllab_table_viz", "can_sqllab", } or ( diff --git a/superset/views/base.py b/superset/views/base.py index 2c16212d0c403..33ca79148e4a5 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -35,7 +35,15 @@ from werkzeug.exceptions import HTTPException from wtforms.fields.core import Field, UnboundField -from superset import appbuilder, conf, db, get_feature_flags, security_manager +from superset import ( + app as superset_app, + appbuilder, + conf, + db, + get_feature_flags, + security_manager, +) +from superset.connectors.sqla import models from superset.exceptions import SupersetException, SupersetSecurityException from superset.translations.utils import get_language_pack from superset.utils import core as utils @@ -54,6 +62,9 @@ ) logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) +config = superset_app.config + def get_error_msg(): if conf.get("SHOW_STACKTRACE"): @@ -150,6 +161,38 @@ def get_datasource_exist_error_msg(full_name: str) -> str: return __("Datasource %(name)s already exists", name=full_name) +def validate_sqlatable(table: models.SqlaTable) -> None: + """Checks the table existence in the database.""" + with db.session.no_autoflush: + table_query = db.session.query(models.SqlaTable).filter( + models.SqlaTable.table_name == table.table_name, + models.SqlaTable.schema == table.schema, + models.SqlaTable.database_id == table.database.id, + ) + if db.session.query(table_query.exists()).scalar(): + raise Exception(get_datasource_exist_error_msg(table.full_name)) + + # Fail before adding if the table can't be found + try: + table.get_sqla_table_object() + except Exception as ex: + logger.exception(f"Got an error in pre_add for {table.name}") + raise Exception( + _( + "Table [%{table}s] could not be found, " + "please double check your " + "database connection, schema, and " + "table name, error: {}" + ).format(table.name, str(ex)) + ) + + +def create_table_permissions(table: models.SqlaTable) -> None: + security_manager.add_permission_view_menu("datasource_access", table.get_perm()) + if table.schema: + security_manager.add_permission_view_menu("schema_access", table.schema_perm) + + def get_user_roles() -> List[Role]: if g.user.is_anonymous: public_role = conf.get("AUTH_ROLE_PUBLIC") diff --git a/superset/views/core.py b/superset/views/core.py index f574d4141e76b..587a7b94cd42f 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -98,6 +98,7 @@ BaseSupersetView, check_ownership, common_bootstrap_payload, + create_table_permissions, CsvResponse, data_payload_response, DeleteMixin, @@ -108,6 +109,7 @@ json_error_response, json_success, SupersetModelView, + validate_sqlatable, ) from .utils import ( apply_display_max_row_limit, @@ -1955,6 +1957,48 @@ def sync_druid_source(self): return json_error_response(utils.error_msg_from_exception(ex)) return Response(status=201) + @has_access + @expose("/get_or_create_table/", methods=["POST"]) + @event_logger.log_this + def sqllab_table_viz(self): + """ Gets or creates a table object with attributes passed to the API. + + It expects the json with params: + * datasourceName - e.g. table name, required + * dbId - database id, required + * schema - table schema, optional + * templateParams - params for the Jinja templating syntax, optional + :return: Response + """ + SqlaTable = ConnectorRegistry.sources["table"] + data = json.loads(request.form.get("data")) + table_name = data.get("datasourceName") + database_id = data.get("dbId") + table = ( + db.session.query(SqlaTable) + .filter_by(database_id=database_id, table_name=table_name) + .one_or_none() + ) + if not table: + # Create table if doesn't exist. + with db.session.no_autoflush: + table = SqlaTable(table_name=table_name, owners=[g.user]) + table.database_id = database_id + table.database = ( + db.session.query(models.Database).filter_by(id=database_id).one() + ) + table.schema = data.get("schema") + table.template_params = data.get("templateParams") + # needed for the table validation. + validate_sqlatable(table) + + db.session.add(table) + table.fetch_metadata() + create_table_permissions(table) + db.session.commit() + + return json_success(json.dumps({"table_id": table.id})) + @has_access @expose("/sqllab_viz/", methods=["POST"]) @event_logger.log_this diff --git a/tests/sqllab_tests.py b/tests/sqllab_tests.py index ad130a8ee2bf4..bbc63d292e1e7 100644 --- a/tests/sqllab_tests.py +++ b/tests/sqllab_tests.py @@ -332,6 +332,22 @@ def test_sqllab_viz(self): table = db.session.query(SqlaTable).filter_by(id=table_id).one() self.assertEqual([owner.username for owner in table.owners], ["admin"]) + def test_sqllab_table_viz(self): + self.login("admin") + examples_dbid = get_example_database().id + payload = {"datasourceName": "ab_role", "columns": [], "dbId": examples_dbid} + + data = {"data": json.dumps(payload)} + resp = self.get_json_resp("/superset/get_or_create_table/", data=data) + self.assertIn("table_id", resp) + + # ensure owner is set correctly + table_id = resp["table_id"] + table = db.session.query(SqlaTable).filter_by(id=table_id).one() + self.assertEqual([owner.username for owner in table.owners], ["admin"]) + db.session.delete(table) + db.session.commit() + def test_sql_limit(self): self.login("admin") test_limit = 1