Allow user to force refresh metadata (apache#5933)

* Allow user to force refresh metadata

* fix javascript test error

* nit

* fix styling

* allow custom cache timeout configuration on any database

* minor improvement

* nit

* fix test

* nit

* preserve the old endpoint

(cherry picked from commit 712c1aa)
(cherry picked from commit a3441c1)

CONTRIBUTING.md

@@ -307,9 +307,13 @@ commands are invoked.
 
 We use [Mocha](https://mochajs.org/), [Chai](http://chaijs.com/) and [Enzyme](http://airbnb.io/enzyme/) to test Javascript. Tests can be run with:
 
-    cd /superset/superset/assets/javascripts
-    npm i
-    npm run test
+```bash
+cd superset/assets/spec
+npm install
+npm run test
+```
+
+### Integration testing
 
 ## Linting
 

superset/assets/spec/javascripts/sqllab/SqlEditorLeftBar_spec.jsx

@@ -132,7 +132,7 @@ describe('SqlEditorLeftBar', () => {
         return d.promise();
       });
       wrapper.instance().fetchSchemas(1);
-      expect(ajaxStub.getCall(0).args[0]).to.equal('/superset/schemas/1/');
+      expect(ajaxStub.getCall(0).args[0]).to.equal('/superset/schemas/1/false/');
       expect(wrapper.state().schemaOptions).to.have.length(3);
     });
     it('should handle error', () => {

superset/assets/src/SqlLab/components/SqlEditorLeftBar.jsx

@@ -8,6 +8,7 @@ import createFilterOptions from 'react-select-fast-filter-options';
 
 import TableElement from './TableElement';
 import AsyncSelect from '../../components/AsyncSelect';
+import RefreshLabel from '../../components/RefreshLabel';
 import { t } from '../../locales';
 
 const $ = require('jquery');
@@ -39,7 +40,7 @@ class SqlEditorLeftBar extends React.PureComponent {
     this.fetchSchemas(this.props.queryEditor.dbId);
     this.fetchTables(this.props.queryEditor.dbId, this.props.queryEditor.schema);
   }
-  onDatabaseChange(db) {
+  onDatabaseChange(db, force) {
     const val = db ? db.value : null;
     this.setState({ schemaOptions: [] });
     this.props.actions.queryEditorSetSchema(this.props.queryEditor, null);
@@ -48,7 +49,7 @@ class SqlEditorLeftBar extends React.PureComponent {
       this.setState({ tableOptions: [] });
     } else {
       this.fetchTables(val, this.props.queryEditor.schema);
-      this.fetchSchemas(val);
+      this.fetchSchemas(val, force || false);
     }
   }
   getTableNamesBySubStr(input) {
@@ -116,11 +117,12 @@ class SqlEditorLeftBar extends React.PureComponent {
     this.props.actions.queryEditorSetSchema(this.props.queryEditor, schema);
     this.fetchTables(this.props.queryEditor.dbId, schema);
   }
-  fetchSchemas(dbId) {
+  fetchSchemas(dbId, force) {
     const actualDbId = dbId || this.props.queryEditor.dbId;
+    const forceRefresh = force || false;
     if (actualDbId) {
       this.setState({ schemaLoading: true });
-      const url = `/superset/schemas/${actualDbId}/`;
+      const url = `/superset/schemas/${actualDbId}/${forceRefresh}/`;
       $.get(url).done((data) => {
         const schemaOptions = data.schemas.map(s => ({ value: s, label: s }));
         this.setState({ schemaOptions, schemaLoading: false });
@@ -146,6 +148,7 @@ class SqlEditorLeftBar extends React.PureComponent {
       tableSelectPlaceholder = t('Select table ');
       tableSelectDisabled = true;
     }
+    const database = this.props.database || {};
     return (
       <div className="clearfix sql-toolbar">
         <div>
@@ -174,20 +177,31 @@ class SqlEditorLeftBar extends React.PureComponent {
           />
         </div>
         <div className="m-t-5">
-          <Select
-            name="select-schema"
-            placeholder={t('Select a schema (%s)', this.state.schemaOptions.length)}
-            options={this.state.schemaOptions}
-            value={this.props.queryEditor.schema}
-            valueRenderer={o => (
-              <div>
-                <span className="text-muted">{t('Schema:')}</span> {o.label}
-              </div>
-            )}
-            isLoading={this.state.schemaLoading}
-            autosize={false}
-            onChange={this.changeSchema.bind(this)}
-          />
+          <div className="row">
+            <div className="col-md-11 col-xs-11" style={{ paddingRight: '2px' }}>
+              <Select
+                name="select-schema"
+                placeholder={t('Select a schema (%s)', this.state.schemaOptions.length)}
+                options={this.state.schemaOptions}
+                value={this.props.queryEditor.schema}
+                valueRenderer={o => (
+                  <div>
+                    <span className="text-muted">{t('Schema:')}</span> {o.label}
+                  </div>
+                )}
+                isLoading={this.state.schemaLoading}
+                autosize={false}
+                onChange={this.changeSchema.bind(this)}
+              />
+            </div>
+            <div className="col-md-1 col-xs-1" style={{ paddingTop: '8px', paddingLeft: '0px' }}>
+              <RefreshLabel
+                onClick={this.onDatabaseChange.bind(
+                    this, { value: database.id }, true)}
+                tooltipContent="force refresh schema list"
+              />
+            </div>
+          </div>
         </div>
         <hr />
         <div className="m-t-5">

superset/assets/src/components/RefreshLabel.jsx

@@ -0,0 +1,51 @@
+import React from 'react';
+import PropTypes from 'prop-types';
+import { Label } from 'react-bootstrap';
+import TooltipWrapper from './TooltipWrapper';
+
+const propTypes = {
+  onClick: PropTypes.func,
+  className: PropTypes.string,
+  tooltipContent: PropTypes.string.isRequired,
+};
+
+class RefreshLabel extends React.PureComponent {
+  constructor(props) {
+    super(props);
+    this.state = {
+      hovered: false,
+    };
+  }
+
+  mouseOver() {
+    this.setState({ hovered: true });
+  }
+
+  mouseOut() {
+    this.setState({ hovered: false });
+  }
+
+  render() {
+    const labelStyle = this.state.hovered ? 'primary' : 'default';
+    const tooltip = 'Click to ' + this.props.tooltipContent;
+    return (
+      <TooltipWrapper
+        tooltip={tooltip}
+        label="cache-desc"
+      >
+        <Label
+          className={this.props.className}
+          bsStyle={labelStyle}
+          style={{ fontSize: '13px', marginRight: '5px', cursor: 'pointer' }}
+          onClick={this.props.onClick}
+          onMouseOver={this.mouseOver.bind(this)}
+          onMouseOut={this.mouseOut.bind(this)}
+        >
+          <i className="fa fa-refresh" />
+        </Label>
+      </TooltipWrapper>);
+  }
+}
+RefreshLabel.propTypes = propTypes;
+
+export default RefreshLabel;

superset/cache_util.py

@@ -7,30 +7,56 @@
 
 from flask import request
 
-from superset import tables_cache
+from superset import cache, tables_cache
 
 
 def view_cache_key(*unused_args, **unused_kwargs):
     args_hash = hash(frozenset(request.args.items()))
     return 'view/{}/{}'.format(request.path, args_hash)
 
 
-def memoized_func(timeout=5 * 60, key=view_cache_key):
+def default_timeout(*unused_args, **unused_kwargs):
+    return 5 * 60
+
+
+def default_enable_cache(*unused_args, **unused_kwargs):
+    return True
+
+
+def memoized_func(timeout=default_timeout,
+                  key=view_cache_key,
+                  enable_cache=default_enable_cache,
+                  use_tables_cache=False):
     """Use this decorator to cache functions that have predefined first arg.
 
+    If enable_cache() is False,
+        the function will never be cached.
+    If enable_cache() is True,
+        cache is adopted and will timeout in timeout() seconds.
+        If force is True, cache will be refreshed.
+
     memoized_func uses simple_cache and stored the data in memory.
     Key is a callable function that takes function arguments and
     returns the caching key.
     """
     def wrap(f):
-        if tables_cache:
+        selected_cache = None
+        if use_tables_cache and tables_cache:
+            selected_cache = tables_cache
+        elif cache:
+            selected_cache = cache
+
+        if selected_cache:
             def wrapped_f(cls, *args, **kwargs):
+                if not enable_cache(*args, **kwargs):
+                    return f(cls, *args, **kwargs)
+
                 cache_key = key(*args, **kwargs)
-                o = tables_cache.get(cache_key)
+                o = selected_cache.get(cache_key)
                 if not kwargs['force'] and o is not None:
                     return o
                 o = f(cls, *args, **kwargs)
-                tables_cache.set(cache_key, o, timeout=timeout)
+                selected_cache.set(cache_key, o, timeout=timeout(*args, **kwargs))
                 return o
         else:
             # noop

superset/db_engine_specs.py

@@ -235,7 +235,8 @@ def convert_dttm(cls, target_type, dttm):
     @classmethod
     @cache_util.memoized_func(
         timeout=600,
-        key=lambda *args, **kwargs: 'db:{}:{}'.format(args[0].id, args[1]))
+        key=lambda *args, **kwargs: 'db:{}:{}'.format(args[0].id, args[1]),
+        use_tables_cache=True)
     def fetch_result_sets(cls, db, datasource_type, force=False):
         """Returns the dictionary {schema : [result_set_name]}.
 
@@ -299,7 +300,21 @@ def patch(cls):
         pass
 
     @classmethod
-    def get_schema_names(cls, inspector):
+    @cache_util.memoized_func(
+        enable_cache=lambda *args, **kwargs: kwargs.get('enable_cache', False),
+        timeout=lambda *args, **kwargs: kwargs.get('cache_timeout'),
+        key=lambda *args, **kwargs: 'db:{}:schema_list'.format(kwargs.get('db_id')))
+    def get_schema_names(cls, inspector, db_id,
+                         enable_cache, cache_timeout, force=False):
+        """A function to get all schema names in this db.
+
+        :param inspector: URI string
+        :param db_id: database id
+        :param enable_cache: whether to enable cache for the function
+        :param cache_timeout: timeout settings for cache in second.
+        :param force: force to refresh
+        :return: a list of schema names
+        """
         return inspector.get_schema_names()
 
     @classmethod
@@ -562,7 +577,8 @@ def epoch_to_dttm(cls):
     @classmethod
     @cache_util.memoized_func(
         timeout=600,
-        key=lambda *args, **kwargs: 'db:{}:{}'.format(args[0].id, args[1]))
+        key=lambda *args, **kwargs: 'db:{}:{}'.format(args[0].id, args[1]),
+        use_tables_cache=True)
     def fetch_result_sets(cls, db, datasource_type, force=False):
         schemas = db.inspector.get_schema_names()
         result_sets = {}
@@ -712,7 +728,8 @@ def epoch_to_dttm(cls):
     @classmethod
     @cache_util.memoized_func(
         timeout=600,
-        key=lambda *args, **kwargs: 'db:{}:{}'.format(args[0].id, args[1]))
+        key=lambda *args, **kwargs: 'db:{}:{}'.format(args[0].id, args[1]),
+        use_tables_cache=True)
     def fetch_result_sets(cls, db, datasource_type, force=False):
         """Returns the dictionary {schema : [result_set_name]}.
 
@@ -979,7 +996,8 @@ def patch(cls):
     @classmethod
     @cache_util.memoized_func(
         timeout=600,
-        key=lambda *args, **kwargs: 'db:{}:{}'.format(args[0].id, args[1]))
+        key=lambda *args, **kwargs: 'db:{}:{}'.format(args[0].id, args[1]),
+        use_tables_cache=True)
     def fetch_result_sets(cls, db, datasource_type, force=False):
         return BaseEngineSpec.fetch_result_sets(
             db, datasource_type, force=force)
@@ -1435,7 +1453,12 @@ def convert_dttm(cls, target_type, dttm):
         return "'{}'".format(dttm.strftime('%Y-%m-%d %H:%M:%S'))
 
     @classmethod
-    def get_schema_names(cls, inspector):
+    @cache_util.memoized_func(
+        enable_cache=lambda *args, **kwargs: kwargs.get('enable_cache', False),
+        timeout=lambda *args, **kwargs: kwargs.get('cache_timeout'),
+        key=lambda *args, **kwargs: 'db:{}:schema_list'.format(kwargs.get('db_id')))
+    def get_schema_names(cls, inspector, db_id,
+                         enable_cache, cache_timeout, force=False):
         schemas = [row[0] for row in inspector.engine.execute('SHOW SCHEMAS')
                    if not row[0].startswith('_')]
         return schemas

superset/models/core.py

@@ -655,6 +655,7 @@ class Database(Model, AuditMixinNullable, ImportMixin):
     {
         "metadata_params": {},
         "engine_params": {},
+        "metadata_cache_timeout": {},
         "schemas_allowed_for_csv_upload": []
     }
     """))
@@ -878,8 +879,17 @@ def all_view_names(self, schema=None, force=False):
             pass
         return views
 
-    def all_schema_names(self):
-        return sorted(self.db_engine_spec.get_schema_names(self.inspector))
+    def all_schema_names(self, force_refresh=False):
+        extra = self.get_extra()
+        medatada_cache_timeout = extra.get('metadata_cache_timeout', {})
+        schema_cache_timeout = medatada_cache_timeout.get('schema_cache_timeout')
+        enable_cache = 'schema_cache_timeout' in medatada_cache_timeout
+        return sorted(self.db_engine_spec.get_schema_names(
+            inspector=self.inspector,
+            enable_cache=enable_cache,
+            cache_timeout=schema_cache_timeout,
+            db_id=self.id,
+            force=force_refresh))
 
     @property
     def db_engine_spec(self):