Merge branch 'master' into john-bodley--reformat-updating-md

.github/CODEOWNERS

@@ -1,14 +1,19 @@
 # Notify all committers of DB migration changes, per SIP-59
+
 # https://github.com/apache/superset/issues/13351
+
 /superset/migrations/ @apache/superset-committers
 
 # Notify Preset team when ephemeral env settings are changed
+
 .github/workflows/ecs-task-definition.json @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
 .github/workflows/docker-ephemeral-env.yml @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
 .github/workflows/ephemeral*.yml @robdiciuccio @craig-rueda @rusackas @eschutho @dpgaspar @nytai @mistercrunch
 
 # Notify some committers of changes in the Select component
-/superset-frontend/src/components/Select/ @michael-s-molina @geido
+
+/superset-frontend/src/components/Select/ @michael-s-molina @geido @ktmud
 
 # Notify Helm Chart maintainers about changes in it
+
 /helm/superset/ @craig-rueda @dpgaspar @villebro

.github/ISSUE_TEMPLATE/feature_request.md

@@ -5,14 +5,10 @@ labels: "#enhancement"
 
 ---
 
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+Github Discussions is our new home for discussing features and improvements!
 
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
+https://github.com/apache/superset/discussions/categories/ideas
 
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
+We'd like to keep Github Issues focuses on bugs and SIP's (Superset Improvement Proposals)!
 
-**Additional context**
-Add any other context or screenshots about the feature request here.
+Please note that feature requests opened as Github Issues will be moved to Discussions.

.github/workflows/superset-python-unittest.yml

@@ -51,7 +51,7 @@ jobs:
       - name: Python unit tests
         if: steps.check.outcome == 'failure'
         run: |
-          pytest --durations=0 ./tests/common ./tests/unit_tests --cache-clear
+          pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear
       - name: Upload code coverage
         if: steps.check.outcome == 'failure'
         run: |

.pre-commit-config.yaml

@@ -51,3 +51,9 @@ repos:
       - id: prettier
         args: ['--ignore-path=./superset-frontend/.prettierignore']
         files: 'superset-frontend'
+  # blacklist unsafe functions like make_url (see #19526)
+  - repo: https://github.com/skorokithakis/blacklist-pre-commit-hook
+    rev: e2f070289d8eddcaec0b580d3bde29437e7c8221
+    hooks:
+      - id: blacklist
+        args: ["--blacklisted-names=make_url", "--ignore=tests/"]

README.md

@@ -65,7 +65,7 @@ Superset provides:
 
 **Large Gallery of Visualizations**
 
-<kbd><a href="https://superset.apache.org/gallery"><img title="Gallery" src="superset-frontend/src/assets/images/screenshots/gallery.jpg"/></a></kbd><br/>
+<kbd><img title="Gallery" src="superset-frontend/src/assets/images/screenshots/gallery.jpg"/></kbd><br/>
 
 **Craft Beautiful, Dynamic Dashboards**
 

RELEASING/Dockerfile.from_local_tarball

@@ -34,7 +34,7 @@ RUN apt-get install -y build-essential libssl-dev \
 
 # Install nodejs for custom build
 # https://nodejs.org/en/download/package-manager/
-RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - \
+RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - \
     && apt-get install -y nodejs
 
 RUN mkdir -p /home/superset

RELEASING/Dockerfile.from_svn_tarball

@@ -34,7 +34,7 @@ RUN apt-get install -y build-essential libssl-dev \
 
 # Install nodejs for custom build
 # https://nodejs.org/en/download/package-manager/
-RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - \
+RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - \
     && apt-get install -y nodejs
 
 RUN mkdir -p /home/superset

RELEASING/README.md

@@ -30,6 +30,7 @@ partaking in the process should join the channel.
 
 ## Release notes for recent releases
 
+- [1.5](release-notes-1-5/README.md)
 - [1.4](release-notes-1-4/README.md)
 - [1.3](release-notes-1-3/README.md)
 - [1.2](release-notes-1-2/README.md)

RELEASING/release-notes-1-5/README.md

@@ -0,0 +1,142 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Release Notes for Superset 1.5
+
+Superset 1.5 focuses on polishing the dashboard native filters experience, while
+improving performance and stability. Superset 1.5 is likely the last minor release of
+version 1 of Superset, and will be succeeded by Superset 2.0. The 1.5 branch
+introduces the notion of a Long Term Support (LTS) version of Superset, and will
+receive security and other critical fixes even after Superset 2.x is released.
+Therefore, users will have the choice of staying on the 1.5 branch or upgrading to 2.x
+when available.
+
+- [**User Experience**](#user-facing-features)
+- [**Feature flags**](#feature-flags)
+- [**Database Experience**](#database-experience)
+- [**Breaking Changes and Full Changelog**](#breaking-changes-and-full-changelog)
+
+## User Facing Features
+
+- Complex dashboards with lots of native filters and charts will render considerably
+  faster. See the videos that shows the rendering time of a complex dashboard go from
+  11 to 3 seconds: [#19064](https://github.com/apache/superset/pull/19064). In
+  addition, applying filters and switching tabs is also much smoother.
+- The Native Filter Bar has been redesigned, along with moving the "Apply" and
+  "Clear all" buttons to the bottom:
+
+![Filter bar](media/filter_bar.png)
+
+- Native filters can now be made dependent on multiple filters. This makes it possible
+  to restrict the available values in a filter based on the selection of other filters.
+
+![Dependent filters](media/dependent_filters.png)
+
+- In addition to being able to write Custom SQL for adhoc metrics and filters, the
+  column control now also features a Custom SQL tab. This makes it possible to write
+  custom expressions directly in charts without adding them to the dataset as saved
+  expressions.
+
+![Adhoc columns](media/adhoc_columns.png)
+
+- A new `SupersetMetastoreCache` has been added which makes it possible to cache data
+  in the Superset Metastore without the need for running a dedicated cache like Redis
+  or Memcached. The new cache will be used by default for required caches, but can also
+  be used for caching chart or other data. See the
+  [documentation](https://superset.apache.org/docs/installation/cache#caching) for
+  details on using the new cache.
+- Previously it was possible for Dashboards with lots of filters to cause an error.
+  A similar issue existed on Explore. Now Superset stores Dashboard and Explore state
+  in the cache (as opposed to the URL), eliminating the infamous
+  [Long URL Problem](https://github.com/apache/superset/issues/17086).
+- Previously permanent links to Dashboard and Explore pages were in fact shortened URLS
+  that relied on state being stored in the URL (see Long URL Problem above). In
+  addition, the links used numerical ids and didn't check user permissions making it
+  easy to iterate through links that were stored in the metastore. Now permanent links
+  state is stored as JSON objects in the metastore, making it possible to store
+  arbitrarily large Dashboard and Explore state in permalinks. In addition, the ids
+  are encoded using [`hashids`](https://hashids.org/) and check permissions, making
+  permalink state more secure.
+
+![Dashboard permalink](media/permalink.png)
+
+## Feature flags
+
+- A new feature flag `GENERIC_CHART_AXES` has been added that makes it possible to
+  use a non-temporal x-axis on the ECharts Timeseries chart
+  ([#17917](https://github.com/apache/superset/pull/17917)). When enabled, a new
+  control "X Axis" is added to the control panel of ECharts line, area, bar, step and
+  scatter charts, which makes it possible to use categorical or numerical x-axes on
+  those charts.
+
+![Categorical line chart](media/categorical_line.png)
+
+## Database Experience
+
+- DuckDB: Add support for database:
+  [#19317](https://github.com/apache/superset/pull/19317)
+
+- Kusto: Add support for Azure Data Explorer (Kusto):
+  [#17898](https://github.com/apache/superset/pull/17898)
+
+- Trino: Add server cert support and new auth methods:
+  [#17593](https://github.com/apache/superset/pull/17593) and
+  [#16346](https://github.com/apache/superset/pull/16346)
+
+- Microsoft SQL Server (MSSQL): support using CTEs in virtual tables:
+  [#18567](https://github.com/apache/superset/pull/18567)
+
+- Teradata and MSSQL: add support for TOP limit syntax:
+  [#18746](https://github.com/apache/superset/pull/18746) and
+  [#18240](https://github.com/apache/superset/pull/18240)
+
+- Apache Drill: User impersonation using `drill+sadrill`:
+  [#19252](https://github.com/apache/superset/pull/19252)
+
+## Developer Experience
+
+- `superset-ui` has now been integrated into the Superset codebase as per
+  [SIP-58](https://github.com/apache/superset/issues/13013) dubbed "Monorepo". This
+  makes development of plugins that ship with Superset considerably simpler. In
+  addition, it makes it possible to align `superset-ui` releases with official Superset
+  releases.
+
+## Breaking Changes and Full Changelog
+
+**Breaking Changes**
+
+- Bump `mysqlclient` from v1 to v2:
+  [#17556](https://github.com/apache/superset/pull/17556)
+- Single and double quotes will no longer be removed from filter values:
+  [#17881](https://github.com/apache/superset/pull/17881)
+- Previously `QUERY_COST_FORMATTERS_BY_ENGINE`, `SQL_VALIDATORS_BY_ENGINE` and
+  `SCHEDULED_QUERIES` were expected to be defined in the feature flag dictionary in
+  the `config.py` file. These should now be defined as a top-level config, with the
+  feature flag dictionary being reserved for boolean only values:
+  [#15254](https://github.com/apache/superset/pull/15254)
+- All Superset CLI commands (init, load_examples and etc) require setting the
+  `FLASK_APP` environment variable (which is set by default when `.flaskenv` is loaded):
+  [#17539](https://github.com/apache/superset/pull/17539)
+
+**Changelog**
+
+To see the complete changelog in this release, head to
+[CHANGELOG.MD](https://github.com/apache/superset/blob/1.5/CHANGELOG.md).
+As mentioned earlier, this release has a MASSIVE amount of bug fixes. The full
+changelog lists all of them!

UPDATING.md

@@ -26,6 +26,7 @@ assists people when migrating to a new version.
 
 - [19046](https://github.com/apache/superset/pull/19046): Enables the drag and drop interface in Explore control panel by default. Flips `ENABLE_EXPLORE_DRAG_AND_DROP` and `ENABLE_DND_WITH_CLICK_UX` feature flags to `True`.
 - [18936](https://github.com/apache/superset/pull/18936): Removes legacy SIP-15 interm logic/flags—specifically the `SIP_15_ENABLED`, `SIP_15_GRACE_PERIOD_END`, `SIP_15_DEFAULT_TIME_RANGE_ENDPOINTS`, and `SIP_15_TOAST_MESSAGE` flags. Time range endpoints are no longer configurable and strictly adhere to the `[start, end)` paradigm, i.e., inclusive of the start and exclusive of the end. Additionally this change removes the now obsolete `time_range_endpoints` from the form-data and resulting in the cache being busted.
+- [19570](https://github.com/apache/superset/pull/19570): makes [sqloxide](https://pypi.org/project/sqloxide/) optional so the SIP-68 migration can be run on aarch64. If the migration is taking too long installing sqloxide manually should improve the performance.
 
 ### Breaking Changes
 

docker/docker-init.sh

@@ -41,7 +41,7 @@ ADMIN_PASSWORD="admin"
 # If Cypress run – overwrite the password for admin and export env variables
 if [ "$CYPRESS_CONFIG" == "true" ]; then
     ADMIN_PASSWORD="general"
-    export SUPERSET_CONFIG=tests.superset_test_config
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
     export SUPERSET_TESTENV=true
     export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
 fi

docs/docs/installation/installing-superset-from-scratch.mdx

@@ -18,13 +18,13 @@ level dependencies.
 The following command will ensure that the required dependencies are installed:
 
 ```
-sudo apt-get install build-essential libssl-dev libffi-dev python-dev python-pip libsasl2-dev libldap2-dev
+sudo apt-get install build-essential libssl-dev libffi-dev python-dev python-pip libsasl2-dev libldap2-dev default-libmysqlclient-dev
 ```
 
 In Ubuntu 20.04 the following command will ensure that the required dependencies are installed:
 
 ```
-sudo apt-get install build-essential libssl-dev libffi-dev python3-dev python3-pip libsasl2-dev libldap2-dev
+sudo apt-get install build-essential libssl-dev libffi-dev python3-dev python3-pip libsasl2-dev libldap2-dev default-libmysqlclient-dev
 ```
 
 **Fedora and RHEL-derivative Linux distributions**

docs/docs/installation/sql-templating.mdx

@@ -33,7 +33,7 @@ For example, to add a time range to a virtual dataset, you can write the followi
 SELECT * from tbl where dttm_col > '{{ from_dttm }}' and dttm_col < '{{ to_dttm }}'
 ```
 
-To add custom functionality to the Jinja context, you need to to to overload the default Jinja
+To add custom functionality to the Jinja context, you need to overload the default Jinja
 context in your environment by defining the `JINJA_CONTEXT_ADDONS` in your superset configuration
 (`superset_config.py`). Objects referenced in this dictionary are made available for users to use
 where the Jinja context is made available.
@@ -206,10 +206,12 @@ Here's a concrete example:
 SELECT action, count(*) as times
 FROM logs
 WHERE
-    action in ({{ "'" + "','".join(filter_values('action_type')) + "'" }})
+    action in {{ filter_values('action_type')|where_in }}
 GROUP BY action
 ```
 
+There `where_in` filter converts the list of values from `filter_values('action_type')` into a string suitable for an `IN` expression.
+
 **Filters for a Specific Column**
 
 The `{{ get_filters() }}` macro returns the filters applied to a given column. In addition to
@@ -243,7 +245,7 @@ Here's a concrete example:
 
     {%- if filter.get('op') == 'IN' -%}
         AND
-        full_name IN ( {{ "'" + "', '".join(filter.get('val')) + "'" }} )
+        full_name IN {{ filter.get('val')|where_in }}
     {%- endif -%}
 
     {%- if filter.get('op') == 'LIKE' -%}

docs/yarn.lock

@@ -7429,9 +7429,9 @@ minimatch@^3.0.4:
     brace-expansion "^1.1.7"
 
 minimist@^1.2.0, minimist@^1.2.5:
-  version "1.2.5"
-  resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz"
-  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
+  version "1.2.6"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
+  integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
 
 mkdirp@^0.5.5, mkdirp@~0.5.1:
   version "0.5.5"
@@ -7441,9 +7441,9 @@ mkdirp@^0.5.5, mkdirp@~0.5.1:
     minimist "^1.2.5"
 
 moment@^2.24.0, moment@^2.25.3:
-  version "2.29.1"
-  resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.1.tgz#b2be769fa31940be9eeea6469c075e35006fa3d3"
-  integrity sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==
+  version "2.29.2"
+  resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.2.tgz#00910c60b20843bcba52d37d58c628b47b1f20e4"
+  integrity sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==
 
 ms@2.0.0:
   version "2.0.0"
@@ -7529,9 +7529,9 @@ node-fetch@2.6.1:
   integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==
 
 node-forge@^1.2.0:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.2.1.tgz#82794919071ef2eb5c509293325cec8afd0fd53c"
-  integrity sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
+  integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==
 
 node-releases@^2.0.1:
   version "2.0.1"

requirements/development.in

@@ -18,7 +18,7 @@
 -r base.in
 flask-cors>=2.0.0
 mysqlclient>=2.1.0
-pillow>=8.3.2,<10
+pillow>=9.0.1,<10
 pydruid>=0.6.1,<0.7
 pyhive[hive]>=0.6.1
 psycopg2-binary==2.9.1

requirements/development.txt

@@ -1,4 +1,4 @@
-# SHA1:cb189e676baa2e397294abb48eaefba5bf408522
+# SHA1:b4a3e0dd12a4937fc5a21bdbf63644be9222c65f
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -40,7 +40,7 @@ mysqlclient==2.1.0
     # via -r requirements/development.in
 openpyxl==3.0.7
     # via tabulator
-pillow==9.0.0
+pillow==9.1.0
     # via -r requirements/development.in
 progress==1.6
     # via -r requirements/development.in

requirements/testing.in

@@ -36,6 +36,7 @@ pytest
 pytest-cov
 statsd
 pytest-mock
+sqloxide
 # DB dependencies
 -e file:.[bigquery]
 -e file:.[trino]