Merge branch 'master' into danielgaspar/sc-42037/migrate-superset-val…

…idate-sql-json-to-api

.github/workflows/bashlib.sh

@@ -183,7 +183,7 @@ cypress-run-all() {
   nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
-  cypress-run "*/**/*"
+  cypress-run "*/**/!(*.applitools.test.ts)"
 
   # After job is done, print out Flask log for debugging
   say "::group::Flask log for default run"
@@ -198,7 +198,7 @@ cypress-run-all() {
   nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
-  cypress-run "sqllab/*" "Backend persist"
+  cypress-run "sqllab/!(*.applitools.test.ts)" "Backend persist"
 
   # Upload code coverage separately so each page can have separate flags
   # -c will clean existing coverage reports, -F means add flags
@@ -212,3 +212,23 @@ cypress-run-all() {
   # make sure the program exits
   kill $flaskProcessId
 }
+
+cypress-run-applitools() {
+  local flasklog="${HOME}/flask.log"
+  local port=8081
+  export CYPRESS_BASE_URL="http://localhost:${port}"
+
+  nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
+  local flaskProcessId=$!
+
+  cypress-run "*/**/*.applitools.test.ts"
+
+  codecov -c -F "cypress" || true
+
+  say "::group::Flask log for default run"
+  cat "$flasklog"
+  say "::endgroup::"
+
+  # make sure the program exits
+  kill $flaskProcessId
+}

.github/workflows/superset-applitool-cypress.yml

@@ -0,0 +1,113 @@
+name: Applitools Cypress
+
+on: pull_request_target
+
+jobs:
+  cypress-applitools:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: ["chrome"]
+    env:
+      FLASK_ENV: development
+      SUPERSET_CONFIG: tests.integration_tests.superset_test_config
+      SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
+      PYTHONPATH: ${{ github.workspace }}
+      REDIS_PORT: 16379
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      APPLITOOLS_APP_NAME: Superset
+      APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
+      APPLITOOLS_BATCH_ID: ${{ github.sha }}
+      APPLITOOLS_BATCH_NAME: Superset Cypress
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_USER: superset
+          POSTGRES_PASSWORD: superset
+        ports:
+          - 15432:5432
+      redis:
+        image: redis:5-alpine
+        ports:
+          - 16379:6379
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
+      - name: Check if python or frontend changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python frontend
+      - name: Setup Python
+        if: steps.check.outcome == 'failure'
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: OS dependencies
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: apt-get-install
+      - name: Install python dependencies
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: |
+            pip-upgrade
+            pip install -r requirements/testing.txt
+      - name: Setup postgres
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: setup-postgres
+      - name: Import test data
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: testdata
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: "16"
+      - name: Install npm dependencies
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: npm-install
+      - name: Build javascript packages
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: build-instrumented-assets
+      - name: Install cypress
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: cypress-install
+      - name: Run Applitools Cypress
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        env:
+          CYPRESS_BROWSER: ${{ matrix.browser }}
+        with:
+          run: cypress-run-applitools
+  batch-completion-notification:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    needs: cypress-applitools
+    steps:
+      - name: Update Applitools batch status
+        uses: wei/curl@v1.1.1
+        env:
+          APPLITOOLS_BATCH_ID: ${{ github.sha }}
+          APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
+        with:
+          args: -d "" -X POST https://eyesapi.applitools.com/api/externals/github/servers/github.com/commit/${{ env.APPLITOOLS_BATCH_ID }}/complete?apiKey=${{ env.APPLITOOLS_API_KEY }}

.github/workflows/superset-applitools-storybook.yml

@@ -0,0 +1,34 @@
+name: Applitools Storybook
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+
+env:
+  APPLITOOLS_APP_NAME: Superset
+  APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
+  APPLITOOLS_BATCH_ID: ${{ github.sha }}
+  APPLITOOLS_BATCH_NAME: Superset Storybook
+
+jobs:
+  cron:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node: [16]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Node.js
+        uses: actions/setup-node@v3.1.1
+        with:
+          node-version: ${{ matrix.node }}
+      - name: Install Chrome-related packages
+        run: sudo apt-get update -y && sudo apt-get -y install gconf-service ca-certificates libxshmfence-dev fonts-liberation libappindicator3-1 libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgbm1 libgcc1 libgconf-2-4 libglib2.0-0 libgdk-pixbuf2.0-0 libgtk-3-0 libnspr4 libnss3 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release xdg-utils libappindicator1
+      - name: Install NPM dependencies
+        working-directory: ./superset-frontend
+        run: |
+          npm ci
+          ls ./node_modules/puppeteer/.local-chromium
+      - name: Run Applitools Eyes-Storybook
+        working-directory: ./superset-frontend
+        run: npx eyes-storybook -u https://superset-storybook.netlify.app/

.github/workflows/superset-e2e.yml

@@ -92,7 +92,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v2
         with:
-          node-version:  '16'
+          node-version: "16"
       - name: Install npm dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies

RESOURCES/INTHEWILD.md

@@ -126,7 +126,7 @@ Join our growing community!
 
 ### Energy
 - [Airboxlab](https://foobot.io) [@antoine-galataud]
-- [DouroECI](http://douroeci.com/en/) [@nunohelibeires]
+- [DouroECI](https://www.douroeci.com/) [@nunohelibeires]
 - [Safaricom](https://www.safaricom.co.ke/) [@mmutiso]
 - [Scoot](https://scoot.co/) [@haaspt]
 

UPDATING.md

@@ -30,6 +30,7 @@ assists people when migrating to a new version.
 
 ### Breaking Changes
 
+- [19770](https://github.com/apache/superset/pull/19770): As per SIPs 11 and 68, the native NoSQL Druid connector is deprecated and has been removed. Druid is still supported through SQLAlchemy via pydruid. The config keys `DRUID_IS_ACTIVE` and `DRUID_METADATA_LINKS_ENABLED` have also been removed.
 - [19274](https://github.com/apache/superset/pull/19274): The `PUBLIC_ROLE_LIKE_GAMMA` config key has been removed, set `PUBLIC_ROLE_LIKE = "Gamma"` to have the same functionality.
 - [19273](https://github.com/apache/superset/pull/19273): The `SUPERSET_CELERY_WORKERS` and `SUPERSET_WORKERS` config keys has been removed. Configure Celery directly using `CELERY_CONFIG` on Superset.
 - [19262](https://github.com/apache/superset/pull/19262): Per [SIP-11](https://github.com/apache/superset/issues/6032) and [SIP-68](https://github.com/apache/superset/issues/14909) the native NoSQL Druid connector is deprecated and will no longer be supported. Druid SQL is still [supported](https://superset.apache.org/docs/databases/druid).

docs/docs/databases/trino.mdx

@@ -9,19 +9,93 @@ version: 1
 
 Supported trino version 352 and higher
 
-The [sqlalchemy-trino](https://pypi.org/project/sqlalchemy-trino/) library is the recommended way to connect to Trino through SQLAlchemy.
+### Connection String
+The connection string format is as follows:
+```
+trino://{username}:{password}@{hostname}:{port}/{catalog}
+```
+
+If you are running Trino with docker on local machine, please use the following connection URL
+```
+trino://trino@host.docker.internal:8080
+```
+
+### Authentications
+#### 1. Basic Authentication
+You can provide `username`/`password` in the connection string or in the `Secure Extra` field at `Advanced / Security`
+* In Connection String
+    ```
+    trino://{username}:{password}@{hostname}:{port}/{catalog}
+    ```
 
-The expected connection string is formatted as follows:
+* In `Secure Extra` field
+    ```json
+    {
+        "auth_method": "basic",
+        "auth_params": {
+            "username": "<username>",
+            "password": "<password>"
+        }
+    }
+    ```
 
+NOTE: if both are provided, `Secure Extra` always takes higher priority.
+
+#### 2. Kerberos Authentication
+In `Secure Extra` field, config as following example:
+```json
+{
+    "auth_method": "kerberos",
+    "auth_params": {
+        "service_name": "superset",
+        "config": "/path/to/krb5.config",
+        ...
+    }
+}
 ```
-trino://{username}:{password}@{hostname}:{port}/{catalog}
+
+All fields in `auth_params` are passed directly to the [`KerberosAuthentication`](https://github.com/trinodb/trino-python-client/blob/0.306.0/trino/auth.py#L40) class.
+
+#### 3. JWT Authentication
+Config `auth_method` and provide token in `Secure Extra` field
+```json
+{
+    "auth_method": "jwt",
+    "auth_params": {
+        "token": "<your-jwt-token>"
+    }
+}
 ```
 
-If you are running trino with docker on local machine please use the following connection URL
+#### 4. Custom Authentication
+To use custom authentication, first you need to add it into
+`ALLOWED_EXTRA_AUTHENTICATIONS` allow list in Superset config file:
+```python
+from your.module import AuthClass
+from another.extra import auth_method
 
+ALLOWED_EXTRA_AUTHENTICATIONS: Dict[str, Dict[str, Callable[..., Any]]] = {
+    "trino": {
+        "custom_auth": AuthClass,
+        "another_auth_method": auth_method,
+    },
+}
 ```
-trino://trino@host.docker.internal:8080
+
+Then in `Secure Extra` field:
+```json
+{
+    "auth_method": "custom_auth",
+    "auth_params": {
+        ...
+    }
+}
 ```
 
-Reference:
-[Trino-Superset-Podcast](https://trino.io/episodes/12.html)
+You can also use custom authentication by providing reference to your `trino.auth.Authentication` class
+or factory function (which returns an `Authentication` instance) to `auth_method`.
+
+All fields in `auth_params` are passed directly to your class/function.
+
+**Reference**:
+* [Trino-Superset-Podcast](https://trino.io/episodes/12.html)

setup.py

@@ -64,7 +64,7 @@ def get_git_sha() -> str:
     zip_safe=False,
     entry_points={
         "console_scripts": ["superset=superset.cli.main:superset"],
-        "sqlalchemy.dialects": ["trinonative = sqlalchemy_trino.dialect:TrinoDialect"],
+        "sqlalchemy.dialects": ["trinonative = trino.sqlalchemy.dialect:TrinoDialect"],
     },
     install_requires=[
         "backoff>=1.8.0",
@@ -152,7 +152,7 @@ def get_git_sha() -> str:
         "pinot": ["pinotdb>=0.3.3, <0.4"],
         "postgres": ["psycopg2-binary==2.9.1"],
         "presto": ["pyhive[presto]>=0.4.0"],
-        "trino": ["sqlalchemy-trino>=0.2"],
+        "trino": ["trino>=0.313.0"],
         "prophet": ["prophet>=1.0.1, <1.1", "pystan<3.0"],
         "redshift": ["sqlalchemy-redshift>=0.8.1, < 0.9"],
         "rockset": ["rockset>=0.8.10, <0.9"],

superset-frontend/applitools.config.js

@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+module.exports = {
+  apiKey: process.env.APPLITOOLS_API_KEY,
+  batchId: process.env.APPLITOOLS_BATCH_ID,
+  batchName: process.env.APPLITOOLS_BATCH_NAME,
+  puppeteerOptions: {
+    headless: true,
+    args: ['--no-sandbox', '--disable-setuid-sandbox'],
+  },
+};

superset-frontend/cypress-base/applitools.config.js

@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+module.exports = {
+  apiKey: process.env.APPLITOOLS_API_KEY,
+  batchId: process.env.APPLITOOLS_BATCH_ID,
+  batchName: process.env.APPLITOOLS_BATCH_NAME,
+  browser: [{ width: 1000, height: 660, name: 'chrome' }],
+  failCypressOnDiff: false,
+  isDisabled: false,
+  showLogs: false,
+  testConcurrency: 10,
+};