Merge branch 'master' into embedded

apache · Jan 13, 2022 · d2774e4 · d2774e4
2 parents d705236 + 343d3f8
commit d2774e4
Show file tree

Hide file tree

Showing 733 changed files with 56,754 additions and 18,471 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -16,7 +16,7 @@ coverage:
         target: auto
         threshold: 0%
       core-packages-ts:
-        target: 95%
+        target: 100%
         paths:
           - 'superset-frontend/packages'
           - '!superset-frontend/packages/**/*.jsx'

diff --git a/...superset-ui/.github/workflows/release.yml → .github/workflows/release.yml b/...superset-ui/.github/workflows/release.yml → .github/workflows/release.yml
@@ -53,11 +53,14 @@ jobs:
             ${{ runner.os }}-npm-
 
       - name: Install dependencies
+        working-directory: ./superset-frontend
         run: npm ci
-      - name: Build packages
-        run: npm run build
       - name: Run unit tests
-        run: npm run test
+        working-directory: ./superset-frontend
+        run: npm run test -- plugins packages
+      - name: Build packages
+        working-directory: ./superset-frontend
+        run: npm run plugins:build
 
       - name: Configure npm and git
         run: |
@@ -73,9 +76,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Bump version and publish package(s)
+        working-directory: ./superset-frontend
         run: |
           git tag -d `git tag | grep -E '^trigger-'`
-          npm run ci:release-from-tag
+          npm run plugins:release-from-tag
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml
@@ -66,14 +66,17 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
-          cache-dependency-path: 'requirements/integration.txt'
+          cache-dependency-path: |
+            requirements/base.txt
+            requirements/integration.txt
       - name: Install dependencies
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             apt-get-install
             pip-upgrade
             pip install wheel
+            pip install -r requirements/base.txt
             pip install -r requirements/integration.txt
       - name: pre-commit
         run: pre-commit run --all-files

diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml
@@ -80,7 +80,7 @@ jobs:
       - name: Python unit tests (PostgreSQL)
         if: steps.check.outcome == 'failure'
         run: |
-          ./scripts/python_tests.sh
+          ./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
       - name: Upload code coverage
         if: steps.check.outcome == 'failure'
         run: |
@@ -158,7 +158,7 @@ jobs:
       - name: Python unit tests (PostgreSQL)
         if: steps.check.outcome == 'failure'
         run: |
-          ./scripts/python_tests.sh
+          ./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
       - name: Upload code coverage
         if: steps.check.outcome == 'failure'
         run: |

diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml
@@ -51,7 +51,7 @@ jobs:
       - name: Python unit tests
         if: steps.check.outcome == 'failure'
         run: |
-          pytest ./tests/unit_tests --cache-clear
+          pytest --durations=0 ./tests/common ./tests/unit_tests --cache-clear
       - name: Upload code coverage
         if: steps.check.outcome == 'failure'
         run: |

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -69,10 +69,10 @@ little bit helps, and credit will always be given.
   - [Git Hooks](#git-hooks)
   - [Linting](#linting)
   - [Conventions](#conventions)
-    - [Python](#python)
+    - [Python](#python-conventions)
   - [Typing](#typing)
-    - [Python](#python-1)
-    - [TypeScript](#typescript)
+    - [Python](#python-typing)
+    - [TypeScript](#typeScript-typing)
   - [Testing](#testing)
     - [Python Testing](#python-testing)
     - [Frontend Testing](#frontend-testing)
@@ -82,12 +82,10 @@ little bit helps, and credit will always be given.
     - [Storybook](#storybook)
   - [Translating](#translating)
     - [Enabling language selection](#enabling-language-selection)
-    - [Extracting new strings for translation](#extracting-new-strings-for-translation)
     - [Updating language files](#updating-language-files)
     - [Creating a new language dictionary](#creating-a-new-language-dictionary)
   - [Tips](#tips)
     - [Adding a new datasource](#adding-a-new-datasource)
-    - [Improving visualizations](#improving-visualizations)
     - [Visualization Plugins](#visualization-plugins)
     - [Adding a DB migration](#adding-a-db-migration)
     - [Merging DB migrations](#merging-db-migrations)
@@ -117,17 +115,6 @@ Here's a list of repositories that contain Superset-related packages:
   also includes Superset's main TypeScript/JavaScript bundles and react apps under
   the [superset-frontend](https://github.com/apache/superset/tree/master/superset-frontend)
   folder.
-- [apache-superset/superset-ui](https://github.com/apache-superset/superset-ui)
-  contains core Superset's
-  [npm packages](https://github.com/apache-superset/superset-ui/tree/master/packages).
-  These packages are shared across the React apps in the main repository,
-  and in visualization plugins.
-- [apache-superset/superset-ui-plugins](https://github.com/apache-superset/superset-ui-plugins)
-  contains the code for the default visualizations that ship with Superset
-  and are maintained by the core community.
-- [apache-superset/superset-ui-plugins-deckgl](https://github.com/apache-superset/superset-ui-plugins-deckgl)
-  contains the code for the geospatial visualizations that ship with Superset
-  and are maintained by the core community.
 - [github.com/apache-superset](https://github.com/apache-superset) is the
   Github organization under which we manage Superset-related
   small tools, forks and Superset-related experimental ideas.
@@ -489,6 +476,20 @@ $ python3 -m pip install -r requirements/integration.txt
 $ pip-compile-multi --no-upgrade
 ```
 
+When upgrading the version number of a single package, you should run `pip-compile-multi` with the `-P` flag:
+
+```bash
+$ pip-compile-multi -P my-package
+```
+
+To bring all dependencies up to date as per the restrictions defined in `setup.py` and `requirements/*.in`, run pip-compile-multi` without any flags:
+
+```bash
+$ pip-compile-multi
+```
+
+This should be done periodically, but it is rcommended to do thorough manual testing of the application to ensure no breaking changes have been introduced that aren't caught by the unit and integration tests.
+
 #### Logging to the browser console
 
 This feature is only available on Python 3. When debugging your application, you can have the server logs sent directly to the browser console using the [ConsoleLog](https://github.com/betodealmeida/consolelog) package. You need to mutate the app, by adding the following to your `config.py` or `superset_config.py`:
@@ -574,7 +575,7 @@ The dev server by default starts at `http://localhost:9000` and proxies the back
 npm run dev-server
 
 # Run the dev server on a non-default port
-npm run dev-server -- --devserverPort=9001
+npm run dev-server -- --port=9001
 
 # Proxy backend requests to a Flask server running on a non-default port
 npm run dev-server -- --supersetPort=8081
@@ -683,7 +684,7 @@ If using the eslint extension with vscode, put the following in your workspace `
 
 ## Conventions
 
-### Python
+### Python Conventions
 
 Parameters in the `config.py` (which are accessible via the Flask app.config dictionary) are assumed to always be defined and thus should be accessed directly via,
 
@@ -701,7 +702,7 @@ or similar as the later will cause typing issues. The former is of type `List[Ca
 
 ## Typing
 
-### Python
+### Python Typing
 
 To ensure clarity, consistency, all readability, _all_ new functions should use
 [type hints](https://docs.python.org/3/library/typing.html) and include a
@@ -728,7 +729,7 @@ def sqrt(x: Union[float, int]) -> Union[float, int]:
     return math.sqrt(x)
 ```
 
-### TypeScript
+### TypeScript Typing
 
 TypeScript is fully supported and is the recommended language for writing all new frontend components. When modifying existing functions/components, migrating to TypeScript is appreciated, but not required. Examples of migrating functions/components to TypeScript can be found in [#9162](https://github.com/apache/superset/pull/9162) and [#9180](https://github.com/apache/superset/pull/9180).
 
@@ -964,7 +965,7 @@ You are now ready to attach a debugger to the process. Using VSCode you can conf
 }
 ```
 
-VSCode will not stop on breakpoints right away. We've attached to PID 6 however it does not yet know of any sub-processes. In order to "wakeup" the debugger you need to modify a python file. This will trigger Flask to reload the code and create a new sub-process. This new sub-process will be detected by VSCode and breakpoints will be activated.
+VSCode will not stop on breakpoints right away. We've attached to PID 6 however it does not yet know of any sub-processes. In order to "wake up" the debugger you need to modify a python file. This will trigger Flask to reload the code and create a new sub-process. This new sub-process will be detected by VSCode and breakpoints will be activated.
 
 ### Debugging Server App in Kubernetes Environment
 
@@ -1089,7 +1090,7 @@ pip install -r superset/translations/requirements.txt
 pybabel init -i superset/translations/messages.pot -d superset/translations -l LANGUAGE_CODE
 ```
 
-Then, [extract strings for the new language](#extracting-new-strings-for-translation).
+Then, [Updating language files](#updating-language-files).
 
 ## Tips
 
@@ -1111,44 +1112,13 @@ Then, [extract strings for the new language](#extracting-new-strings-for-transla
 
    This means it'll register MyDatasource and MyOtherDatasource in superset.my_models module in the source registry.
 
-### Improving visualizations
-
-To edit the frontend code for visualizations, you will have to check out a copy of [apache-superset/superset-ui](https://github.com/apache-superset/superset-ui):
-
-```bash
-git clone https://github.com/apache-superset/superset-ui.git
-cd superset-ui
-yarn
-yarn build
-```
-
-Then use `npm link` to create symlinks of the plugins/superset-ui packages you want to edit in `superset-frontend/node_modules`:
-
-```bash
-# Since npm 7, you have to install plugin dependencies separately, too
-cd ../../superset-ui/plugins/[PLUGIN NAME] && npm install --legacy-peer-deps
-
-cd superset/superset-frontend
-npm link ../../superset-ui/plugins/[PLUGIN NAME]
-
-# Or to link all core superset-ui and plugin packages:
-# npm link ../../superset-ui/{packages,plugins}/*
-
-# Start developing
-npm run dev-server
-```
-
-When `superset-ui` packages are linked with `npm link`, the dev server will automatically load a package's source code from its `/src` directory, instead of the built modules in `lib/` or `esm/`.
-
-Note that every time you do `npm install`, you will lose the symlink(s) and may have to run `npm link` again.
-
 ### Visualization Plugins
 
 The topic of authoring new plugins, whether you'd like to contribute
 it back or not has been well documented in the
 [So, You Want to Build a Superset Viz Plugin...](https://preset.io/blog/2020-07-02-hello-world/) blog post
 
-To contribute a plugin to Superset-UI, your plugin must meet the following criteria:
+To contribute a plugin to Superset, your plugin must meet the following criteria:
 
 - The plugin should be applicable to the community at large, not a particularly specialized use case
 - The plugin should be written with TypeScript
@@ -1348,7 +1318,7 @@ Note not all fields are correctly categorized. The fields vary based on visualiz
 | Field             | Type     | Notes                               |
 | ----------------- | -------- | ----------------------------------- |
 | `database_name`   | _string_ | _Deprecated?_                       |
-| `datasource`      | _string_ | `<datasouce_id>__<datasource_type>` |
+| `datasource`      | _string_ | `<datasource_id>__<datasource_type>` |
 | `datasource_id`   | _string_ | _Deprecated?_ See `datasource`      |
 | `datasource_name` | _string_ | _Deprecated?_                       |
 | `datasource_type` | _string_ | _Deprecated?_ See `datasource`      |
@@ -1402,7 +1372,7 @@ Note not all fields are correctly categorized. The fields vary based on visualiz
 | `columns`                                                                                              | _array(string)_                                   | The **Breakdowns** widget                         |
 | `groupby`                                                                                              | _array(string)_                                   | The **Group by** or **Series** widget             |
 | `limit`                                                                                                | _number_                                          | The **Series Limit** widget                       |
-| `metric`<br>`metric_2`<br>`metrics`<br>`percent_mertics`<br>`secondary_metric`<br>`size`<br>`x`<br>`y` | _string_,_object_,_array(string)_,_array(object)_ | The metric(s) depending on the visualization type |
+| `metric`<br>`metric_2`<br>`metrics`<br>`percent_metrics`<br>`secondary_metric`<br>`size`<br>`x`<br>`y` | _string_,_object_,_array(string)_,_array(object)_ | The metric(s) depending on the visualization type |
 | `order_asc`                                                                                            | _boolean_                                         | The **Sort Descending** widget                    |
 | `row_limit`                                                                                            | _number_                                          | The **Row limit** widget                          |
 | `timeseries_limit_metric`                                                                              | _object_                                          | The **Sort By** widget                            |

diff --git a/Dockerfile b/Dockerfile
@@ -104,7 +104,8 @@ COPY superset /app/superset
 COPY setup.py MANIFEST.in README.md /app/
 RUN cd /app \
         && chown -R superset:superset * \
-        && pip install -e .
+        && pip install -e . \
+        && flask fab babel-compile --target superset/translations
 
 COPY ./docker/run-server.sh /usr/bin/
 

diff --git a/Makefile b/Makefile
@@ -46,6 +46,9 @@ superset:
 	# Load some data to play with
 	superset load-examples
 
+	# Install node packages
+	cd superset-frontend; npm install
+
 update: update-py update-js
 
 update-py: