diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index a73debe69d23..c7ebc2a8a261 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -3,6 +3,6 @@
/superset/migrations/ @apache/superset-committers
# Notify Preset team when ephemeral env settings are changed
-.github/workflows/ecs-task-definition.json @robdiciuccio @craig-rueda @benjreinhart
-.github/workflows/docker-ephemeral-env.yml @robdiciuccio @craig-rueda @benjreinhart
-.github/workflows/ephemeral*.yml @robdiciuccio @craig-rueda @benjreinhart
+.github/workflows/ecs-task-definition.json @robdiciuccio @craig-rueda @willbarrett @rusackas @eschutho @dpgaspar @nytai @mistercrunch
+.github/workflows/docker-ephemeral-env.yml @robdiciuccio @craig-rueda @willbarrett @rusackas @eschutho @dpgaspar @nytai @mistercrunch
+.github/workflows/ephemeral*.yml @robdiciuccio @craig-rueda @willbarrett @rusackas @eschutho @dpgaspar @nytai @mistercrunch
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index f57658a6e071..12fee9c96759 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -33,6 +33,7 @@ If applicable, add screenshots to help explain your problem.
- superset version: `superset version`
- python version: `python --version`
- node.js version: `node -v`
+- any feature flags active:
### Checklist
diff --git a/.github/ISSUE_TEMPLATE/sip.md b/.github/ISSUE_TEMPLATE/sip.md
index d6a668f7a34b..6c526d6d1fa0 100644
--- a/.github/ISSUE_TEMPLATE/sip.md
+++ b/.github/ISSUE_TEMPLATE/sip.md
@@ -6,9 +6,9 @@ labels: "#SIP"
---
*Please make sure you are familiar with the SIP process documented*
-(here)[https://github.com/apache/superset/issues/5602]
+(here)[https://github.com/apache/superset/issues/5602]. The SIP number should be the next number after the latest SIP listed [here](https://github.com/apache/superset/issues?q=is%3Aissue+label%3Asip).
-## [SIP] Proposal for XXX
+## [SIP-\
It is generally used to pass dashboard filter parameters to a chart.
It can be used for appending additional filters to a chart that has been saved with its own filters on an ad-hoc basis if the chart is being used as a standalone widget.
For implementation examples see : [utils test.py](https://github.com/apache/superset/blob/66a4c94a1ed542e69fe6399bab4c01d4540486cf/tests/utils_tests.py#L181)
For insight into how superset processes the contents of this parameter see: [exploreUtils/index.js](https://github.com/apache/superset/blob/93c7f5bb446ec6895d7702835f3157426955d5a9/superset-frontend/src/explore/exploreUtils/index.js#L159) |
| `columns` | _array(string)_ | The **Breakdowns** widget |
| `groupby` | _array(string)_ | The **Group by** or **Series** widget |
| `limit` | _number_ | The **Series Limit** widget |
@@ -1410,7 +1434,6 @@ Note the `y_axis_format` is defined under various section for some charts.
| `default_filters` | _N/A_ | |
| `entity` | _N/A_ | |
| `expanded_slices` | _N/A_ | |
-| `extra_filters` | _N/A_ | |
| `filter_immune_slice_fields` | _N/A_ | |
| `filter_immune_slices` | _N/A_ | |
| `flt_col_0` | _N/A_ | |
diff --git a/Dockerfile b/Dockerfile
index 6ca89e889fd7..c73e6cad6198 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -64,7 +64,7 @@ RUN /frontend-mem-nag.sh \
# Next, copy in the rest and let webpack do its thing
COPY ./superset-frontend /app/superset-frontend
-# This is BY FAR the most expensive step (thanks Terser!)
+# This seems to be the most expensive step
RUN cd /app/superset-frontend \
&& npm run ${BUILD_CMD} \
&& rm -rf node_modules
diff --git a/INSTALL.md b/INSTALL.md
index c96ba33a8a65..afe091241f8a 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -18,6 +18,6 @@ under the License.
-->
# INSTALL / BUILD instructions for Apache Superset
-At this time, the docker file at RELEASING/Dockerfile.from_tarball
+At this time, the docker file at RELEASING/Dockerfile.from_local_tarball
constitutes the recipe on how to get to a working release from a source
release tarball.
diff --git a/Makefile b/Makefile
index a51b96fdb23f..ef0d3edbec13 100644
--- a/Makefile
+++ b/Makefile
@@ -15,6 +15,9 @@
# limitations under the License.
#
+# Python version installed; we need 3.8 or 3.7
+PYTHON=`command -v python3.8 || command -v python3.7`
+
.PHONY: install superset venv pre-commit
install: superset pre-commit
@@ -55,12 +58,13 @@ update-py:
update-js:
# Install js packages
- cd superset-frontend; npm install
+ cd superset-frontend; npm ci
venv:
# Create a virtual environment and activate it (recommended)
- python3 -m venv venv # setup a python3 virtualenv
- source venv/bin/activate
+ if ! [ -x "${PYTHON}" ]; then echo "You need Python 3.7 or 3.8 installed"; exit 1; fi
+ test -d venv || ${PYTHON} -m venv venv # setup a python3 virtualenv
+ . venv/bin/activate
pre-commit:
# setup pre commit dependencies
@@ -72,5 +76,14 @@ format: py-format js-format
py-format: pre-commit
pre-commit run black --all-files
+py-lint: pre-commit
+ pylint -j 0 superset
+
js-format:
cd superset-frontend; npm run prettier
+
+flask-app:
+ flask run -p 8088 --with-threads --reload --debugger
+
+node-app:
+ cd superset-frontend; npm run dev-server
diff --git a/README.md b/README.md
index d1cb206a65c0..a2992d1a341f 100644
--- a/README.md
+++ b/README.md
@@ -121,6 +121,7 @@ Here are some of the major database solutions that are supported:
+
Optional CA_BUNDLE contents to validate HTTPS requests. Only available on certain database engines.
", "nullable": true, @@ -2404,8 +2743,7 @@ } }, "required": [ - "database_name", - "sqlalchemy_uri" + "database_name" ], "type": "object" }, @@ -2441,6 +2779,10 @@ "nullable": true, "type": "integer" }, + "configuration_method": { + "default": "sqlalchemy_form", + "description": "Configuration_method is used on the frontend to inform the backend whether to explode parameters or to provide only a sqlalchemy_uri." + }, "database_name": { "description": "A database name to identify this connection.", "maxLength": 250, @@ -2453,6 +2795,11 @@ "nullable": true, "type": "string" }, + "engine": { + "description": "SQLAlchemy engine to use", + "nullable": true, + "type": "string" + }, "expose_in_sqllab": { "description": "Expose this database to SQLLab", "type": "boolean" @@ -2472,6 +2819,11 @@ "description": "If Presto, all the queries in SQL Lab are going to be executed as the currently logged on user who must have permission to run them.Optional CA_BUNDLE contents to validate HTTPS requests. Only available on certain database engines.
", "nullable": true, @@ -2481,7 +2833,6 @@ "description": "Refer to the SqlAlchemy docs for more information on how to structure your URI.
", "maxLength": 1024, "minLength": 0, - "nullable": true, "type": "string" } }, @@ -2489,6 +2840,10 @@ }, "DatabaseTestConnectionSchema": { "properties": { + "configuration_method": { + "default": "sqlalchemy_form", + "description": "Configuration_method is used on the frontend to inform the backend whether to explode parameters or to provide only a sqlalchemy_uri." + }, "database_name": { "description": "A database name to identify this connection.", "maxLength": 250, @@ -2501,6 +2856,11 @@ "nullable": true, "type": "string" }, + "engine": { + "description": "SQLAlchemy engine to use", + "nullable": true, + "type": "string" + }, "extra": { "description": "JSON string containing extra configuration elements.
1. The engine_params
object gets unpacked into the sqlalchemy.create_engine call, while the metadata_params
gets unpacked into the sqlalchemy.MetaData call.
2. The metadata_cache_timeout
is a cache timeout setting in seconds for metadata fetch of this database. Specify it as \"metadata_cache_timeout\": {\"schema_cache_timeout\": 600, \"table_cache_timeout\": 600}. If unset, cache will not be enabled for the functionality. A timeout of 0 indicates that the cache never expires.
3. The schemas_allowed_for_csv_upload
is a comma separated list of schemas that CSVs are allowed to upload to. Specify it as \"schemas_allowed_for_csv_upload\": [\"public\", \"csv_upload\"]. If database flavor does not support schema or any schema is allowed to be accessed, just leave the list empty
4. the version
field is a string specifying the this db's version. This should be used with Presto DBs so that the syntax is correct
5. The allows_virtual_table_explore
field is a boolean specifying whether or not the Explore button in SQL Lab results is shown.
Optional CA_BUNDLE contents to validate HTTPS requests. Only available on certain database engines.
", "nullable": true, @@ -2521,8 +2886,53 @@ "type": "string" } }, + "type": "object" + }, + "DatabaseValidateParametersSchema": { + "properties": { + "configuration_method": { + "description": "Configuration_method is used on the frontend to inform the backend whether to explode parameters or to provide only a sqlalchemy_uri." + }, + "database_name": { + "description": "A database name to identify this connection.", + "maxLength": 250, + "minLength": 1, + "nullable": true, + "type": "string" + }, + "encrypted_extra": { + "description": "JSON string containing additional connection configuration.
This is used to provide connection information for systems like Hive, Presto, and BigQuery, which do not conform to the username:password syntax normally used by SQLAlchemy.
JSON string containing extra configuration elements.
1. The engine_params
object gets unpacked into the sqlalchemy.create_engine call, while the metadata_params
gets unpacked into the sqlalchemy.MetaData call.
2. The metadata_cache_timeout
is a cache timeout setting in seconds for metadata fetch of this database. Specify it as \"metadata_cache_timeout\": {\"schema_cache_timeout\": 600, \"table_cache_timeout\": 600}. If unset, cache will not be enabled for the functionality. A timeout of 0 indicates that the cache never expires.
3. The schemas_allowed_for_csv_upload
is a comma separated list of schemas that CSVs are allowed to upload to. Specify it as \"schemas_allowed_for_csv_upload\": [\"public\", \"csv_upload\"]. If database flavor does not support schema or any schema is allowed to be accessed, just leave the list empty
4. the version
field is a string specifying the this db's version. This should be used with Presto DBs so that the syntax is correct
5. The allows_virtual_table_explore
field is a boolean specifying whether or not the Explore button in SQL Lab results is shown.
Optional CA_BUNDLE contents to validate HTTPS requests. Only available on certain database engines.
", + "nullable": true, + "type": "string" + } + }, "required": [ - "sqlalchemy_uri" + "configuration_method", + "engine" ], "type": "object" }, @@ -2582,6 +2992,78 @@ ], "type": "object" }, + "DatasetColumnsRestApi.get": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, + "DatasetColumnsRestApi.get_list": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, + "DatasetColumnsRestApi.post": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, + "DatasetColumnsRestApi.put": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, + "DatasetMetricRestApi.get": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, + "DatasetMetricRestApi.get_list": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, + "DatasetMetricRestApi.post": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, + "DatasetMetricRestApi.put": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + } + }, + "type": "object" + }, "DatasetMetricsPut": { "properties": { "d3format": { @@ -2709,10 +3191,10 @@ "type": "integer" }, "columns": { - "$ref": "#/components/schemas/Meta23" + "$ref": "#/components/schemas/Meta22" }, "database": { - "$ref": "#/components/schemas/Meta26" + "$ref": "#/components/schemas/Meta23" }, "datasource_type": { "readOnly": true @@ -2794,7 +3276,7 @@ "DatasetRestApi.get_list": { "properties": { "changed_by": { - "$ref": "#/components/schemas/Meta21" + "$ref": "#/components/schemas/Meta19" }, "changed_by_name": { "readOnly": true @@ -2809,7 +3291,7 @@ "readOnly": true }, "database": { - "$ref": "#/components/schemas/Meta22" + "$ref": "#/components/schemas/Meta21" }, "default_endpoint": { "nullable": true, @@ -3073,7 +3555,7 @@ "type": "integer" }, "user": { - "$ref": "#/components/schemas/Meta44" + "$ref": "#/components/schemas/Meta43" }, "user_id": { "format": "int32", @@ -3120,7 +3602,7 @@ "type": "integer" }, "user": { - "$ref": "#/components/schemas/Meta43" + "$ref": "#/components/schemas/Meta42" }, "user_id": { "format": "int32", @@ -3375,65 +3857,60 @@ }, "Meta16": { "properties": { - "first_name": { - "maxLength": 64, - "type": "string" - }, "id": { "format": "int32", "type": "integer" }, - "last_name": { - "maxLength": 64, - "type": "string" - }, - "username": { + "name": { "maxLength": 64, "type": "string" } }, "required": [ - "first_name", - "last_name", - "username" + "name" ], "type": "object" }, "Meta17": { "properties": { + "first_name": { + "maxLength": 64, + "type": "string" + }, + "id": { + "format": "int32", + "type": "integer" + }, + "last_name": { + "maxLength": 64, + "type": "string" + }, "username": { "maxLength": 64, "type": "string" } }, "required": [ + "first_name", + "last_name", "username" ], "type": "object" }, - "Meta18": { - "properties": { - "first_name": { - "maxLength": 64, - "type": "string" - }, - "id": { - "format": "int32", - "type": "integer" - }, - "last_name": { + "Meta18": { + "properties": { + "first_name": { "maxLength": 64, "type": "string" }, - "username": { + "last_name": { "maxLength": 64, "type": "string" } }, "required": [ "first_name", - "last_name", - "username" + "last_name" ], "type": "object" }, @@ -3443,14 +3920,14 @@ "maxLength": 64, "type": "string" }, - "last_name": { + "username": { "maxLength": 64, "type": "string" } }, "required": [ "first_name", - "last_name" + "username" ], "type": "object" }, @@ -3495,23 +3972,6 @@ "type": "object" }, "Meta21": { - "properties": { - "first_name": { - "maxLength": 64, - "type": "string" - }, - "username": { - "maxLength": 64, - "type": "string" - } - }, - "required": [ - "first_name", - "username" - ], - "type": "object" - }, - "Meta22": { "properties": { "database_name": { "maxLength": 250, @@ -3527,7 +3987,7 @@ ], "type": "object" }, - "Meta23": { + "Meta22": { "properties": { "changed_on": { "format": "date-time", @@ -3581,6 +4041,9 @@ "nullable": true, "type": "string" }, + "type_generic": { + "readOnly": true + }, "uuid": { "format": "uuid", "nullable": true, @@ -3597,6 +4060,22 @@ ], "type": "object" }, + "Meta23": { + "properties": { + "database_name": { + "maxLength": 250, + "type": "string" + }, + "id": { + "format": "int32", + "type": "integer" + } + }, + "required": [ + "database_name" + ], + "type": "object" + }, "Meta24": { "properties": { "first_name": { @@ -3686,22 +4165,6 @@ "type": "object" }, "Meta26": { - "properties": { - "database_name": { - "maxLength": 250, - "type": "string" - }, - "id": { - "format": "int32", - "type": "integer" - } - }, - "required": [ - "database_name" - ], - "type": "object" - }, - "Meta27": { "properties": { "first_name": { "maxLength": 64, @@ -3727,7 +4190,7 @@ ], "type": "object" }, - "Meta28": { + "Meta27": { "properties": { "database_name": { "maxLength": 250, @@ -3739,7 +4202,7 @@ ], "type": "object" }, - "Meta29": { + "Meta28": { "properties": { "id": { "format": "int32", @@ -3748,12 +4211,16 @@ }, "type": "object" }, - "Meta3": { + "Meta29": { "properties": { "first_name": { "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "last_name": { "maxLength": 64, "type": "string" @@ -3765,16 +4232,12 @@ ], "type": "object" }, - "Meta30": { + "Meta3": { "properties": { "first_name": { "maxLength": 64, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, "last_name": { "maxLength": 64, "type": "string" @@ -3786,7 +4249,7 @@ ], "type": "object" }, - "Meta31": { + "Meta30": { "properties": { "database_name": { "maxLength": 250, @@ -3802,7 +4265,7 @@ ], "type": "object" }, - "Meta32": { + "Meta31": { "properties": { "first_name": { "maxLength": 64, @@ -3823,7 +4286,7 @@ ], "type": "object" }, - "Meta33": { + "Meta32": { "properties": { "database_name": { "maxLength": 250, @@ -3839,23 +4302,24 @@ ], "type": "object" }, - "Meta34": { + "Meta33": { "properties": { - "id": { - "format": "int32", - "type": "integer" + "first_name": { + "maxLength": 64, + "type": "string" }, - "type": { - "maxLength": 50, + "last_name": { + "maxLength": 64, "type": "string" } }, "required": [ - "type" + "first_name", + "last_name" ], "type": "object" }, - "Meta35": { + "Meta34": { "properties": { "first_name": { "maxLength": 64, @@ -3872,12 +4336,32 @@ ], "type": "object" }, + "Meta35": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + }, + "type": { + "maxLength": 50, + "type": "string" + } + }, + "required": [ + "type" + ], + "type": "object" + }, "Meta36": { "properties": { "first_name": { "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "last_name": { "maxLength": 64, "type": "string" @@ -3891,22 +4375,17 @@ }, "Meta37": { "properties": { - "first_name": { - "maxLength": 64, + "database_name": { + "maxLength": 250, "type": "string" }, "id": { "format": "int32", "type": "integer" - }, - "last_name": { - "maxLength": 64, - "type": "string" } }, "required": [ - "first_name", - "last_name" + "database_name" ], "type": "object" }, @@ -3915,17 +4394,40 @@ "id": { "format": "int32", "type": "integer" + }, + "recipient_config_json": { + "nullable": true, + "type": "string" + }, + "type": { + "maxLength": 50, + "type": "string" } }, + "required": [ + "type" + ], "type": "object" }, "Meta39": { "properties": { + "first_name": { + "maxLength": 64, + "type": "string" + }, "id": { "format": "int32", "type": "integer" + }, + "last_name": { + "maxLength": 64, + "type": "string" } }, + "required": [ + "first_name", + "last_name" + ], "type": "object" }, "Meta4": { @@ -3951,51 +4453,34 @@ "format": "int32", "type": "integer" }, - "recipient_config_json": { + "slice_name": { + "maxLength": 250, "nullable": true, "type": "string" }, - "type": { - "maxLength": 50, + "viz_type": { + "maxLength": 250, + "nullable": true, "type": "string" } }, - "required": [ - "type" - ], "type": "object" }, "Meta41": { "properties": { - "first_name": { - "maxLength": 64, + "dashboard_title": { + "maxLength": 500, + "nullable": true, "type": "string" }, "id": { "format": "int32", "type": "integer" - }, - "last_name": { - "maxLength": 64, - "type": "string" } }, - "required": [ - "first_name", - "last_name" - ], "type": "object" }, "Meta42": { - "properties": { - "id": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Meta43": { "properties": { "username": { "maxLength": 64, @@ -4007,7 +4492,7 @@ ], "type": "object" }, - "Meta44": { + "Meta43": { "properties": { "username": { "maxLength": 64, @@ -4025,6 +4510,10 @@ "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "last_name": { "maxLength": 64, "type": "string" @@ -4038,22 +4527,17 @@ }, "Meta6": { "properties": { - "first_name": { - "maxLength": 64, + "default_endpoint": { + "nullable": true, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, - "last_name": { - "maxLength": 64, + "table_name": { + "maxLength": 250, "type": "string" } }, "required": [ - "first_name", - "last_name" + "table_name" ], "type": "object" }, @@ -4085,17 +4569,18 @@ }, "Meta8": { "properties": { - "default_endpoint": { - "nullable": true, + "first_name": { + "maxLength": 64, "type": "string" }, - "table_name": { - "maxLength": 250, + "last_name": { + "maxLength": 64, "type": "string" } }, "required": [ - "table_name" + "first_name", + "last_name" ], "type": "object" }, @@ -4125,7 +4610,7 @@ "type": "string" }, "database": { - "$ref": "#/components/schemas/Meta29" + "$ref": "#/components/schemas/Meta28" }, "end_result_backend_time": { "nullable": true, @@ -4240,7 +4725,7 @@ "type": "string" }, "database": { - "$ref": "#/components/schemas/Meta28" + "$ref": "#/components/schemas/Meta27" }, "end_time": { "nullable": true, @@ -4295,7 +4780,7 @@ "type": "string" }, "user": { - "$ref": "#/components/schemas/Meta27" + "$ref": "#/components/schemas/Meta26" } }, "required": [ @@ -4379,6 +4864,11 @@ "maxLength": 50, "type": "string" }, + "uuid": { + "format": "uuid", + "nullable": true, + "type": "string" + }, "value": { "format": "float", "nullable": true, @@ -4423,6 +4913,11 @@ "maxLength": 50, "type": "string" }, + "uuid": { + "format": "uuid", + "nullable": true, + "type": "string" + }, "value": { "format": "float", "nullable": true, @@ -4491,21 +4986,26 @@ "type": "boolean" }, "chart": { - "$ref": "#/components/schemas/Meta39" + "$ref": "#/components/schemas/Meta40" }, "context_markdown": { "nullable": true, "type": "string" }, + "creation_method": { + "maxLength": 255, + "nullable": true, + "type": "string" + }, "crontab": { "maxLength": 1000, "type": "string" }, "dashboard": { - "$ref": "#/components/schemas/Meta38" + "$ref": "#/components/schemas/Meta41" }, "database": { - "$ref": "#/components/schemas/Meta42" + "$ref": "#/components/schemas/Meta37" }, "description": { "nullable": true, @@ -4549,15 +5049,24 @@ "type": "string" }, "owners": { - "$ref": "#/components/schemas/Meta41" + "$ref": "#/components/schemas/Meta39" }, "recipients": { - "$ref": "#/components/schemas/Meta40" + "$ref": "#/components/schemas/Meta38" + }, + "report_format": { + "maxLength": 50, + "nullable": true, + "type": "string" }, "sql": { "nullable": true, "type": "string" }, + "timezone": { + "maxLength": 100, + "type": "string" + }, "type": { "maxLength": 50, "type": "string" @@ -4592,7 +5101,7 @@ "type": "boolean" }, "changed_by": { - "$ref": "#/components/schemas/Meta36" + "$ref": "#/components/schemas/Meta34" }, "changed_on": { "format": "date-time", @@ -4603,13 +5112,18 @@ "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/Meta35" + "$ref": "#/components/schemas/Meta33" }, "created_on": { "format": "date-time", "nullable": true, "type": "string" }, + "creation_method": { + "maxLength": 255, + "nullable": true, + "type": "string" + }, "crontab": { "maxLength": 1000, "type": "string" @@ -4617,6 +5131,10 @@ "crontab_humanized": { "readOnly": true }, + "description": { + "nullable": true, + "type": "string" + }, "id": { "format": "int32", "type": "integer" @@ -4636,10 +5154,14 @@ "type": "string" }, "owners": { - "$ref": "#/components/schemas/Meta37" + "$ref": "#/components/schemas/Meta36" }, "recipients": { - "$ref": "#/components/schemas/Meta34" + "$ref": "#/components/schemas/Meta35" + }, + "timezone": { + "maxLength": 100, + "type": "string" }, "type": { "maxLength": 50, @@ -4661,6 +5183,7 @@ }, "chart": { "format": "int32", + "nullable": true, "type": "integer" }, "context_markdown": { @@ -4668,6 +5191,9 @@ "nullable": true, "type": "string" }, + "creation_method": { + "description": "Creation method is used to inform the frontend whether the report/alert was created in the dashboard, chart, or alerts and reports UI." + }, "crontab": { "description": "A CRON expression.[Crontab Guru](https://crontab.guru/) is a helpful resource that can help you craft a CRON expression.", "example": "*/5 * * * *", @@ -4677,6 +5203,7 @@ }, "dashboard": { "format": "int32", + "nullable": true, "type": "integer" }, "database": { @@ -4693,12 +5220,14 @@ "description": "Once an alert is triggered, how long, in seconds, before Superset nags you again. (in seconds)", "example": 14400, "format": "int32", + "minimum": 1, "type": "integer" }, "log_retention": { "description": "How long to keep the logs around for this report (in days)", "example": 90, "format": "int32", + "minimum": 1, "type": "integer" }, "name": { @@ -4722,11 +5251,23 @@ }, "type": "array" }, + "report_format": { + "enum": [ + "PNG", + "CSV", + "TEXT" + ], + "type": "string" + }, "sql": { "description": "A SQL statement that defines whether the alert should get triggered or not. The query is expected to return either NULL or a number value.", "example": "SELECT value FROM time_series_table", "type": "string" }, + "timezone": { + "description": "A timezone string that represents the location of the timezone.", + "type": "string" + }, "type": { "description": "The report schedule type", "enum": [ @@ -4750,6 +5291,7 @@ "description": "If an alert is staled at a working state, how long until it's state is reseted to error", "example": 3600, "format": "int32", + "minimum": 1, "type": "integer" } }, @@ -4767,6 +5309,7 @@ }, "chart": { "format": "int32", + "nullable": true, "type": "integer" }, "context_markdown": { @@ -4774,6 +5317,10 @@ "nullable": true, "type": "string" }, + "creation_method": { + "description": "Creation method is used to inform the frontend whether the report/alert was created in the dashboard, chart, or alerts and reports UI.", + "nullable": true + }, "crontab": { "description": "A CRON expression.[Crontab Guru](https://crontab.guru/) is a helpful resource that can help you craft a CRON expression.", "maxLength": 1000, @@ -4782,6 +5329,7 @@ }, "dashboard": { "format": "int32", + "nullable": true, "type": "integer" }, "database": { @@ -4798,12 +5346,14 @@ "description": "Once an alert is triggered, how long, in seconds, before Superset nags you again. (in seconds)", "example": 14400, "format": "int32", + "minimum": 1, "type": "integer" }, "log_retention": { "description": "How long to keep the logs around for this report (in days)", "example": 90, "format": "int32", + "minimum": 1, "type": "integer" }, "name": { @@ -4826,12 +5376,24 @@ }, "type": "array" }, + "report_format": { + "enum": [ + "PNG", + "CSV", + "TEXT" + ], + "type": "string" + }, "sql": { "description": "A SQL statement that defines whether the alert should get triggered or not. The query is expected to return either NULL or a number value.", "example": "SELECT value FROM time_series_table", "nullable": true, "type": "string" }, + "timezone": { + "description": "A timezone string that represents the location of the timezone.", + "type": "string" + }, "type": { "description": "The report schedule type", "enum": [ @@ -4856,19 +5418,32 @@ "description": "If an alert is staled at a working state, how long until it's state is reseted to error", "example": 3600, "format": "int32", + "minimum": 1, "nullable": true, "type": "integer" } }, "type": "object" }, + "Roles": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + }, + "name": { + "type": "string" + } + }, + "type": "object" + }, "SavedQueryRestApi.get": { "properties": { "created_by": { - "$ref": "#/components/schemas/Meta32" + "$ref": "#/components/schemas/Meta31" }, "database": { - "$ref": "#/components/schemas/Meta33" + "$ref": "#/components/schemas/Meta32" }, "description": { "nullable": true, @@ -4904,7 +5479,7 @@ "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/Meta30" + "$ref": "#/components/schemas/Meta29" }, "created_on": { "format": "date-time", @@ -4912,7 +5487,7 @@ "type": "string" }, "database": { - "$ref": "#/components/schemas/Meta31" + "$ref": "#/components/schemas/Meta30" }, "db_id": { "format": "int32", @@ -4923,6 +5498,9 @@ "nullable": true, "type": "string" }, + "extra": { + "readOnly": true + }, "id": { "format": "int32", "type": "integer" @@ -5174,6 +5752,24 @@ }, "type": "object" }, + "User": { + "properties": { + "first_name": { + "type": "string" + }, + "id": { + "format": "int32", + "type": "integer" + }, + "last_name": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "type": "object" + }, "ValidatorConfigJSON": { "properties": { "op": { @@ -5374,6 +5970,12 @@ "filter": { "type": "string" }, + "include_ids": { + "items": { + "type": "integer" + }, + "type": "array" + }, "page": { "type": "integer" }, @@ -7052,13 +7654,16 @@ "schema": { "properties": { "formData": { + "description": "upload file (ZIP)", "format": "binary", "type": "string" }, "overwrite": { - "type": "bool" + "description": "overwrite existing databases?", + "type": "boolean" }, "passwords": { + "description": "JSON map of passwords for each file", "type": "string" } }, @@ -7109,7 +7714,7 @@ }, "/chart/related/{column_name}": { "get": { - "description": "Get a list of all possible owners for a chart.", + "description": "Get a list of all possible owners for a chart. Use `owners` has the `column_name` parameter", "parameters": [ { "in": "path", @@ -7451,6 +8056,77 @@ ] } }, + "/chart/{pk}/data/": { + "get": { + "description": "Takes a chart ID and uses the query context stored when the chart was saved to return payload data response.", + "parameters": [ + { + "description": "The chart ID", + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + }, + { + "description": "The format in which the data should be returned", + "in": "query", + "name": "format", + "schema": { + "type": "string" + } + }, + { + "description": "The type in which the data should be returned", + "in": "query", + "name": "type", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChartDataResponseSchema" + } + } + }, + "description": "Query result" + }, + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ChartDataAsyncResponseSchema" + } + } + }, + "description": "Async job details" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Charts" + ] + } + }, "/chart/{pk}/screenshot/{digest}/": { "get": { "description": "Get a computed screenshot from cache.", @@ -8560,13 +9236,16 @@ "schema": { "properties": { "formData": { + "description": "upload file (ZIP or JSON)", "format": "binary", "type": "string" }, "overwrite": { - "type": "bool" + "description": "overwrite existing databases?", + "type": "boolean" }, "passwords": { + "description": "JSON map of passwords for each file", "type": "string" } }, @@ -8648,7 +9327,118 @@ } } }, - "description": "Related column data" + "description": "Related column data" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Dashboards" + ] + } + }, + "/dashboard/{id_or_slug}": { + "get": { + "description": "Get a dashboard detail information.", + "parameters": [ + { + "description": "Either the id of the dashboard, or its slug", + "in": "path", + "name": "id_or_slug", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "$ref": "#/components/schemas/DashboardGetResponseSchema" + } + }, + "type": "object" + } + } + }, + "description": "Dashboard" + }, + "302": { + "description": "Redirects to the current digest" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Dashboards" + ] + } + }, + "/dashboard/{id_or_slug}/charts": { + "get": { + "description": "Get the chart definitions for a given dashboard", + "parameters": [ + { + "in": "path", + "name": "id_or_slug", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "items": { + "$ref": "#/components/schemas/ChartEntityResponseSchema" + }, + "type": "array" + } + }, + "type": "object" + } + } + }, + "description": "Dashboard chart definitions" + }, + "302": { + "description": "Redirects to the current digest" }, "400": { "$ref": "#/components/responses/400" @@ -8658,9 +9448,6 @@ }, "404": { "$ref": "#/components/responses/404" - }, - "500": { - "$ref": "#/components/responses/500" } }, "security": [ @@ -8673,16 +9460,17 @@ ] } }, - "/dashboard/{pk}": { - "delete": { - "description": "Deletes a Dashboard.", + "/dashboard/{id_or_slug}/datasets": { + "get": { + "description": "Returns a list of a dashboard's datasets. Each dataset includes only the information necessary to render the dashboard's charts.", "parameters": [ { + "description": "Either the id of the dashboard, or its slug", "in": "path", - "name": "pk", + "name": "id_or_slug", "required": true, "schema": { - "type": "integer" + "type": "string" } } ], @@ -8692,30 +9480,30 @@ "application/json": { "schema": { "properties": { - "message": { - "type": "string" + "result": { + "items": { + "$ref": "#/components/schemas/DashboardDatasetSchema" + }, + "type": "array" } }, "type": "object" } } }, - "description": "Dashboard deleted" + "description": "Dashboard dataset definitions" + }, + "302": { + "description": "Redirects to the current digest" + }, + "400": { + "$ref": "#/components/responses/400" }, "401": { "$ref": "#/components/responses/401" }, - "403": { - "$ref": "#/components/responses/403" - }, "404": { "$ref": "#/components/responses/404" - }, - "422": { - "$ref": "#/components/responses/422" - }, - "500": { - "$ref": "#/components/responses/500" } }, "security": [ @@ -8726,9 +9514,11 @@ "tags": [ "Dashboards" ] - }, - "get": { - "description": "Get a dashboard detail information.", + } + }, + "/dashboard/{pk}": { + "delete": { + "description": "Deletes a Dashboard.", "parameters": [ { "in": "path", @@ -8737,17 +9527,6 @@ "schema": { "type": "integer" } - }, - { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/get_item_schema" - } - } - }, - "in": "query", - "name": "q" } ], "responses": { @@ -8756,43 +9535,7 @@ "application/json": { "schema": { "properties": { - "description_columns": { - "properties": { - "column_name": { - "description": "The description for the column name. Will be translated by babel", - "example": "A Nice description for the column", - "type": "string" - } - }, - "type": "object" - }, - "id": { - "description": "The item id", - "type": "string" - }, - "label_columns": { - "properties": { - "column_name": { - "description": "The label for the column name. Will be translated by babel", - "example": "A Nice label for the column", - "type": "string" - } - }, - "type": "object" - }, - "result": { - "$ref": "#/components/schemas/DashboardRestApi.get" - }, - "show_columns": { - "description": "A list of columns", - "items": { - "type": "string" - }, - "type": "array" - }, - "show_title": { - "description": "A title to render. Will be translated by babel", - "example": "Show Item Details", + "message": { "type": "string" } }, @@ -8800,14 +9543,14 @@ } } }, - "description": "Item from Model" - }, - "400": { - "$ref": "#/components/responses/400" + "description": "Dashboard deleted" }, "401": { "$ref": "#/components/responses/401" }, + "403": { + "$ref": "#/components/responses/403" + }, "404": { "$ref": "#/components/responses/404" }, @@ -9233,6 +9976,73 @@ ] } }, + "/database/available/": { + "get": { + "description": "Get names of databases currently available", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "properties": { + "available_drivers": { + "description": "Installed drivers for the engine", + "items": { + "type": "string" + }, + "type": "array" + }, + "default_driver": { + "description": "Default driver for the engine", + "type": "string" + }, + "engine": { + "description": "Name of the SQLAlchemy engine", + "type": "string" + }, + "name": { + "description": "Name of the database", + "type": "string" + }, + "parameters": { + "description": "JSON schema defining the needed parameters", + "type": "object" + }, + "preferred": { + "description": "Is the database preferred?", + "type": "boolean" + }, + "sqlalchemy_uri_placeholder": { + "description": "Example placeholder for the SQLAlchemy URI", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + } + }, + "description": "Database names" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Database" + ] + } + }, "/database/export/": { "get": { "description": "Download database(s) and associated dataset(s) as a zip file", @@ -9289,13 +10099,16 @@ "schema": { "properties": { "formData": { + "description": "upload file (ZIP)", "format": "binary", "type": "string" }, "overwrite": { - "type": "bool" + "description": "overwrite existing databases?", + "type": "boolean" }, "passwords": { + "description": "JSON map of passwords for each file", "type": "string" } }, @@ -9394,6 +10207,56 @@ ] } }, + "/database/validate_parameters": { + "post": { + "description": "Validates parameters used to connect to a database", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatabaseValidateParametersSchema" + } + } + }, + "description": "DB-specific parameters", + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Database Test Connection" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Database" + ] + } + }, "/database/{pk}": { "delete": { "description": "Deletes a Database.", @@ -9547,9 +10410,81 @@ "tags": [ "Database" ] - }, - "put": { - "description": "Changes a Database.", + }, + "put": { + "description": "Changes a Database.", + "parameters": [ + { + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/DatabaseRestApi.put" + } + } + }, + "description": "Database schema", + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "id": { + "type": "number" + }, + "result": { + "$ref": "#/components/schemas/DatabaseRestApi.put" + } + }, + "type": "object" + } + } + }, + "description": "Database changed" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "403": { + "$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Database" + ] + } + }, + "/database/{pk}/function_names/": { + "get": { + "description": "Get function names supported by a database", "parameters": [ { "in": "path", @@ -9560,51 +10495,23 @@ } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DatabaseRestApi.put" - } - } - }, - "description": "Database schema", - "required": true - }, "responses": { "200": { "content": { "application/json": { "schema": { - "properties": { - "id": { - "type": "number" - }, - "result": { - "$ref": "#/components/schemas/DatabaseRestApi.put" - } - }, - "type": "object" + "$ref": "#/components/schemas/DatabaseFunctionNamesResponse" } } }, - "description": "Database changed" - }, - "400": { - "$ref": "#/components/responses/400" + "description": "Query result" }, "401": { "$ref": "#/components/responses/401" }, - "403": { - "$ref": "#/components/responses/403" - }, "404": { "$ref": "#/components/responses/404" }, - "422": { - "$ref": "#/components/responses/422" - }, "500": { "$ref": "#/components/responses/500" } @@ -10352,13 +11259,16 @@ "schema": { "properties": { "formData": { + "description": "upload file (ZIP or YAML)", "format": "binary", "type": "string" }, "overwrite": { - "type": "bool" + "description": "overwrite existing datasets?", + "type": "boolean" }, "passwords": { + "description": "JSON map of passwords for each file", "type": "string" } }, @@ -10630,11 +11540,10 @@ } }, { - "in": "path", + "in": "query", "name": "override_columns", - "required": true, "schema": { - "type": "bool" + "type": "boolean" } } ], @@ -10697,6 +11606,136 @@ ] } }, + "/dataset/{pk}/column/{column_id}": { + "delete": { + "description": "Delete a Dataset column", + "parameters": [ + { + "description": "The dataset pk for this column", + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + }, + { + "description": "The column id for this dataset", + "in": "path", + "name": "column_id", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Column deleted" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "403": { + "$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Datasets" + ] + } + }, + "/dataset/{pk}/metric/{metric_id}": { + "delete": { + "description": "Delete a Dataset metric", + "parameters": [ + { + "description": "The dataset pk for this column", + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + }, + { + "description": "The metric id for this dataset", + "in": "path", + "name": "metric_id", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Metric deleted" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "403": { + "$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Datasets" + ] + } + }, "/dataset/{pk}/refresh": { "put": { "description": "Refreshes and updates columns of a dataset", @@ -12639,6 +13678,72 @@ ] } }, + "/saved_query/import/": { + "post": { + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "properties": { + "formData": { + "description": "upload file (ZIP)", + "format": "binary", + "type": "string" + }, + "overwrite": { + "description": "overwrite existing saved queries?", + "type": "boolean" + }, + "passwords": { + "description": "JSON map of passwords for each file", + "type": "string" + } + }, + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Saved Query import result" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Queries" + ] + } + }, "/saved_query/related/{column_name}": { "get": { "parameters": [ @@ -12909,6 +14014,42 @@ ] } }, + "/security/csrf_token/": { + "get": { + "description": "Fetch the CSRF token", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Result contains the CSRF token" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Security" + ] + } + }, "/security/login": { "post": { "description": "Authenticate and get a JWT access and refresh token", diff --git a/docs/static/images/tutorial_dashboard_access.png b/docs/static/images/tutorial_dashboard_access.png new file mode 100644 index 000000000000..f1ce5d6273a1 Binary files /dev/null and b/docs/static/images/tutorial_dashboard_access.png differ diff --git a/helm/superset/Chart.yaml b/helm/superset/Chart.yaml index d06eff1859fd..30a409e083d8 100644 --- a/helm/superset/Chart.yaml +++ b/helm/superset/Chart.yaml @@ -22,13 +22,13 @@ maintainers: - name: craig-rueda email: craig@craigrueda.com url: https://github.com/craig-rueda -version: 0.1.3 +version: 0.3.6 dependencies: - name: postgresql version: 10.2.0 repository: https://charts.bitnami.com/bitnami condition: postgresql.enabled - name: redis - version: 12.3.0 + version: 12.3.3 repository: https://charts.bitnami.com/bitnami condition: redis.enabled diff --git a/helm/superset/templates/deployment-beat.yaml b/helm/superset/templates/deployment-beat.yaml index 0ec76da15e88..2775d287498c 100644 --- a/helm/superset/templates/deployment-beat.yaml +++ b/helm/superset/templates/deployment-beat.yaml @@ -69,7 +69,7 @@ spec: env: - name: "SUPERSET_PORT" value: {{ .Values.service.port | quote}} - {{ if .Values.extraEnv }} + {{- if .Values.extraEnv }} {{- range $key, $value := .Values.extraEnv }} - name: {{ $key | quote}} value: {{ $value | quote }} @@ -78,10 +78,17 @@ spec: envFrom: - secretRef: name: {{ tpl .Values.envFromSecret . | quote }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl . $ | quote }} + {{- end }} volumeMounts: - name: superset-config mountPath: {{ .Values.configMountPath | quote }} readOnly: true + {{- with .Values.extraVolumeMounts }} + {{- tpl (toYaml .) $ | nindent 12 -}} + {{- end }} resources: {{ toYaml .Values.resources | indent 12 }} {{- with .Values.nodeSelector }} @@ -104,4 +111,7 @@ spec: - name: superset-config secret: secretName: {{ tpl .Values.configFromSecret . }} + {{- with .Values.extraVolumes }} + {{- tpl (toYaml .) $ | nindent 8 -}} + {{- end }} {{- end -}} diff --git a/helm/superset/templates/deployment-worker.yaml b/helm/superset/templates/deployment-worker.yaml index fe0ce20d8580..8bb2cc81c15d 100644 --- a/helm/superset/templates/deployment-worker.yaml +++ b/helm/superset/templates/deployment-worker.yaml @@ -53,6 +53,9 @@ spec: app: {{ template "superset.name" . }}-worker release: {{ .Release.Name }} spec: + {{- if .Values.serviceAccountName }} + serviceAccountName: {{ .Values.serviceAccountName }} + {{- end }} securityContext: runAsUser: {{ .Values.runAsUser }} {{- if .Values.supersetWorker.initContainers }} @@ -67,7 +70,7 @@ spec: env: - name: "SUPERSET_PORT" value: {{ .Values.service.port | quote}} - {{ if .Values.extraEnv }} + {{- if .Values.extraEnv }} {{- range $key, $value := .Values.extraEnv }} - name: {{ $key | quote}} value: {{ $value | quote }} @@ -76,10 +79,17 @@ spec: envFrom: - secretRef: name: {{ tpl .Values.envFromSecret . | quote }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl . $ | quote }} + {{- end }} volumeMounts: - name: superset-config mountPath: {{ .Values.configMountPath | quote }} readOnly: true + {{- with .Values.extraVolumeMounts }} + {{- tpl (toYaml .) $ | nindent 12 -}} + {{- end }} resources: {{ toYaml .Values.resources | indent 12 }} {{- with .Values.nodeSelector }} @@ -102,3 +112,6 @@ spec: - name: superset-config secret: secretName: {{ tpl .Values.configFromSecret . }} + {{- with .Values.extraVolumes }} + {{- tpl (toYaml .) $ | nindent 8 -}} + {{- end }} diff --git a/helm/superset/templates/deployment.yaml b/helm/superset/templates/deployment.yaml index 8e807daf15e8..ec6b4f453ea5 100644 --- a/helm/superset/templates/deployment.yaml +++ b/helm/superset/templates/deployment.yaml @@ -56,6 +56,9 @@ spec: app: {{ template "superset.name" . }} release: {{ .Release.Name }} spec: + {{- if .Values.serviceAccountName }} + serviceAccountName: {{ .Values.serviceAccountName }} + {{- end }} securityContext: runAsUser: {{ .Values.runAsUser }} {{- if .Values.supersetNode.initContainers }} @@ -70,7 +73,7 @@ spec: env: - name: "SUPERSET_PORT" value: {{ .Values.service.port | quote}} - {{ if .Values.extraEnv }} + {{- if .Values.extraEnv }} {{- range $key, $value := .Values.extraEnv }} - name: {{ $key | quote}} value: {{ $value | quote }} @@ -79,6 +82,10 @@ spec: envFrom: - secretRef: name: {{ tpl .Values.envFromSecret . | quote }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl . $ | quote }} + {{- end }} volumeMounts: - name: superset-config mountPath: {{ .Values.configMountPath | quote }} @@ -88,6 +95,9 @@ spec: mountPath: {{ .Values.extraConfigMountPath | quote }} readOnly: true {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- tpl (toYaml .) $ | nindent 12 -}} + {{- end }} ports: - name: http containerPort: {{ .Values.service.port }} @@ -120,3 +130,6 @@ spec: configMap: name: {{ template "superset.fullname" . }}-extra-config {{- end }} + {{- with .Values.extraVolumes }} + {{- tpl (toYaml .) $ | nindent 8 -}} + {{- end }} diff --git a/helm/superset/templates/ingress.yaml b/helm/superset/templates/ingress.yaml index b0888e4ede04..7c1dd72f0ebe 100644 --- a/helm/superset/templates/ingress.yaml +++ b/helm/superset/templates/ingress.yaml @@ -16,8 +16,7 @@ # {{ if .Values.ingress.enabled -}} {{- $fullName := include "superset.fullname" . -}} -{{- $ingressPath := .Values.ingress.path -}} -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: {{ $fullName }} @@ -46,9 +45,12 @@ spec: - host: {{ . }} http: paths: - - path: {{ $ingressPath }} + - path: {{ $.Values.ingress.path }} + pathType: {{ $.Values.ingress.pathType }} backend: - serviceName: {{ $fullName }} - servicePort: http + service: + name: {{ $fullName }} + port: + name: http {{- end }} {{- end }} diff --git a/helm/superset/templates/init-job.yaml b/helm/superset/templates/init-job.yaml index b3c4fd42f67a..fb30abc8cef2 100644 --- a/helm/superset/templates/init-job.yaml +++ b/helm/superset/templates/init-job.yaml @@ -46,16 +46,23 @@ spec: envFrom: - secretRef: name: {{ tpl .Values.envFromSecret . }} + {{- range .Values.envFromSecrets }} + - secretRef: + name: {{ tpl . $ }} + {{- end }} imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: - name: superset-config mountPath: {{ .Values.configMountPath | quote }} readOnly: true - {{ if .Values.extraConfigs }} + {{- if .Values.extraConfigs }} - name: superset-extra-config mountPath: {{ .Values.extraConfigMountPath | quote }} readOnly: true {{- end }} + {{- with .Values.extraVolumeMounts }} + {{- tpl (toYaml .) $ | nindent 10 -}} + {{- end }} command: {{ tpl (toJson .Values.init.command) . }} resources: {{ toYaml .Values.init.resources | indent 10 }} @@ -72,5 +79,8 @@ spec: configMap: name: {{ template "superset.fullname" . }}-extra-config {{- end }} + {{- with .Values.extraVolumes }} + {{- tpl (toYaml .) $ | nindent 8 -}} + {{- end }} restartPolicy: Never {{- end }} diff --git a/helm/superset/values.yaml b/helm/superset/values.yaml index 58e9faaef5fa..c8577774afb8 100644 --- a/helm/superset/values.yaml +++ b/helm/superset/values.yaml @@ -29,10 +29,10 @@ runAsUser: 0 # For production clusters it's recommended to build own image with this step done in CI bootstrapScript: | #!/bin/bash - apt-get update -y &&\ - apt-get install -y --no-install-recommends nano &&\ - rm -rf /var/lib/apt/lists/* - pip install psycopg2==2.8.5 redis==3.2.1 + rm -rf /var/lib/apt/lists/* && \ + pip install \ + psycopg2==2.8.5 \ + redis==3.2.1 && \ if [ ! -f ~/bootstrap ]; then echo "Running Superset with uid {{ .Values.runAsUser }}" > ~/bootstrap; fi ## The name of the secret which we will use to generate a superset_config.py file @@ -44,15 +44,26 @@ configFromSecret: '{{ template "superset.fullname" . }}-config' ## This can be useful for secret keys, etc. ## envFromSecret: '{{ template "superset.fullname" . }}-env' +## This can be a list of template strings +envFromSecrets: [] ## Extra environment variables that will be passed into pods ## extraEnv: {} + # Extend timeout to allow long running queries. + # GUNICORN_TIMEOUT: 300 + + + # OAUTH_HOME_DOMAIN: .. + # # If a whitelist is not set, any address that can use your OAuth2 endpoint will be able to login. + # # this includes any random Gmail address if your OAuth2 Web App is set to External. + # OAUTH_WHITELIST_REGEX: ... ## Extra environment variables to pass as secrets ## extraSecretEnv: {} # MAPBOX_API_KEY: ... + # # Google API Keys: https://console.cloud.google.com/apis/credentials # GOOGLE_KEY: ... # GOOGLE_SECRET: ... @@ -69,20 +80,38 @@ extraConfigs: {} # sqlalchemy_uri: example://example-db.local # tables: [] - extraSecrets: {} +extraVolumes: [] + # - name: customConfig + # configMap: + # name: '{{ template "superset.fullname" . }}-custom-config' + # - name: additionalSecret + # secret: + # secretName: my-secret + # defaultMode: 0600 + +extraVolumeMounts: [] + # - name: customConfig + # mountPath: /mnt/config + # readOnly: true + # - name: additionalSecret: + # mountPath: /mnt/secret # A dictionary of overrides to append at the end of superset_config.py - the name does not matter # WARNING: the order is not guaranteed configOverrides: {} + # extend_timeout: | + # # Extend timeout to allow long running queries. + # SUPERSET_WEBSERVER_TIMEOUT = ... # enable_oauth: | - # from flask_appbuilder.security.manager import AUTH_DB + # from flask_appbuilder.security.manager import (AUTH_DB, AUTH_OAUTH) # AUTH_TYPE = AUTH_OAUTH # OAUTH_PROVIDERS = [ # { # "name": "google", + # "whitelist": [ os.getenv("OAUTH_WHITELIST_REGEX", "") ], # "icon": "fa-google", # "token_key": "access_token", # "remote_app": { @@ -93,9 +122,17 @@ configOverrides: {} # "request_token_url": None, # "access_token_url": "https://accounts.google.com/o/oauth2/token", # "authorize_url": "https://accounts.google.com/o/oauth2/auth", - # }, + # "authorize_params": {"hd": os.getenv("OAUTH_HOME_DOMAIN", "")} + # } # } # ] + # # Map Authlib roles to superset roles + # AUTH_ROLE_ADMIN = 'Admin' + # AUTH_ROLE_PUBLIC = 'Public' + # # Will allow user self registration, allowing to create Flask users from Authorized User + # AUTH_USER_REGISTRATION = True + # # The default user self registration role + # AUTH_USER_REGISTRATION_ROLE = "Admin" configMountPath: "/app/pythonpath" @@ -121,7 +158,12 @@ ingress: annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" + ## Extend timeout to allow long running queries. + # nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + # nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + # nginx.ingress.kubernetes.io/proxy-send-timeout: "300" path: / + pathType: ImplementationSpecific hosts: - chart-example.local tls: [] @@ -165,10 +207,8 @@ supersetNode: - secretRef: name: '{{ tpl .Values.envFromSecret . }}' command: [ "/bin/sh", "-c", "until nc -zv $DB_HOST $DB_PORT -w1; do echo 'waiting for db'; sleep 1; done" ] - ## Annotations to be added to supersetNode deployment deploymentAnnotations: {} - ## Annotations to be added to supersetNode pods podAnnotations: {} @@ -188,10 +228,8 @@ supersetWorker: - secretRef: name: '{{ tpl .Values.envFromSecret . }}' command: [ "/bin/sh", "-c", "until nc -zv $DB_HOST $DB_PORT -w1; do echo 'waiting for db'; sleep 1; done" ] - ## Annotations to be added to supersetWorker deployment deploymentAnnotations: {} - ## Annotations to be added to supersetWorker pods podAnnotations: {} @@ -203,7 +241,7 @@ supersetCeleryBeat: command: - "/bin/sh" - "-c" - - ". {{ .Values.configMountPath }}/superset_bootstrap.sh; celery beat --app=superset.tasks.celery_app:app --pidfile /tmp/celerybeat.pid --schedule /tmp/celerybeat-schedule" + - ". {{ .Values.configMountPath }}/superset_bootstrap.sh; celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid --schedule /tmp/celerybeat-schedule" forceReload: false # If true, forces deployment to reload on each upgrade initContainers: - name: wait-for-postgres @@ -213,10 +251,8 @@ supersetCeleryBeat: - secretRef: name: '{{ tpl .Values.envFromSecret . }}' command: [ "/bin/sh", "-c", "until nc -zv $DB_HOST $DB_PORT -w1; do echo 'waiting for db'; sleep 1; done" ] - ## Annotations to be added to supersetCeleryBeat deployment deploymentAnnotations: {} - ## Annotations to be added to supersetCeleryBeat pods podAnnotations: {} @@ -224,7 +260,7 @@ supersetCeleryBeat: ## Init job configuration init: # Configure resources - # Warning: fab commant consumes a lot of ram and can + # Warning: fab command consumes a lot of ram and can # cause the process to be killed due to OOM if it exceeds limit resources: {} # limits: @@ -239,6 +275,7 @@ init: - ". {{ .Values.configMountPath }}/superset_bootstrap.sh; . {{ .Values.configMountPath }}/superset_init.sh" enabled: true loadExamples: false + createAdmin: true adminUser: username: admin firstname: Superset @@ -259,6 +296,7 @@ init: superset db upgrade echo "Initializing roles..." superset init + {{ if .Values.init.createAdmin }} echo "Creating admin user..." superset fab create-admin \ --username {{ .Values.init.adminUser.username }} \ @@ -267,10 +305,15 @@ init: --email {{ .Values.init.adminUser.email }} \ --password {{ .Values.init.adminUser.password }} \ || true + {{- end }} {{ if .Values.init.loadExamples }} echo "Loading examples..." superset load_examples {{- end }} + if [ -f "{{ .Values.extraConfigMountPath }}/import_datasources.yaml" ]; then + echo "Importing database connections.... " + superset import_datasources -p {{ .Values.extraConfigMountPath }}/import_datasources.yaml + fi ## ## Configuration values for the postgresql dependency. ## ref: https://github.com/kubernetes/charts/blob/master/stable/postgresql/README.md @@ -279,14 +322,11 @@ postgresql: ## Use the PostgreSQL chart dependency. ## Set to false if bringing your own PostgreSQL. enabled: true - ## ## The name of an existing secret that contains the postgres password. existingSecret: - ## Name of the key containing the secret. existingSecretKey: postgresql-password - ## ## If you are bringing your own PostgreSQL, you should set postgresHost and ## also probably service.port, postgresqlUsername, postgresqlPassword, and postgresqlDatabase @@ -326,16 +366,12 @@ redis: ## Use the redis chart dependency. ## Set to false if bringing your own redis. enabled: true - usePassword: false - ## ## The name of an existing secret that contains the redis password. existingSecret: - ## Name of the key containing the secret. existingSecretKey: redis-password - ## ## If you are bringing your own redis, you can set the host in redisHost. ## redisHost: diff --git a/requirements/base.in b/requirements/base.in index 294583801824..00071c4ac8f1 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -18,3 +18,4 @@ -e file:. pyrsistent>=0.16.1,<0.17 zipp==3.4.1 +sasl==0.2.1 diff --git a/requirements/base.txt b/requirements/base.txt index 0f433e0d6647..6c903a098df4 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:0862095245a068ae2fc00217da78331e1e7ae505 +# SHA1:57a754a4cf09b58d8e02c45bfb1058d2ce4286a6 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -7,9 +7,9 @@ # -e file:. # via -r requirements/base.in -aiohttp==3.7.2 +aiohttp==3.7.4.post0 # via slackclient -alembic==1.4.3 +alembic==1.6.5 # via flask-migrate amqp==2.6.1 # via kombu @@ -17,17 +17,17 @@ apispec[yaml]==3.3.2 # via flask-appbuilder async-timeout==3.0.1 # via aiohttp -attrs==20.2.0 +attrs==21.2.0 # via # aiohttp # jsonschema -babel==2.8.0 +babel==2.9.1 # via flask-babel -backoff==1.10.0 +backoff==1.11.1 # via apache-superset -billiard==3.6.3.0 +billiard==3.6.4.0 # via celery -bleach==3.3.0 +bleach==3.3.1 # via apache-superset brotli==1.0.9 # via flask-compress @@ -35,9 +35,9 @@ cachelib==0.1.1 # via apache-superset celery==4.4.7 # via apache-superset -cffi==1.14.3 +cffi==1.14.6 # via cryptography -chardet==3.0.4 +chardet==4.0.0 # via aiohttp click==7.1.2 # via @@ -48,66 +48,64 @@ colorama==0.4.4 # via # apache-superset # flask-appbuilder -contextlib2==0.6.0.post1 - # via apache-superset -convertdate==2.3.0 +convertdate==2.3.2 # via holidays cron-descriptor==1.2.24 # via apache-superset -croniter==0.3.36 +croniter==1.0.15 # via apache-superset -cryptography==3.3.2 +cryptography==3.4.7 # via apache-superset -decorator==4.4.2 - # via retry -defusedxml==0.6.0 +defusedxml==0.7.1 # via python3-openid -dnspython==2.0.0 +deprecation==2.1.0 + # via apache-superset +dnspython==2.1.0 # via email-validator -email-validator==1.1.1 +email-validator==1.1.3 # via flask-appbuilder -flask-appbuilder==3.3.0 +flask==1.1.4 + # via + # apache-superset + # flask-appbuilder + # flask-babel + # flask-caching + # flask-compress + # flask-jwt-extended + # flask-login + # flask-migrate + # flask-openid + # flask-sqlalchemy + # flask-wtf +flask-appbuilder==3.3.2 # via apache-superset flask-babel==1.0.0 # via flask-appbuilder -flask-caching==1.9.0 +flask-caching==1.10.1 # via apache-superset -flask-compress==1.8.0 +flask-compress==1.10.1 # via apache-superset -flask-jwt-extended==3.24.1 +flask-jwt-extended==3.25.1 # via flask-appbuilder flask-login==0.4.1 # via flask-appbuilder -flask-migrate==2.5.3 +flask-migrate==3.1.0 # via apache-superset flask-openid==1.2.5 # via flask-appbuilder -flask-sqlalchemy==2.4.4 +flask-sqlalchemy==2.5.1 # via # flask-appbuilder # flask-migrate -flask-talisman==0.7.0 +flask-talisman==0.8.1 # via apache-superset flask-wtf==0.14.3 # via # apache-superset # flask-appbuilder -flask==1.1.2 - # via - # apache-superset - # flask-appbuilder - # flask-babel - # flask-caching - # flask-compress - # flask-jwt-extended - # flask-login - # flask-migrate - # flask-openid - # flask-sqlalchemy - # flask-wtf -geographiclib==1.50 +geographiclib==1.52 # via geopy -geopy==2.0.0 +geopy==2.2.0 # via apache-superset graphlib-backport==1.0.3 # via apache-superset @@ -115,17 +113,12 @@ gunicorn==20.0.4 # via apache-superset holidays==0.10.3 # via apache-superset -humanize==3.1.0 +humanize==3.11.0 # via apache-superset -idna==2.10 +idna==3.2 # via # email-validator # yarl -importlib-metadata==2.1.1 - # via - # jsonschema - # kombu - # markdown isodate==0.6.0 # via apache-superset itsdangerous==1.1.0 @@ -143,53 +136,49 @@ kombu==4.6.11 # via celery korean-lunar-calendar==0.2.1 # via holidays -mako==1.1.3 +mako==1.1.4 # via alembic -markdown==3.3.3 +markdown==3.3.4 # via apache-superset -markupsafe==1.1.1 +markupsafe==2.0.1 # via # jinja2 # mako # wtforms -marshmallow-enum==1.5.1 - # via flask-appbuilder -marshmallow-sqlalchemy==0.23.1 - # via flask-appbuilder -marshmallow==3.9.0 +marshmallow==3.13.0 # via # flask-appbuilder # marshmallow-enum # marshmallow-sqlalchemy -msgpack==1.0.0 +marshmallow-enum==1.5.1 + # via flask-appbuilder +marshmallow-sqlalchemy==0.23.1 + # via flask-appbuilder +msgpack==1.0.2 # via apache-superset -multidict==5.0.0 +multidict==5.1.0 # via # aiohttp # yarl -natsort==7.0.1 - # via croniter -numpy==1.19.4 +numpy==1.21.1 # via # pandas # pyarrow -packaging==20.4 - # via bleach -pandas==1.2.2 +packaging==21.0 + # via + # bleach + # deprecation +pandas==1.2.5 # via apache-superset parsedatetime==2.6 # via apache-superset -pathlib2==2.3.5 - # via apache-superset pgsanity==0.2.9 # via apache-superset polyline==1.4.0 # via apache-superset prison==0.1.3 # via flask-appbuilder -py==1.9.0 - # via retry -pyarrow==3.0.0 +pyarrow==4.0.1 # via apache-superset pycparser==2.20 # via cffi @@ -198,7 +187,7 @@ pyjwt==1.7.1 # apache-superset # flask-appbuilder # flask-jwt-extended -pymeeus==0.3.7 +pymeeus==0.5.11 # via convertdate pyparsing==2.4.7 # via @@ -208,7 +197,7 @@ pyrsistent==0.16.1 # via # -r requirements/base.in # jsonschema -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # alembic # apache-superset @@ -216,7 +205,7 @@ python-dateutil==2.8.1 # flask-appbuilder # holidays # pandas -python-dotenv==0.15.0 +python-dotenv==0.19.0 # via apache-superset python-editor==1.0.4 # via alembic @@ -224,7 +213,7 @@ python-geohash==0.8.5 # via apache-superset python3-openid==3.2.0 # via flask-openid -pytz==2020.4 +pytz==2021.1 # via # babel # celery @@ -237,36 +226,30 @@ pyyaml==5.4.1 # apispec redis==3.5.3 # via apache-superset -retry==0.9.2 - # via apache-superset +sasl==0.2.1 + # via -r requirements/base.in selenium==3.141.0 # via apache-superset -simplejson==3.17.2 +simplejson==3.17.3 # via apache-superset -six==1.15.0 +six==1.16.0 # via # bleach - # cryptography # flask-jwt-extended # flask-talisman # holidays # isodate # jsonschema - # packaging - # pathlib2 # polyline # prison # pyrsistent # python-dateutil + # sasl # sqlalchemy-utils # wtforms-json slackclient==2.5.0 # via apache-superset -sqlalchemy-utils==0.36.8 - # via - # apache-superset - # flask-appbuilder -sqlalchemy==1.3.20 +sqlalchemy==1.3.24 # via # alembic # apache-superset @@ -274,14 +257,19 @@ sqlalchemy==1.3.20 # flask-sqlalchemy # marshmallow-sqlalchemy # sqlalchemy-utils +sqlalchemy-utils==0.36.8 + # via + # apache-superset + # flask-appbuilder sqlparse==0.3.0 # via apache-superset -typing-extensions==3.7.4.3 +tabulate==0.8.9 + # via apache-superset +typing-extensions==3.10.0.0 # via # aiohttp # apache-superset - # yarl -urllib3==1.25.11 +urllib3==1.26.6 # via selenium vine==1.3.0 # via @@ -293,18 +281,16 @@ werkzeug==1.0.1 # via # flask # flask-jwt-extended -wtforms-json==0.3.3 - # via apache-superset wtforms==2.3.3 # via # flask-wtf # wtforms-json -yarl==1.6.2 +wtforms-json==0.3.3 + # via apache-superset +yarl==1.6.3 # via aiohttp zipp==3.4.1 - # via - # -r requirements/base.in - # importlib-metadata + # via -r requirements/base.in # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements/development.in b/requirements/development.in index 5ac06dc54339..db2162341dac 100644 --- a/requirements/development.in +++ b/requirements/development.in @@ -24,4 +24,5 @@ pyhive[hive]>=0.6.1 psycopg2-binary==2.8.5 tableschema thrift>=0.11.0,<1.0.0 -pygithub>=1.54.1,<2.0.0 +progress>=1.5,<2 +pyinstrument>=4.0.2,<5 diff --git a/requirements/development.txt b/requirements/development.txt index 7b6ca3ffb7a5..82133b43a320 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,4 +1,4 @@ -# SHA1:1b4d15a41f3498d2eb930ac3d3d4ce5d1f218a2f +# SHA1:e4f3ea65026a8aec3735d6d9977f89fef4a1a4f9 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -8,82 +8,77 @@ -r base.txt -e file:. # via -r requirements/base.in -boto3==1.16.10 +boto3==1.18.19 # via tabulator -botocore==1.19.10 +botocore==1.21.19 # via # boto3 # s3transfer cached-property==1.5.2 # via tableschema -certifi==2020.6.20 +certifi==2021.5.30 # via requests -deprecated==1.2.11 - # via pygithub -et-xmlfile==1.0.1 +charset-normalizer==2.0.4 + # via requests +et-xmlfile==1.1.0 # via openpyxl -flask-cors==3.0.9 +flask-cors==3.0.10 # via -r requirements/development.in future==0.18.2 # via pyhive -ijson==3.1.2.post0 +ijson==3.1.4 # via tabulator -jdcal==1.4.1 - # via openpyxl jmespath==0.10.0 # via # boto3 # botocore -jsonlines==1.2.0 +jsonlines==2.0.0 # via tabulator linear-tsv==1.1.0 # via tabulator mysqlclient==1.4.2.post1 # via -r requirements/development.in -openpyxl==3.0.5 +openpyxl==3.0.7 # via tabulator pillow==7.2.0 # via -r requirements/development.in +progress==1.6 + # via -r requirements/development.in psycopg2-binary==2.8.5 # via -r requirements/development.in -pydruid==0.6.1 +pure-sasl==0.6.2 + # via thrift-sasl +pydruid==0.6.2 # via -r requirements/development.in -pygithub==1.54.1 +pyhive[hive]==0.6.4 # via -r requirements/development.in -pyhive[hive]==0.6.3 +pyinstrument==4.0.2 # via -r requirements/development.in -requests==2.24.0 +requests==2.26.0 # via # pydruid - # pygithub # tableschema # tabulator -rfc3986==1.4.0 +rfc3986==1.5.0 # via tableschema -s3transfer==0.3.3 +s3transfer==0.5.0 # via boto3 -sasl==0.2.1 - # via - # pyhive - # thrift-sasl -tableschema==1.20.0 +tableschema==1.20.2 # via -r requirements/development.in -tabulator==1.52.5 +tabulator==1.53.5 # via tableschema -thrift-sasl==0.4.2 - # via pyhive thrift==0.13.0 # via # -r requirements/development.in # pyhive # thrift-sasl +thrift-sasl==0.4.3 + # via pyhive unicodecsv==0.14.1 # via # tableschema # tabulator -wrapt==1.12.1 - # via deprecated -xlrd==1.2.0 +xlrd==2.0.1 # via tabulator # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/docker.txt b/requirements/docker.txt index e1d3f07003ab..545e919791bd 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -8,15 +8,15 @@ -r base.txt -e file:. # via -r requirements/base.in -gevent==20.9.0 +gevent==21.8.0 # via -r requirements/docker.in -greenlet==0.4.17 +greenlet==1.1.1 # via gevent -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.1 # via -r requirements/docker.in zope.event==4.5.0 # via gevent -zope.interface==5.1.2 +zope.interface==5.4.0 # via gevent # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/integration.in b/requirements/integration.in index 1aaee9255f97..1bc94c7fb71d 100644 --- a/requirements/integration.in +++ b/requirements/integration.in @@ -17,3 +17,6 @@ pip-compile-multi!=1.5.9 pre-commit tox +py>=1.10.0 +click==7.1.2 +packaging==21.0 diff --git a/requirements/integration.txt b/requirements/integration.txt index e1cb0edb3e2a..b7da99e3da36 100644 --- a/requirements/integration.txt +++ b/requirements/integration.txt @@ -1,70 +1,74 @@ -# SHA1:f95c1152ed0bcc554f3668440d63eec2a7d1567c +# SHA1:17ab2346746deadfc557e1df96014e77c8337f4b # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # -appdirs==1.4.4 +backports.entry-points-selectable==1.1.0 # via virtualenv -cfgv==3.2.0 +cfgv==3.3.0 # via pre-commit click==7.1.2 # via + # -r requirements/integration.in # pip-compile-multi # pip-tools -distlib==0.3.1 +distlib==0.3.2 # via virtualenv filelock==3.0.12 # via # tox # virtualenv -identify==1.5.9 +identify==2.2.13 # via pre-commit -importlib-metadata==2.1.1 +nodeenv==1.6.0 + # via pre-commit +packaging==21.0 # via - # pluggy - # pre-commit + # -r requirements/integration.in # tox - # virtualenv -nodeenv==1.5.0 - # via pre-commit -packaging==20.4 - # via tox +pep517==0.11.0 + # via pip-tools pip-compile-multi==2.4.1 # via -r requirements/integration.in -pip-tools==5.3.1 +pip-tools==6.2.0 # via pip-compile-multi +platformdirs==2.2.0 + # via virtualenv pluggy==0.13.1 # via tox -pre-commit==2.8.2 +pre-commit==2.14.0 # via -r requirements/integration.in -py==1.9.0 - # via tox +py==1.10.0 + # via + # -r requirements/integration.in + # tox pyparsing==2.4.7 # via packaging pyyaml==5.4.1 # via pre-commit -six==1.15.0 +six==1.16.0 # via - # packaging - # pip-tools # tox # virtualenv toml==0.10.2 # via # pre-commit # tox -toposort==1.5 +tomli==1.2.1 + # via pep517 +toposort==1.6 # via pip-compile-multi -tox==3.20.1 +tox==3.24.1 # via -r requirements/integration.in -virtualenv==20.1.0 +virtualenv==20.7.2 # via # pre-commit # tox -zipp==3.4.1 - # via importlib-metadata +wheel==0.37.0 + # via pip-tools # The following packages are considered to be unsafe in a requirements file: # pip +# setuptools diff --git a/requirements/testing.in b/requirements/testing.in index e95cab68f04c..3355d71fa433 100644 --- a/requirements/testing.in +++ b/requirements/testing.in @@ -22,13 +22,15 @@ freezegun ipdb # pinning ipython as pip-compile-multi was bringing higher version # of the ipython that was not found in CI -ipython==7.16.1 +ipython openapi-spec-validator openpyxl parameterized pyfakefs pyhive[presto]>=0.6.3 -pylint +pylint==2.6.0 pytest pytest-cov statsd +pytest-mock +packaging==21.0 diff --git a/requirements/testing.txt b/requirements/testing.txt index 87b87060e8ca..ccbd88989e6a 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,4 +1,4 @@ -# SHA1:dba8c39bbdcb85b86e83af855d023b55a2674e87 +# SHA1:5bfcfb5d0ab31dd532ce58caa2aab91d6807b123 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -9,78 +9,91 @@ -r integration.txt -e file:. # via -r requirements/base.in -appnope==0.1.0 +appnope==0.1.2 # via ipython -astroid==2.4.2 +astroid==2.6.6 # via pylint backcall==0.2.0 # via ipython -coverage==5.3 +coverage==5.5 # via pytest-cov -docker==4.3.1 +decorator==5.0.9 + # via + # ipdb + # ipython +docker==5.0.0 # via -r requirements/testing.in -flask-testing==0.8.0 +flask-testing==0.8.1 # via -r requirements/testing.in -freezegun==1.0.0 +freezegun==1.1.0 # via -r requirements/testing.in iniconfig==1.1.1 # via pytest -ipdb==0.13.4 +ipdb==0.13.9 # via -r requirements/testing.in -ipython-genutils==0.2.0 - # via traitlets -ipython==7.16.1 +ipython==7.26.0 # via # -r requirements/testing.in # ipdb -isort==5.6.4 +ipython-genutils==0.2.0 + # via traitlets +isort==5.9.3 # via pylint -jedi==0.17.2 +jedi==0.18.0 # via ipython -lazy-object-proxy==1.4.3 +lazy-object-proxy==1.6.0 # via astroid +matplotlib-inline==0.1.2 + # via ipython mccabe==0.6.1 # via pylint -openapi-spec-validator==0.2.9 +openapi-schema-validator==0.1.5 + # via openapi-spec-validator +openapi-spec-validator==0.3.1 # via -r requirements/testing.in -parameterized==0.7.4 +parameterized==0.8.1 # via -r requirements/testing.in -parso==0.7.1 +parso==0.8.2 # via jedi pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -prompt-toolkit==3.0.8 +prompt-toolkit==3.0.19 # via ipython -ptyprocess==0.6.0 +ptyprocess==0.7.0 # via pexpect -pyfakefs==4.4.0 +pyfakefs==4.5.0 # via -r requirements/testing.in -pygments==2.7.2 +pygments==2.9.0 # via ipython -pyhive[hive,presto]==0.6.3 +pyhive[hive,presto]==0.6.4 # via # -r requirements/development.in # -r requirements/testing.in -pylint==2.6.0 +pylint==2.9.6 # via -r requirements/testing.in -pytest-cov==2.10.1 - # via -r requirements/testing.in -pytest==6.1.2 +pytest==6.2.4 # via # -r requirements/testing.in # pytest-cov + # pytest-mock +pytest-cov==2.12.1 + # via -r requirements/testing.in +pytest-mock==3.6.1 + # via -r requirements/testing.in statsd==3.3.0 # via -r requirements/testing.in traitlets==5.0.5 - # via ipython -typed-ast==1.4.3 - # via astroid + # via + # ipython + # matplotlib-inline wcwidth==0.2.5 # via prompt-toolkit -websocket-client==0.57.0 +websocket-client==1.2.0 # via docker +wrapt==1.12.1 + # via astroid # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/scripts/benchmark_migration.py b/scripts/benchmark_migration.py index 0faa92a88552..27670b5d4d72 100644 --- a/scripts/benchmark_migration.py +++ b/scripts/benchmark_migration.py @@ -25,10 +25,13 @@ from typing import Dict, List, Set, Type import click +from flask import current_app from flask_appbuilder import Model from flask_migrate import downgrade, upgrade from graphlib import TopologicalSorter # pylint: disable=wrong-import-order -from sqlalchemy import inspect +from progress.bar import ChargingBar +from sqlalchemy import create_engine, inspect +from sqlalchemy.ext.automap import automap_base from superset import db from superset.utils.mock_data import add_sample_rows @@ -41,9 +44,13 @@ def import_migration_script(filepath: Path) -> ModuleType: Import migration script as if it were a module. """ spec = importlib.util.spec_from_file_location(filepath.stem, filepath) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) # type: ignore - return module + if spec: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore + return module + raise Exception( + "No module spec found in location: `{path}`".format(path=str(filepath)) + ) def extract_modified_tables(module: ModuleType) -> Set[str]: @@ -83,11 +90,29 @@ def find_models(module: ModuleType) -> List[Type[Model]]: elif isinstance(obj, dict): queue.extend(obj.values()) - # add implicit models - # pylint: disable=no-member, protected-access - for obj in Model._decl_class_registry.values(): - if hasattr(obj, "__table__") and obj.__table__.fullname in tables: - models.append(obj) + # build models by automapping the existing tables, instead of using current + # code; this is needed for migrations that modify schemas (eg, add a column), + # where the current model is out-of-sync with the existing table after a + # downgrade + sqlalchemy_uri = current_app.config["SQLALCHEMY_DATABASE_URI"] + engine = create_engine(sqlalchemy_uri) + Base = automap_base() + Base.prepare(engine, reflect=True) + seen = set() + while tables: + table = tables.pop() + seen.add(table) + model = getattr(Base.classes, table) + model.__tablename__ = table + models.append(model) + + # add other models referenced in foreign keys + inspector = inspect(model) + for column in inspector.columns.values(): + for foreign_key in column.foreign_keys: + table = foreign_key.column.table.name + if table not in seen: + tables.add(table) # sort topologically so we can create entities in order and # maintain relationships (eg, create a database before creating @@ -98,7 +123,8 @@ def find_models(module: ModuleType) -> List[Type[Model]]: dependent_tables: List[str] = [] for column in inspector.columns.values(): for foreign_key in column.foreign_keys: - dependent_tables.append(foreign_key.target_fullname.split(".")[0]) + if foreign_key.column.table.name != model.__tablename__: + dependent_tables.append(foreign_key.column.table.name) sorter.add(model.__tablename__, *dependent_tables) order = list(sorter.static_order()) models.sort(key=lambda model: order.index(model.__tablename__)) @@ -133,15 +159,6 @@ def main( ).scalar() print(f"Current version of the DB is {current_revision}") - print("\nIdentifying models used in the migration:") - models = find_models(module) - model_rows: Dict[Type[Model], int] = {} - for model in models: - rows = session.query(model).count() - print(f"- {model.__name__} ({rows} rows in table {model.__tablename__})") - model_rows[model] = rows - session.close() - if current_revision != down_revision: if not force: click.confirm( @@ -152,6 +169,15 @@ def main( ) downgrade(revision=down_revision) + print("\nIdentifying models used in the migration:") + models = find_models(module) + model_rows: Dict[Type[Model], int] = {} + for model in models: + rows = session.query(model).count() + print(f"- {model.__name__} ({rows} rows in table {model.__tablename__})") + model_rows[model] = rows + session.close() + print("Benchmarking migration") results: Dict[str, float] = {} start = time.time() @@ -168,18 +194,23 @@ def main( for model in models: missing = min_entities - model_rows[model] if missing > 0: + entities: List[Model] = [] print(f"- Adding {missing} entities to the {model.__name__} model") + bar = ChargingBar("Processing", max=missing) try: - added_models = add_sample_rows(session, model, missing) + for entity in add_sample_rows(session, model, missing): + entities.append(entity) + bar.next() except Exception: session.rollback() raise + bar.finish() model_rows[model] = min_entities + session.add_all(entities) session.commit() if auto_cleanup: - new_models[model].extend(added_models) - + new_models[model].extend(entities) start = time.time() upgrade(revision=revision) duration = time.time() - start @@ -187,6 +218,10 @@ def main( results[f"{min_entities}+"] = duration min_entities *= 10 + print("\nResults:\n") + for label, duration in results.items(): + print(f"{label}: {duration:.2f} s") + if auto_cleanup: print("Cleaning up DB") # delete in reverse order of creation to handle relationships @@ -201,10 +236,6 @@ def main( upgrade(revision=revision) print("Reverted") - print("\nResults:\n") - for label, duration in results.items(): - print(f"{label}: {duration:.2f} s") - if __name__ == "__main__": from superset.app import create_app diff --git a/scripts/cancel_github_workflows.py b/scripts/cancel_github_workflows.py index 84acda0979d2..90087fa4f736 100755 --- a/scripts/cancel_github_workflows.py +++ b/scripts/cancel_github_workflows.py @@ -33,7 +33,7 @@ ./cancel_github_workflows.py 1024 --include-last """ import os -from typing import Iterable, List, Optional, Union +from typing import Any, Dict, Iterable, Iterator, List, Optional, Union import click import requests @@ -45,7 +45,9 @@ github_repo = os.environ.get("GITHUB_REPOSITORY", "apache/superset") -def request(method: Literal["GET", "POST", "DELETE", "PUT"], endpoint: str, **kwargs): +def request( + method: Literal["GET", "POST", "DELETE", "PUT"], endpoint: str, **kwargs: Any +) -> Dict[str, Any]: resp = requests.request( method, f"https://api.github.com/{endpoint.lstrip('/')}", @@ -57,10 +59,14 @@ def request(method: Literal["GET", "POST", "DELETE", "PUT"], endpoint: str, **kw return resp -def list_runs(repo: str, params=None): +def list_runs( + repo: str, params: Optional[Dict[str, str]] = None, +) -> Iterator[Dict[str, Any]]: """List all github workflow runs. Returns: An iterator that will iterate through all pages of matching runs.""" + if params is None: + params = {} page = 1 total_count = 10000 while page * 100 < total_count: @@ -75,11 +81,11 @@ def list_runs(repo: str, params=None): page += 1 -def cancel_run(repo: str, run_id: Union[str, int]): +def cancel_run(repo: str, run_id: Union[str, int]) -> Dict[str, Any]: return request("POST", f"/repos/{repo}/actions/runs/{run_id}/cancel") -def get_pull_request(repo: str, pull_number: Union[str, int]): +def get_pull_request(repo: str, pull_number: Union[str, int]) -> Dict[str, Any]: return request("GET", f"/repos/{repo}/pulls/{pull_number}") @@ -89,7 +95,7 @@ def get_runs( user: Optional[str] = None, statuses: Iterable[str] = ("queued", "in_progress"), events: Iterable[str] = ("pull_request", "push"), -): +) -> List[Dict[str, Any]]: """Get workflow runs associated with the given branch""" return [ item @@ -101,7 +107,7 @@ def get_runs( ] -def print_commit(commit, branch): +def print_commit(commit: Dict[str, Any], branch: str) -> None: """Print out commit message for verification""" indented_message = " \n".join(commit["message"].split("\n")) date_str = ( @@ -151,7 +157,7 @@ def cancel_github_workflows( event: List[str], include_last: bool, include_running: bool, -): +) -> None: """Cancel running or queued GitHub workflows by branch or pull request ID""" if not github_token: raise ClickException("Please provide GITHUB_TOKEN as an env variable") @@ -231,7 +237,7 @@ def cancel_github_workflows( try: print(f"[{entry['status']}] {entry['name']}", end="\r") cancel_run(repo, entry["id"]) - print(f"[Cancled] {entry['name']} ") + print(f"[Canceled] {entry['name']} ") except ClickException as error: print(f"[Error: {error.message}] {entry['name']} ") print("") diff --git a/scripts/ci_check_no_file_changes.sh b/scripts/ci_check_no_file_changes.sh index e03a6aca54c6..592faa54628b 100755 --- a/scripts/ci_check_no_file_changes.sh +++ b/scripts/ci_check_no_file_changes.sh @@ -34,7 +34,7 @@ REGEXES=() for CHECK in "$@" do if [[ ${CHECK} == "python" ]]; then - REGEX="(^\.github\/workflows\/.*python|^tests\/|^superset\/|^setup\.py|^requirements\/.+\.txt)" + REGEX="(^\.github\/workflows\/.*python|^tests\/|^superset\/|^setup\.py|^requirements\/.+\.txt|^\.pylintrc)" echo "Searching for changes in python files" elif [[ ${CHECK} == "frontend" ]]; then REGEX="(^\.github\/workflows\/.*(frontend|e2e)|^superset-frontend\/)" diff --git a/scripts/permissions_cleanup.py b/scripts/permissions_cleanup.py index 3656aaa43dcc..99d192919c6c 100644 --- a/scripts/permissions_cleanup.py +++ b/scripts/permissions_cleanup.py @@ -19,7 +19,7 @@ from superset import security_manager -def cleanup_permissions(): +def cleanup_permissions() -> None: # 1. Clean up duplicates. pvms = security_manager.get_session.query( security_manager.permissionview_model diff --git a/scripts/python_tests.sh b/scripts/python_tests.sh index d34e605a21ca..b9ef2cee2104 100755 --- a/scripts/python_tests.sh +++ b/scripts/python_tests.sh @@ -18,7 +18,7 @@ # set -e -export SUPERSET_CONFIG=${SUPERSET_CONFIG:-tests.superset_test_config} +export SUPERSET_CONFIG=${SUPERSET_CONFIG:-tests.integration_tests.superset_test_config} export SUPERSET_TESTENV=true echo "Superset config module: $SUPERSET_CONFIG" diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh index be1a11988e97..9f78318b72b5 100755 --- a/scripts/tests/run.sh +++ b/scripts/tests/run.sh @@ -62,7 +62,7 @@ DB_NAME="test" DB_USER="superset" DB_PASSWORD="superset" export SUPERSET__SQLALCHEMY_DATABASE_URI=${SUPERSET__SQLALCHEMY_DATABASE_URI:-postgresql+psycopg2://"${DB_USER}":"${DB_PASSWORD}"@localhost/"${DB_NAME}"} -export SUPERSET_CONFIG=${SUPERSET_CONFIG:-tests.superset_test_config} +export SUPERSET_CONFIG=${SUPERSET_CONFIG:-tests.integration_tests.superset_test_config} RUN_INIT=1 RUN_RESET_DB=1 RUN_TESTS=1 diff --git a/setup.cfg b/setup.cfg index 8fd75a0bd6c5..9a108f76a480 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,21 +30,23 @@ combine_as_imports = true include_trailing_comma = true line_length = 88 known_first_party = superset -known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,contextlib2,cron_descriptor,croniter,cryptography,dateutil,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pathlib2,pgsanity,pkg_resources,polyline,prison,pyarrow,pyhive,pyparsing,pytest,pytz,redis,requests,retry,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,werkzeug,wtforms,wtforms_json,yaml +known_third_party =alembic,apispec,backoff,bleach,cachelib,celery,click,colorama,cron_descriptor,croniter,cryptography,dateutil,deprecation,flask,flask_appbuilder,flask_babel,flask_caching,flask_compress,flask_jwt_extended,flask_login,flask_migrate,flask_sqlalchemy,flask_talisman,flask_testing,flask_wtf,freezegun,geohash,geopy,graphlib,holidays,humanize,isodate,jinja2,jwt,markdown,markupsafe,marshmallow,marshmallow_enum,msgpack,numpy,pandas,parameterized,parsedatetime,pgsanity,pkg_resources,polyline,prison,progress,pyarrow,pyhive,pyparsing,pytest,pytest_mock,pytz,redis,requests,selenium,setuptools,simplejson,slack,sqlalchemy,sqlalchemy_utils,sqlparse,typing_extensions,urllib3,werkzeug,wtforms,wtforms_json,yaml multi_line_output = 3 order_by_type = false [mypy] +check_untyped_defs = true disallow_any_generics = true +disallow_untyped_calls = true +disallow_untyped_defs = true ignore_missing_imports = true no_implicit_optional = true warn_unused_ignores = true -[mypy-superset.*] -check_untyped_defs = true -disallow_untyped_calls = true -disallow_untyped_defs = true -warn_unused_ignores = false - [mypy-superset.migrations.versions.*] ignore_errors = true + +[mypy-tests.*] +check_untyped_defs = false +disallow_untyped_calls = false +disallow_untyped_defs = false diff --git a/setup.py b/setup.py index 30e4c5479f7e..19a08951df0b 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ long_description = f.read() -def get_git_sha(): +def get_git_sha() -> str: try: s = subprocess.check_output(["git", "rev-parse", "HEAD"]) return s.decode().strip() @@ -70,13 +70,13 @@ def get_git_sha(): "celery>=4.3.0, <5.0.0, !=4.4.1", "click<8", "colorama", - "contextlib2", "croniter>=0.3.28", "cron-descriptor", "cryptography>=3.3.2", + "deprecation>=2.1.0, <2.2.0", "flask>=1.1.0, <2.0.0", "flask-appbuilder>=3.3.0, <4.0.0", - "flask-caching", + "flask-caching>=1.10.0", "flask-compress", "flask-talisman", "flask-migrate", @@ -84,37 +84,36 @@ def get_git_sha(): "geopy", "graphlib-backport", "gunicorn>=20.0.2, <20.1", + "holidays==0.10.3", # PINNED! https://github.com/dr-prodigy/python-holidays/issues/406 "humanize", - "itsdangerous>=1.0.0, <2.0.0", + "itsdangerous>=1.0.0, <2.0.0", # https://github.com/apache/superset/pull/14627 "isodate", "markdown>=3.0", "msgpack>=1.0.0, <1.1", "pandas>=1.2.2, <1.3", "parsedatetime", - "pathlib2", "pgsanity", "polyline", + "pyparsing>=2.4.7, <3.0.0", "python-dateutil", "python-dotenv", "python-geohash", - "pyarrow>=3.0.0, <3.1", + "pyarrow>=4.0.1, <4.1", "pyyaml>=5.4", "PyJWT>=1.7.1, <2", "redis", - "retry>=0.9.2", "selenium>=3.141.0", "simplejson>=3.15.0", "slackclient==2.5.0", # PINNED! slack changes file upload api in the future versions "sqlalchemy>=1.3.16, <1.4, !=1.3.21", - "sqlalchemy-utils>=0.36.6,<0.37", + "sqlalchemy-utils>=0.36.6, <0.37", "sqlparse==0.3.0", # PINNED! see https://github.com/andialbrecht/sqlparse/issues/562 - "typing-extensions>=3.7.4.3,<4", # needed to support typing.Literal on py37 + "tabulate==0.8.9", + "typing-extensions>=3.10, <4", # needed to support Literal (3.8) and TypeGuard (3.10) "wtforms-json", - "pyparsing>=2.4.7, <3.0.0", - "holidays==0.10.3", # PINNED! https://github.com/dr-prodigy/python-holidays/issues/406 ], extras_require={ - "athena": ["pyathena>=1.10.8,<1.11"], + "athena": ["pyathena>=1.10.8, <1.11"], "bigquery": [ "pandas_gbq>=0.10.0", "pybigquery>=0.4.10", @@ -134,7 +133,7 @@ def get_git_sha(): "exasol": ["sqlalchemy-exasol>=2.1.0, <2.2"], "excel": ["xlrd>=1.2.0, <1.3"], "firebird": ["sqlalchemy-firebird>=0.7.0, <0.8"], - "gsheets": ["shillelagh[gsheetsapi]>=0.5, <0.6"], + "gsheets": ["shillelagh[gsheetsapi]>=0.7.1, <0.8"], "hana": ["hdbcli==2.4.162", "sqlalchemy_hana==0.4.0"], "hive": ["pyhive[hive]>=0.6.1", "tableschema", "thrift>=0.11.0, <1.0.0"], "impala": ["impyla>0.16.2, <0.17"], @@ -148,10 +147,12 @@ def get_git_sha(): "trino": ["sqlalchemy-trino>=0.2"], "prophet": ["prophet>=1.0.1, <1.1", "pystan<3.0"], "redshift": ["sqlalchemy-redshift>=0.8.1, < 0.9"], + "rockset": ["rockset>=0.7.68, <0.8"], "snowflake": ["snowflake-sqlalchemy>=1.2.3, <1.3"], "teradata": ["sqlalchemy-teradata==0.9.0.dev0"], "thumbnails": ["Pillow>=7.0.0, <8.0.0"], "vertica": ["sqlalchemy-vertica-python>=0.5.9, < 0.6"], + "netezza": ["nzalchemy>=11.0.2"], }, python_requires="~=3.7", author="Apache Software Foundation", diff --git a/superset-frontend/.storybook/main.js b/superset-frontend/.storybook/main.js index 7d31e0f4f620..7b15d57dae7e 100644 --- a/superset-frontend/.storybook/main.js +++ b/superset-frontend/.storybook/main.js @@ -18,11 +18,11 @@ */ const path = require('path'); -// Suerset's webpack.config.js +// Superset's webpack.config.js const customConfig = require('../webpack.config.js'); module.exports = { -stories: ['../src/@(components|common|filters)/**/*.stories.@(t|j)sx'], + stories: ['../src/@(components|common|filters)/**/*.stories.@(t|j)sx'], addons: [ '@storybook/addon-essentials', '@storybook/addon-links', @@ -43,4 +43,7 @@ stories: ['../src/@(components|common|filters)/**/*.stories.@(t|j)sx'], }, plugins: [...config.plugins, ...customConfig.plugins], }), + typescript: { + reactDocgen: 'none', + }, }; diff --git a/superset-frontend/babel.config.js b/superset-frontend/babel.config.js index 5aa3420cf1ac..1d0cae25843d 100644 --- a/superset-frontend/babel.config.js +++ b/superset-frontend/babel.config.js @@ -45,6 +45,7 @@ module.exports = { ['@babel/plugin-proposal-class-properties', { loose: true }], ['@babel/plugin-proposal-optional-chaining', { loose: true }], ['@babel/plugin-proposal-private-methods', { loose: true }], + ['@babel/plugin-proposal-nullish-coalescing-operator', { loose: true }], ['@babel/plugin-transform-runtime', { corejs: 3 }], 'react-hot-loader/babel', ], @@ -81,5 +82,8 @@ module.exports = { ], ], }, + testableProduction: { + plugins: [], + }, }, }; diff --git a/superset-frontend/cypress-base/cypress/integration/chart_list/card_view.test.ts b/superset-frontend/cypress-base/cypress/integration/chart_list/card_view.test.ts index 22da71912af1..1335fcb42220 100644 --- a/superset-frontend/cypress-base/cypress/integration/chart_list/card_view.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/chart_list/card_view.test.ts @@ -22,7 +22,7 @@ describe('chart card view', () => { beforeEach(() => { cy.login(); cy.visit(CHART_LIST); - cy.get('[data-test="card-view"]').click(); + cy.get('[aria-label="card-view"]').click(); }); it('should load cards', () => { @@ -34,36 +34,36 @@ describe('chart card view', () => { it('should allow to favorite/unfavorite chart card', () => { cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-selected']") + .find("[aria-label='favorite-selected']") .should('not.exist'); cy.get("[data-test='card-actions']") - .find("[data-test='favorite-unselected']") + .find("[aria-label='favorite-unselected']") .first() .click(); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-selected']") + .find("[aria-label='favorite-selected']") .should('be.visible'); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-unselected']") + .find("[aria-label='favorite-unselected']") .should('not.exist'); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-unselected']") + .find("[aria-label='favorite-unselected']") .should('not.exist'); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-selected']") + .find("[aria-label='favorite-selected']") .click(); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-unselected']") + .find("[aria-label='favorite-unselected']") .should('be.visible'); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-selected']") + .find("[aria-label='favorite-selected']") .should('not.exist'); }); diff --git a/superset-frontend/cypress-base/cypress/integration/chart_list/filter.test.ts b/superset-frontend/cypress-base/cypress/integration/chart_list/filter.test.ts index dd9b573ebaf9..6892651e00ef 100644 --- a/superset-frontend/cypress-base/cypress/integration/chart_list/filter.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/chart_list/filter.test.ts @@ -22,7 +22,7 @@ describe('chart card view filters', () => { beforeEach(() => { cy.login(); cy.visit(CHART_LIST); - cy.get('[data-test="card-view"]').click(); + cy.get('[aria-label="card-view"]').click(); }); it('should filter by owners correctly', () => { @@ -89,7 +89,7 @@ describe('chart list view filters', () => { beforeEach(() => { cy.login(); cy.visit(CHART_LIST); - cy.get('[data-test="list-view"]').click(); + cy.get('[aria-label="list-view"]').click(); }); it('should filter by owners correctly', () => { diff --git a/superset-frontend/cypress-base/cypress/integration/chart_list/list_view.test.ts b/superset-frontend/cypress-base/cypress/integration/chart_list/list_view.test.ts index 915bf15f0ae8..6da5d90106d1 100644 --- a/superset-frontend/cypress-base/cypress/integration/chart_list/list_view.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/chart_list/list_view.test.ts @@ -22,7 +22,7 @@ describe('chart list view', () => { beforeEach(() => { cy.login(); cy.visit(CHART_LIST); - cy.get('[data-test="list-view"]').click(); + cy.get('[aria-label="list-view"]').click(); }); it('should load rows', () => { @@ -51,11 +51,11 @@ describe('chart list view', () => { it('should bulk delete correctly', () => { cy.get('[data-test="listview-table"]').should('be.visible'); cy.get('[data-test="bulk-select"]').eq(0).click(); - cy.get('[data-test="checkbox-off"]').eq(1).siblings('input').click(); - cy.get('[data-test="checkbox-off"]').eq(2).siblings('input').click(); + cy.get('[aria-label="checkbox-off"]').eq(1).siblings('input').click(); + cy.get('[aria-label="checkbox-off"]').eq(2).siblings('input').click(); cy.get('[data-test="bulk-select-action"]').eq(0).click(); cy.get('[data-test="delete-modal-input"]').eq(0).type('DELETE'); cy.get('[data-test="modal-confirm-button"]').eq(0).click(); - cy.get('[data-test="checkbox-on"]').should('not.exist'); + cy.get('[aria-label="checkbox-on"]').should('not.exist'); }); }); diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard/controls.test.ts b/superset-frontend/cypress-base/cypress/integration/dashboard/controls.test.ts index 73587ee3acd9..5ba40db499e3 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard/controls.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/dashboard/controls.test.ts @@ -63,7 +63,7 @@ describe('Dashboard top-level controls', () => { // should allow force refresh WORLD_HEALTH_CHARTS.forEach(waitForChartLoad); getChartAliasesBySpec(WORLD_HEALTH_CHARTS).then(aliases => { - cy.get('[data-test="more-horiz"]').click(); + cy.get('[aria-label="more-horiz"]').click(); cy.get('[data-test="refresh-dashboard-menu-item"]').should( 'not.have.class', 'ant-dropdown-menu-item-disabled', @@ -92,7 +92,7 @@ describe('Dashboard top-level controls', () => { }); }); }); - cy.get('[data-test="more-horiz"]').click(); + cy.get('[aria-label="more-horiz"]').click(); cy.get('[data-test="refresh-dashboard-menu-item"]').and( 'not.have.class', 'ant-dropdown-menu-item-disabled', diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard/edit_mode.test.js b/superset-frontend/cypress-base/cypress/integration/dashboard/edit_mode.test.js index 62f40c6162f4..d799dccd3bb6 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard/edit_mode.test.js +++ b/superset-frontend/cypress-base/cypress/integration/dashboard/edit_mode.test.js @@ -23,7 +23,7 @@ describe('Dashboard edit mode', () => { cy.login(); cy.visit(WORLD_HEALTH_DASHBOARD); cy.get('[data-test="dashboard-header"]') - .find('[data-test=edit-alt]') + .find('[aria-label=edit-alt]') .click(); }); @@ -96,7 +96,7 @@ describe('Dashboard edit mode', () => { .click(); cy.get('[data-test="dashboard-header"]').within(() => { cy.get('[data-test="dashboard-edit-actions"]').should('not.be.visible'); - cy.get('[data-test="edit-alt"]').should('be.visible'); + cy.get('[aria-label="edit-alt"]').should('be.visible'); }); }); }); diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard/edit_properties.test.ts b/superset-frontend/cypress-base/cypress/integration/dashboard/edit_properties.test.ts index ad20010a824d..c5cbfb7bba66 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard/edit_properties.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/dashboard/edit_properties.test.ts @@ -77,7 +77,7 @@ describe('Dashboard edit action', () => { cy.get('.dashboard-grid', { timeout: 50000 }) .should('be.visible') // wait for 50 secs to load dashboard .then(() => { - cy.get('.dashboard-header [data-test=edit-alt]') + cy.get('.dashboard-header [aria-label=edit-alt]') .should('be.visible') .click(); openDashboardEditProperties(); diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard/fav_star.test.js b/superset-frontend/cypress-base/cypress/integration/dashboard/fav_star.test.js index 812a3c9e7851..a20b1eb3f597 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard/fav_star.test.js +++ b/superset-frontend/cypress-base/cypress/integration/dashboard/fav_star.test.js @@ -38,23 +38,23 @@ describe('Dashboard add to favorite', () => { it('should allow favor/unfavor', () => { if (!isFavoriteDashboard) { cy.get('[data-test="fave-unfave-icon"]') - .find('svg') - .should('have.attr', 'data-test', 'favorite-unselected'); + .find('span') + .should('have.attr', 'aria-label', 'favorite-unselected'); cy.get('[data-test="fave-unfave-icon"]').trigger('click'); cy.get('[data-test="fave-unfave-icon"]') - .find('svg') - .should('have.attr', 'data-test', 'favorite-selected') - .and('not.have.attr', 'data-test', 'favorite-unselected'); + .find('span') + .should('have.attr', 'aria-label', 'favorite-selected') + .and('not.have.attr', 'aria-label', 'favorite-unselected'); } else { cy.get('[data-test="fave-unfave-icon"]') - .find('svg') - .should('have.attr', 'data-test', 'favorite-unselected') - .and('not.have.attr', 'data-test', 'favorite-selected'); + .find('span') + .should('have.attr', 'aria-label', 'favorite-unselected') + .and('not.have.attr', 'aria-label', 'favorite-selected'); cy.get('[data-test="fave-unfave-icon"]').trigger('click'); cy.get('[data-test="fave-unfave-icon"]') - .find('svg') - .should('have.attr', 'data-test', 'favorite-unselected') - .and('not.have.attr', 'data-test', 'favorite-selected'); + .find('span') + .should('have.attr', 'aria-label', 'favorite-unselected') + .and('not.have.attr', 'aria-label', 'favorite-selected'); } // reset to original fav state diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard/filter.test.ts b/superset-frontend/cypress-base/cypress/integration/dashboard/filter.test.ts index 375f14dacdd8..9b30639c24b6 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard/filter.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/dashboard/filter.test.ts @@ -38,7 +38,7 @@ describe('Dashboard filter', () => { cy.get('.Select__placeholder:first').click(); // should show the filter indicator - cy.get('svg[data-test="filter"]:visible').should(nodes => { + cy.get('span[aria-label="filter"]:visible').should(nodes => { expect(nodes.length).to.least(9); }); @@ -50,7 +50,7 @@ describe('Dashboard filter', () => { cy.get('.Select__menu').first().contains('South Asia').click(); // should still have all filter indicators - cy.get('svg[data-test="filter"]:visible').should(nodes => { + cy.get('span[aria-label="filter"]:visible').should(nodes => { expect(nodes.length).to.least(9); }); diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard/markdown.test.ts b/superset-frontend/cypress-base/cypress/integration/dashboard/markdown.test.ts index a13a1c91460a..3f69c9f804f8 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard/markdown.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/dashboard/markdown.test.ts @@ -30,12 +30,12 @@ describe('Dashboard edit markdown', () => { numScripts = nodes.length; }); cy.get('[data-test="dashboard-header"]') - .find('[data-test="edit-alt"]') + .find('[aria-label="edit-alt"]') .click(); // lazy load - need to open dropdown for the scripts to load cy.get('[data-test="dashboard-header"]') - .find('[data-test="more-horiz"]') + .find('[aria-label="more-horiz"]') .click(); cy.get('script').then(nodes => { // load 5 new script chunks for css editor @@ -70,7 +70,7 @@ describe('Dashboard edit markdown', () => { .type('Test resize'); resize( - '[data-test="dashboard-markdown-editor"] .resizable-container span div', + '[data-test="dashboard-markdown-editor"] .resizable-container span div:last-child', ).to(500, 600); cy.get('[data-test="dashboard-markdown-editor"]').contains('Test resize'); @@ -78,7 +78,7 @@ describe('Dashboard edit markdown', () => { // entering edit mode does not add new scripts // (though scripts may still be removed by others) cy.get('script').then(nodes => { - expect(nodes.length).to.most(numScripts); + expect(nodes.length).to.greaterThan(numScripts); }); cy.get('@component-background-first').click('right'); diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard/save.test.js b/superset-frontend/cypress-base/cypress/integration/dashboard/save.test.js index b56da45b89e8..2555730c38bd 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard/save.test.js +++ b/superset-frontend/cypress-base/cypress/integration/dashboard/save.test.js @@ -25,7 +25,7 @@ import { } from './dashboard.helper'; function openDashboardEditProperties() { - cy.get('.dashboard-header [data-test=edit-alt]').click(); + cy.get('.dashboard-header [aria-label=edit-alt]').click(); cy.get('#save-dash-split-button').trigger('click', { force: true }); cy.get('.dropdown-menu').contains('Edit dashboard properties').click(); } @@ -42,7 +42,7 @@ describe('Dashboard save action', () => { 'copyRequest', ); - cy.get('[data-test="more-horiz"]').trigger('click', { force: true }); + cy.get('[aria-label="more-horiz"]').trigger('click', { force: true }); cy.get('[data-test="save-as-menu-item"]').trigger('click', { force: true, }); @@ -68,7 +68,7 @@ describe('Dashboard save action', () => { WORLD_HEALTH_CHARTS.forEach(waitForChartLoad); // remove box_plot chart from dashboard - cy.get('[data-test="edit-alt"]').click({ timeout: 5000 }); + cy.get('[aria-label="edit-alt"]').click({ timeout: 5000 }); cy.get('[data-test="dashboard-delete-component-button"]') .last() .trigger('moustenter') @@ -87,7 +87,7 @@ describe('Dashboard save action', () => { // go back to view mode cy.wait('@saveRequest'); cy.get('[data-test="dashboard-header"]') - .find('[data-test="edit-alt"]') + .find('[aria-label="edit-alt"]') .click(); // deleted boxplot should still not exist diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard_list/card_view.test.ts b/superset-frontend/cypress-base/cypress/integration/dashboard_list/card_view.test.ts index 841fb560b127..8bfc35d71c84 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard_list/card_view.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/dashboard_list/card_view.test.ts @@ -22,7 +22,7 @@ describe('Dashboard card view', () => { beforeEach(() => { cy.login(); cy.visit(DASHBOARD_LIST); - cy.get('[data-test="card-view"]').click(); + cy.get('[aria-label="card-view"]').click(); }); xit('should load cards', () => { @@ -34,36 +34,36 @@ describe('Dashboard card view', () => { it('should allow to favorite/unfavorite dashboard card', () => { cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-selected']") + .find("[aria-label='favorite-selected']") .should('not.exist'); cy.get("[data-test='card-actions']") - .find("[data-test='favorite-unselected']") + .find("[aria-label='favorite-unselected']") .first() .click(); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-selected']") + .find("[aria-label='favorite-selected']") .should('be.visible'); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-unselected']") + .find("[aria-label='favorite-unselected']") .should('not.exist'); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-unselected']") + .find("[aria-label='favorite-unselected']") .should('not.exist'); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-selected']") + .find("[aria-label='favorite-selected']") .click(); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-unselected']") + .find("[aria-label='favorite-unselected']") .should('be.visible'); cy.get("[data-test='card-actions']") .first() - .find("[data-test='favorite-selected']") + .find("[aria-label='favorite-selected']") .should('not.exist'); }); diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard_list/filter.test.ts b/superset-frontend/cypress-base/cypress/integration/dashboard_list/filter.test.ts index caff4c03f675..aede8e7d5466 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard_list/filter.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/dashboard_list/filter.test.ts @@ -22,7 +22,7 @@ describe('dashboard filters card view', () => { beforeEach(() => { cy.login(); cy.visit(DASHBOARD_LIST); - cy.get('[data-test="card-view"]').click(); + cy.get('[aria-label="card-view"]').click(); }); it('should filter by owners correctly', () => { @@ -59,13 +59,13 @@ describe('dashboard filters card view', () => { // filter by published cy.get('.Select__control').eq(2).click(); cy.get('.Select__menu').contains('Published').click({ timeout: 5000 }); - cy.get('[data-test="styled-card"]').should('have.length', 2); + cy.get('[data-test="styled-card"]').should('have.length', 3); cy.get('[data-test="styled-card"]') .contains('USA Births Names') .should('be.visible'); cy.get('.Select__control').eq(1).click(); cy.get('.Select__control').eq(1).type('unpub{enter}'); - cy.get('[data-test="styled-card"]').should('have.length', 2); + cy.get('[data-test="styled-card"]').should('have.length', 3); }); }); @@ -73,7 +73,7 @@ describe('dashboard filters list view', () => { beforeEach(() => { cy.login(); cy.visit(DASHBOARD_LIST); - cy.get('[data-test="list-view"]').click(); + cy.get('[aria-label="list-view"]').click(); }); it('should filter by owners correctly', () => { @@ -110,12 +110,12 @@ describe('dashboard filters list view', () => { // filter by published cy.get('.Select__control').eq(2).click(); cy.get('.Select__menu').contains('Published').click(); - cy.get('[data-test="table-row"]').should('have.length', 2); + cy.get('[data-test="table-row"]').should('have.length', 3); cy.get('[data-test="table-row"]') .contains('USA Births Names') .should('be.visible'); cy.get('.Select__control').eq(2).click(); cy.get('.Select__control').eq(2).type('unpub{enter}'); - cy.get('[data-test="table-row"]').should('have.length', 2); + cy.get('[data-test="table-row"]').should('have.length', 3); }); }); diff --git a/superset-frontend/cypress-base/cypress/integration/dashboard_list/list_view.test.ts b/superset-frontend/cypress-base/cypress/integration/dashboard_list/list_view.test.ts index 924cf84b6e46..a758552481f9 100644 --- a/superset-frontend/cypress-base/cypress/integration/dashboard_list/list_view.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/dashboard_list/list_view.test.ts @@ -22,7 +22,7 @@ describe('dashboard list view', () => { beforeEach(() => { cy.login(); cy.visit(DASHBOARD_LIST); - cy.get('[data-test="list-view"]').click(); + cy.get('[aria-label="list-view"]').click(); }); xit('should load rows', () => { @@ -51,11 +51,11 @@ describe('dashboard list view', () => { it('should bulk delete correctly', () => { cy.get('[data-test="listview-table"]').should('be.visible'); cy.get('[data-test="bulk-select"]').eq(0).click(); - cy.get('[data-test="checkbox-off"]').eq(1).siblings('input').click(); - cy.get('[data-test="checkbox-off"]').eq(2).siblings('input').click(); + cy.get('[aria-label="checkbox-off"]').eq(1).siblings('input').click(); + cy.get('[aria-label="checkbox-off"]').eq(2).siblings('input').click(); cy.get('[data-test="bulk-select-action"]').eq(0).click(); cy.get('[data-test="delete-modal-input"]').eq(0).type('DELETE'); cy.get('[data-test="modal-confirm-button"]').eq(0).click(); - cy.get('[data-test="checkbox-on"]').should('not.exist'); + cy.get('[aria-label="checkbox-on"]').should('not.exist'); }); }); diff --git a/superset-frontend/cypress-base/cypress/integration/database/modal.test.ts b/superset-frontend/cypress-base/cypress/integration/database/modal.test.ts index e29ec48a9479..225502167656 100644 --- a/superset-frontend/cypress-base/cypress/integration/database/modal.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/database/modal.test.ts @@ -21,71 +21,50 @@ import { DATABASE_LIST } from './helper'; describe('Add database', () => { beforeEach(() => { cy.login(); - }); - - it('should keep create modal open when error', () => { cy.visit(DATABASE_LIST); - - // open modal + cy.wait(3000); cy.get('[data-test="btn-create-database"]').click(); + }); - // values should be blank - cy.get('[data-test="database-modal"] input[name="database_name"]').should( - 'have.value', - '', - ); - cy.get('[data-test="database-modal"] input[name="sqlalchemy_uri"]').should( - 'have.value', - '', - ); - - // type values - cy.get('[data-test="database-modal"] input[name="database_name"]') - .focus() - .type('cypress'); - cy.get('[data-test="database-modal"] input[name="sqlalchemy_uri"]') - .focus() - .type('bad_db_uri'); - - // click save - cy.get('[data-test="modal-confirm-button"]:not(:disabled)').click(); - - // should show error alerts and keep modal open - cy.get('.toast').contains('error'); - cy.wait(1000); // wait for potential (incorrect) closing annimation - cy.get('[data-test="database-modal"]').should('be.visible'); + it('should open dynamic form', () => { + // click postgres dynamic form + cy.get('.preferred > :nth-child(1)').click(); - // should be able to close modal - cy.get('[data-test="modal-cancel-button"]').click(); - cy.get('[data-test="database-modal"]').should('not.be.visible'); + // make sure all the fields are rendering + cy.get('input[name="host"]').should('have.value', ''); + cy.get('input[name="port"]').should('have.value', ''); + cy.get('input[name="database"]').should('have.value', ''); + cy.get('input[name="password"]').should('have.value', ''); + cy.get('input[name="database_name"]').should('have.value', ''); }); - it('should keep update modal open when error', () => { - // open modal - cy.get('[data-test="database-edit"]:last').click(); - - // it should show saved values - cy.get('[data-test="database-modal"]:last input[name="sqlalchemy_uri"]') - .invoke('val') - .should('not.be.empty'); - cy.get('[data-test="database-modal"] input[name="database_name"]') - .invoke('val') - .should('not.be.empty'); + it('should open sqlalchemy form', () => { + // click postgres dynamic form + cy.get('.preferred > :nth-child(1)').click(); - cy.get('[data-test="database-modal"]:last input[name="sqlalchemy_uri"]') - .focus() - .dblclick() - .type('{selectall}{backspace}bad_uri'); + cy.get('[data-test="sqla-connect-btn"]').click(); - // click save - cy.get('[data-test="modal-confirm-button"]:not(:disabled)').click(); + // check if the sqlalchemy form is showing up + cy.get('[data-test=database-name-input]').should('be.visible'); + cy.get('[data-test="sqlalchemy-uri-input"]').should('be.visible'); + }); - // should show error alerts - // TODO(hugh): Update this test - // cy.get('.toast').contains('error').should('be.visible'); + it('show error alerts on dynamic form for bad host', () => { + // click postgres dynamic form + cy.get('.preferred > :nth-child(1)').click(); + cy.get('input[name="host"]').focus().type('badhost'); + cy.get('input[name="port"]').focus().type('5432'); + cy.get('.ant-form-item-explain-error').contains( + "The hostname provided can't be resolved", + ); + }); - // modal should still be open - // cy.wait(1000); // wait for potential (incorrect) closing annimation - // cy.get('[data-test="database-modal"]').should('be.visible'); + it('show error alerts on dynamic form for bad port', () => { + // click postgres dynamic form + cy.get('.preferred > :nth-child(1)').click(); + cy.get('input[name="host"]').focus().type('localhost'); + cy.get('input[name="port"]').focus().type('123'); + cy.get('input[name="database"]').focus(); + cy.get('.ant-form-item-explain-error').contains('The port is closed'); }); }); diff --git a/superset-frontend/cypress-base/cypress/integration/explore/control.test.ts b/superset-frontend/cypress-base/cypress/integration/explore/control.test.ts index e6d8b5676ebd..fc81c92dde00 100644 --- a/superset-frontend/cypress-base/cypress/integration/explore/control.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/explore/control.test.ts @@ -99,20 +99,14 @@ describe('VizType control', () => { cy.visitChartByName('Daily Totals'); cy.verifySliceSuccess({ waitAlias: '@tableChartData' }); - let numScripts = 0; - cy.get('script').then(nodes => { - numScripts = nodes.length; - }); - cy.get('[data-test="visualization-type"]').contains('Table').click(); + cy.get('button').contains('Evolution').click(); // change categories cy.get('[role="button"]').contains('Line Chart').click(); + cy.get('button').contains('Select').click(); // should load mathjs for line chart cy.get('script[src*="mathjs"]').should('have.length', 1); - cy.get('script').then(nodes => { - expect(nodes.length).to.greaterThan(numScripts); - }); cy.get('button[data-test="run-query-button"]').click(); cy.verifySliceSuccess({ diff --git a/superset-frontend/cypress-base/cypress/integration/explore/visualizations/line.test.ts b/superset-frontend/cypress-base/cypress/integration/explore/visualizations/line.test.ts index 26660bc5fa67..f75fe77cceed 100644 --- a/superset-frontend/cypress-base/cypress/integration/explore/visualizations/line.test.ts +++ b/superset-frontend/cypress-base/cypress/integration/explore/visualizations/line.test.ts @@ -73,13 +73,12 @@ describe('Visualization > Line', () => { it('should allow type to search color schemes', () => { cy.get('#controlSections-tab-display').click(); cy.get('.Control[data-test="color_scheme"]').scrollIntoView(); - cy.get('.Control[data-test="color_scheme"] input[type="text"]') + cy.get('.Control[data-test="color_scheme"] input[type="search"]') .focus() - .type('air{enter}'); - cy.get('input[name="select-color_scheme"]').should( - 'have.value', - 'bnbColors', - ); + .type('bnbColors{enter}'); + cy.get( + '.Control[data-test="color_scheme"] .ant-select-selection-item > ul[data-test="bnbColors"]', + ).should('exist'); }); it('should work with adhoc metric', () => { diff --git a/superset-frontend/images/icons/alert_solid_small.svg b/superset-frontend/images/icons/alert_solid_small.svg index 092c37f1ce9a..b590be2a0e60 100644 --- a/superset-frontend/images/icons/alert_solid_small.svg +++ b/superset-frontend/images/icons/alert_solid_small.svg @@ -17,6 +17,6 @@ specific language governing permissions and limitations under the License. --> diff --git a/superset-frontend/images/icons/ballot.svg b/superset-frontend/images/icons/ballot.svg new file mode 100644 index 000000000000..00d1a235ce11 --- /dev/null +++ b/superset-frontend/images/icons/ballot.svg @@ -0,0 +1,22 @@ + + + diff --git a/superset-frontend/images/icons/category.svg b/superset-frontend/images/icons/category.svg new file mode 100644 index 000000000000..471cf254d022 --- /dev/null +++ b/superset-frontend/images/icons/category.svg @@ -0,0 +1,22 @@ + + + diff --git a/superset-frontend/images/icons/default_db_image.svg b/superset-frontend/images/icons/default_db_image.svg new file mode 100644 index 000000000000..f8892bb31241 --- /dev/null +++ b/superset-frontend/images/icons/default_db_image.svg @@ -0,0 +1,21 @@ + + diff --git a/superset-frontend/images/icons/more_vert.svg b/superset-frontend/images/icons/more_vert.svg new file mode 100644 index 000000000000..2fbe6287d86e --- /dev/null +++ b/superset-frontend/images/icons/more_vert.svg @@ -0,0 +1,21 @@ + + diff --git a/superset-frontend/images/icons/tags.svg b/superset-frontend/images/icons/tags.svg new file mode 100644 index 000000000000..36a061ecf33b --- /dev/null +++ b/superset-frontend/images/icons/tags.svg @@ -0,0 +1,22 @@ + + + diff --git a/superset-frontend/images/netezza.png b/superset-frontend/images/netezza.png new file mode 100644 index 000000000000..2658d8629bf1 Binary files /dev/null and b/superset-frontend/images/netezza.png differ diff --git a/superset-frontend/jest.config.js b/superset-frontend/jest.config.js index 99a121561bf3..191f7a5e1df1 100644 --- a/superset-frontend/jest.config.js +++ b/superset-frontend/jest.config.js @@ -20,7 +20,7 @@ module.exports = { testRegex: '(\\/spec|\\/src)\\/.*(_spec|\\.test)\\.(j|t)sx?$', moduleNameMapper: { '\\.(css|less)$': '