From accb48e4296dca9fb6b30ac0eea0ea91de45db9a Mon Sep 17 00:00:00 2001 From: AAfghahi <48933336+AAfghahi@users.noreply.github.com> Date: Thu, 8 Jul 2021 15:58:44 -0400 Subject: [PATCH 01/56] feat: Make Google Sheets Dyanmic (#15576) * first draft * second draft * added tests --- superset/db_engine_specs/gsheets.py | 53 +++++++++++++++++++ .../integration_tests/databases/api_tests.py | 21 ++++++++ 2 files changed, 74 insertions(+) diff --git a/superset/db_engine_specs/gsheets.py b/superset/db_engine_specs/gsheets.py index 0becf050647e..7e069baeaa37 100644 --- a/superset/db_engine_specs/gsheets.py +++ b/superset/db_engine_specs/gsheets.py @@ -19,13 +19,18 @@ from contextlib import closing from typing import Any, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING +from apispec import APISpec +from apispec.ext.marshmallow import MarshmallowPlugin from flask import g from flask_babel import gettext as __ +from marshmallow import fields, Schema +from marshmallow.exceptions import ValidationError from sqlalchemy.engine import create_engine from sqlalchemy.engine.url import URL from typing_extensions import TypedDict from superset import security_manager +from superset.databases.schemas import encrypted_field_properties, EncryptedField from superset.db_engine_specs.sqlite import SqliteEngineSpec from superset.errors import ErrorLevel, SupersetError, SupersetErrorType @@ -35,6 +40,15 @@ SYNTAX_ERROR_REGEX = re.compile('SQLError: near "(?P.*?)": syntax error') +ma_plugin = MarshmallowPlugin() + + +class GSheetsParametersSchema(Schema): + credentials_info = EncryptedField( + required=False, description="Contents of Google Sheets JSON credentials.", + ) + table_catalog = fields.Dict(required=False) + class GSheetsParametersType(TypedDict): credentials_info: Dict[str, Any] @@ -50,6 +64,10 @@ class GSheetsEngineSpec(SqliteEngineSpec): allows_joins = True allows_subqueries = True + parameters_schema = GSheetsParametersSchema() + default_driver = "apsw" + sqlalchemy_uri_placeholder = "gsheets://" + custom_errors: Dict[Pattern[str], Tuple[str, SupersetErrorType, Dict[str, Any]]] = { SYNTAX_ERROR_REGEX: ( __( @@ -87,6 +105,41 @@ def extra_table_metadata( return {"metadata": metadata["extra"]} + @classmethod + def build_sqlalchemy_uri(cls) -> str: # pylint: disable=unused-variable + + return "gsheets://" + + @classmethod + def get_parameters_from_uri( + cls, encrypted_extra: Optional[Dict[str, str]] = None, + ) -> Any: + # Building parameters from encrypted_extra and uri + if encrypted_extra: + return {**encrypted_extra} + + raise ValidationError("Invalid service credentials") + + @classmethod + def parameters_json_schema(cls) -> Any: + """ + Return configuration parameters as OpenAPI. + """ + if not cls.parameters_schema: + return None + + spec = APISpec( + title="Database Parameters", + version="1.0.0", + openapi_version="3.0.0", + plugins=[ma_plugin], + ) + + ma_plugin.init_spec(spec) + ma_plugin.converter.add_attribute_function(encrypted_field_properties) + spec.components.schema(cls.__name__, schema=cls.parameters_schema) + return spec.to_dict()["components"]["schemas"][cls.__name__] + @classmethod def validate_parameters( cls, parameters: GSheetsParametersType, diff --git a/tests/integration_tests/databases/api_tests.py b/tests/integration_tests/databases/api_tests.py index 2c9fbb9f9cc1..d1590a97fba1 100644 --- a/tests/integration_tests/databases/api_tests.py +++ b/tests/integration_tests/databases/api_tests.py @@ -39,6 +39,7 @@ from superset.db_engine_specs.postgres import PostgresEngineSpec from superset.db_engine_specs.redshift import RedshiftEngineSpec from superset.db_engine_specs.bigquery import BigQueryEngineSpec +from superset.db_engine_specs.gsheets import GSheetsEngineSpec from superset.db_engine_specs.hana import HanaEngineSpec from superset.errors import SupersetError from superset.models.core import Database, ConfigurationMethod @@ -1437,6 +1438,7 @@ def test_available(self, app, get_available_engine_specs): PostgresEngineSpec: {"psycopg2"}, BigQueryEngineSpec: {"bigquery"}, MySQLEngineSpec: {"mysqlconnector", "mysqldb"}, + GSheetsEngineSpec: {"apsw"}, RedshiftEngineSpec: {"psycopg2"}, HanaEngineSpec: {""}, } @@ -1564,6 +1566,25 @@ def test_available(self, app, get_available_engine_specs): "preferred": False, "sqlalchemy_uri_placeholder": "redshift+psycopg2://user:password@host:port/dbname[?key=value&key=value...]", }, + { + "available_drivers": ["apsw"], + "default_driver": "apsw", + "engine": "gsheets", + "name": "Google Sheets", + "parameters": { + "properties": { + "credentials_info": { + "description": "Contents of Google Sheets JSON credentials.", + "type": "string", + "x-encrypted-extra": True, + }, + "table_catalog": {"type": "object"}, + }, + "type": "object", + }, + "preferred": False, + "sqlalchemy_uri_placeholder": "gsheets://", + }, { "available_drivers": ["mysqlconnector", "mysqldb"], "default_driver": "mysqldb", From eb5f6075a5b10bbada22df483f8a1df2a82ef4af Mon Sep 17 00:00:00 2001 From: Arash Date: Thu, 8 Jul 2021 17:55:35 -0400 Subject: [PATCH 02/56] first draft --- .../DatabaseModal/DatabaseConnectionForm.tsx | 52 ++++++++++++++----- .../data/database/DatabaseModal/index.tsx | 15 +++++- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx index ca4fd3a3af54..0ae851420055 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx @@ -66,6 +66,7 @@ interface FieldPropTypes { sslForced?: boolean; defaultDBName?: string; editNewDb?: boolean; + setPublicSheets: (value: string) => void; } const CredentialsInfo = ({ @@ -299,19 +300,42 @@ const displayField = ({ getValidation, validationErrors, db, + setPublicSheets, }: FieldPropTypes) => ( - + <> + + + {db?.engine === 'gsheets' && ( + <> + {t('Type of Google Sheets Allowed')} + + + )} + ); const queryField = ({ @@ -388,10 +412,13 @@ const DatabaseConnectionForm = ({ isEditMode = false, sslForced, editNewDb, + setPublicSheets, }: { isEditMode?: boolean; sslForced: boolean; editNewDb?: boolean; + isPublic?: boolean; + setPublicSheets: (value: string) => void; dbModel: DatabaseForm; db: Partial | null; onParametersChange: ( @@ -434,6 +461,7 @@ const DatabaseConnectionForm = ({ isEditMode, sslForced, editNewDb, + setPublicSheets, }), )} diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx index 6c397d41d6b1..df5c8ab13030 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx @@ -338,6 +338,7 @@ const DatabaseModal: FunctionComponent = ({ const [dbName, setDbName] = useState(''); const [editNewDb, setEditNewDb] = useState(false); const [isLoading, setLoading] = useState(false); + const [isPublic, setPublic] = useState(true); const conf = useCommonConf(); const dbImages = getDatabaseImages(); const connectionAlert = getConnectionAlert(); @@ -360,7 +361,7 @@ const DatabaseModal: FunctionComponent = ({ t('database'), addDangerToast, ); - + console.log(isPublic); const isDynamic = (engine: string | undefined) => availableDbs?.databases.filter( (DB: DatabaseObject) => DB.backend === engine || DB.engine === engine, @@ -530,6 +531,14 @@ const DatabaseModal: FunctionComponent = ({ } }; + const setPublicSheets = (value: string) => { + if (value === 'true') { + setPublic(true); + } else { + setPublic(false); + } + }; + const setDatabaseModel = (engine: string) => { const selectedDbModel = availableDbs?.databases.filter( (db: DatabaseObject) => db.engine === engine, @@ -802,6 +811,7 @@ const DatabaseModal: FunctionComponent = ({ isEditMode sslForced={sslForced} dbModel={dbModel} + setPublicSheets={setPublicSheets} db={db as DatabaseObject} onParametersChange={({ target }: { target: HTMLInputElement }) => onChange(ActionType.parametersChange, { @@ -912,6 +922,8 @@ const DatabaseModal: FunctionComponent = ({ ) : ( = ({ )} Date: Fri, 9 Jul 2021 13:24:53 -0400 Subject: [PATCH 03/56] added table_catalog --- .../DatabaseModal/DatabaseConnectionForm.tsx | 150 ++++++++++-------- .../data/database/DatabaseModal/index.tsx | 4 +- 2 files changed, 88 insertions(+), 66 deletions(-) diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx index 0ae851420055..c151a067586e 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx @@ -46,6 +46,7 @@ export const FormFieldOrder = [ 'password', 'database_name', 'credentials_info', + 'table_catalog', 'query', 'encryption', ]; @@ -67,6 +68,7 @@ interface FieldPropTypes { defaultDBName?: string; editNewDb?: boolean; setPublicSheets: (value: string) => void; + isPublic?: boolean; } const CredentialsInfo = ({ @@ -74,6 +76,7 @@ const CredentialsInfo = ({ isEditMode, db, editNewDb, + isPublic, }: FieldPropTypes) => { const [uploadOption, setUploadOption] = useState( CredentialInfoOptions.jsonUpload.valueOf(), @@ -81,9 +84,10 @@ const CredentialsInfo = ({ const [fileToUpload, setFileToUpload] = useState( null, ); + console.log('in credentials', isPublic); return ( - {!isEditMode && ( + {!isEditMode && db?.engine === 'bigquery' && ( <> {t('How do you want to enter service account credentials?')} @@ -105,9 +109,10 @@ const CredentialsInfo = ({ )} {uploadOption === CredentialInfoOptions.copyPaste || isEditMode || - editNewDb ? ( + editNewDb || + (db?.engine === 'gsheets' && !isPublic) ? (
- {t('Service Account')} + {t('Service Account Information')}