diff --git a/.pylintrc b/.pylintrc index 9d84756f162a..e94c7becd4b3 100644 --- a/.pylintrc +++ b/.pylintrc @@ -217,7 +217,7 @@ max-nested-blocks=5 [FORMAT] # Maximum number of characters on a single line. -max-line-length=88 +max-line-length=90 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx index d114ba06c0a0..f6997fc84e9c 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/DatabaseConnectionForm.tsx @@ -19,19 +19,21 @@ import React, { FormEvent, useState } from 'react'; import { SupersetTheme, JsonObject, t } from '@superset-ui/core'; import { InputProps } from 'antd/lib/input'; -import { Switch, Select, Button } from 'src/common/components'; +import { Input, Switch, Select, Button } from 'src/common/components'; import InfoTooltip from 'src/components/InfoTooltip'; import ValidatedInput from 'src/components/Form/LabeledErrorBoundInput'; import FormLabel from 'src/components/Form/FormLabel'; -import { DeleteFilled } from '@ant-design/icons'; +import { DeleteFilled, CloseOutlined } from '@ant-design/icons'; import { formScrollableStyles, validatedFormStyles, CredentialInfoForm, toggleStyle, infoTooltip, + StyledFooterButton, + StyledCatalogTable, } from './styles'; -import { DatabaseForm, DatabaseObject } from '../types'; +import { CatalogObject, DatabaseForm, DatabaseObject } from '../types'; enum CredentialInfoOptions { jsonUpload, @@ -46,6 +48,7 @@ export const FormFieldOrder = [ 'password', 'database_name', 'credentials_info', + 'catalog', 'query', 'encryption', ]; @@ -58,7 +61,10 @@ interface FieldPropTypes { onParametersUploadFileChange: (value: any) => string; changeMethods: { onParametersChange: (value: any) => string } & { onChange: (value: any) => string; - } & { onParametersUploadFileChange: (value: any) => string }; + } & { onParametersUploadFileChange: (value: any) => string } & { + onAddTableCatalog: () => void; + onRemoveTableCatalog: (idx: number) => void; + }; validationErrors: JsonObject | null; getValidation: () => void; db?: DatabaseObject; @@ -187,6 +193,89 @@ const CredentialsInfo = ({ ); }; +const TableCatalog = ({ + required, + changeMethods, + getValidation, + validationErrors, + db, +}: FieldPropTypes) => { + const tableCatalog = db?.catalog || []; + const catalogError = validationErrors || {}; + return ( + +
+ {t('Type of Google Sheets Allowed')} + +
+

+ {t('Connect Google Sheets as tables to this database')} +

+
+ {tableCatalog?.map((sheet: CatalogObject, idx: number) => ( + <> + + {t('Google Sheet Name and URL')} + +
+ { + changeMethods.onParametersChange({ + target: { + type: `catalog-${idx}`, + name: 'name', + value: e.target.value, + }, + }); + }} + value={sheet.name} + /> + + {tableCatalog?.length > 1 && ( + changeMethods.onRemoveTableCatalog(idx)} + /> + )} +
+ + changeMethods.onParametersChange({ + target: { + type: `catalog-${idx}`, + name: 'value', + value: e.target.value, + }, + }) + } + value={sheet.value} + /> + + ))} + { + changeMethods.onAddTableCatalog(); + }} + > + + {t('Add sheet')} + +
+ + ); +}; + const hostField = ({ required, changeMethods, @@ -300,18 +389,22 @@ const displayField = ({ validationErrors, db, }: FieldPropTypes) => ( - + <> + + ); const queryField = ({ @@ -375,6 +468,7 @@ const FORM_FIELD_MAP = { query: queryField, encryption: forceSSLField, credentials_info: CredentialsInfo, + catalog: TableCatalog, }; const DatabaseConnectionForm = ({ @@ -382,6 +476,8 @@ const DatabaseConnectionForm = ({ onParametersChange, onChange, onParametersUploadFileChange, + onAddTableCatalog, + onRemoveTableCatalog, validationErrors, getValidation, db, @@ -403,6 +499,8 @@ const DatabaseConnectionForm = ({ onParametersUploadFileChange?: ( event: FormEvent | { target: HTMLInputElement }, ) => void; + onAddTableCatalog: () => void; + onRemoveTableCatalog: (idx: number) => void; validationErrors: JsonObject | null; getValidation: () => void; }) => ( @@ -426,6 +524,8 @@ const DatabaseConnectionForm = ({ onParametersChange, onChange, onParametersUploadFileChange, + onAddTableCatalog, + onRemoveTableCatalog, }, validationErrors, getValidation, diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx index 8e047be4b9ef..13c0fc15c3fb 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/index.tsx @@ -50,6 +50,7 @@ import { DatabaseObject, DatabaseForm, CONFIGURATION_METHOD, + CatalogObject, } from 'src/views/CRUD/data/database/types'; import Loading from 'src/components/Loading'; import ExtraOptions from './ExtraOptions'; @@ -101,10 +102,15 @@ const errorAlertMapping = { message: 'Invalid account information', description: 'Either the username or password is incorrect.', }, - INVALID_PAYLOAD_SCHEMA: { + INVALID_PAYLOAD_SCHEMA_ERROR: { message: 'Incorrect Fields', description: 'Please make sure all fields are filled out correctly', }, + TABLE_DOES_NOT_EXIST_ERROR: { + message: 'URL could not be identified', + description: + 'The URL could not be identified. Please check for typos and make sure that "Type of google sheet allowed" selection matches the input', + }, }; interface DatabaseModalProps { addDangerToast: (msg: string) => void; @@ -126,6 +132,8 @@ enum ActionType { textChange, extraInputChange, extraEditorChange, + addTableCatalogSheet, + removeTableCatalogSheet, } interface DBReducerPayloadType { @@ -161,7 +169,13 @@ type DBReducerActionType = }; } | { - type: ActionType.reset; + type: ActionType.reset | ActionType.addTableCatalogSheet; + } + | { + type: ActionType.removeTableCatalogSheet; + payload: { + indexToDelete: number; + }; } | { type: ActionType.configMethodChange; @@ -180,6 +194,9 @@ function dbReducer( ...(state || {}), }; let query = ''; + let deserializeExtraJSON = {}; + let extra_json: DatabaseObject['extra_json']; + switch (action.type) { case ActionType.extraEditorChange: return { @@ -227,6 +244,29 @@ function dbReducer( [action.payload.name]: action.payload.value, }; case ActionType.parametersChange: + if ( + trimmedState.catalog !== undefined && + action.payload.type?.startsWith('catalog') + ) { + // Formatting wrapping google sheets table catalog + const idx = action.payload.type?.split('-')[1]; + const catalogToUpdate = trimmedState?.catalog[idx] || {}; + catalogToUpdate[action.payload.name] = action.payload.value; + + const paramatersCatalog = {}; + // eslint-disable-next-line array-callback-return + trimmedState.catalog?.map((item: CatalogObject) => { + paramatersCatalog[item.name] = item.value; + }); + + return { + ...trimmedState, + parameters: { + ...trimmedState.parameters, + catalog: paramatersCatalog, + }, + }; + } return { ...trimmedState, parameters: { @@ -234,6 +274,22 @@ function dbReducer( [action.payload.name]: action.payload.value, }, }; + case ActionType.addTableCatalogSheet: + if (trimmedState.catalog !== undefined) { + return { + ...trimmedState, + catalog: [...trimmedState.catalog, { name: '', value: '' }], + }; + } + return { + ...trimmedState, + catalog: [{ name: '', value: '' }], + }; + case ActionType.removeTableCatalogSheet: + trimmedState.catalog?.splice(action.payload.indexToDelete, 1); + return { + ...trimmedState, + }; case ActionType.editorChange: return { ...trimmedState, @@ -246,10 +302,8 @@ function dbReducer( }; case ActionType.fetched: // convert all the keys in this payload into strings - // eslint-disable-next-line no-case-declarations - let deserializeExtraJSON = {}; if (action.payload.extra) { - const extra_json = { + extra_json = { ...JSON.parse(action.payload.extra || ''), } as DatabaseObject['extra_json']; @@ -262,13 +316,6 @@ function dbReducer( }; } - if (action.payload?.parameters?.query) { - // convert query into URI params string - query = new URLSearchParams( - action.payload.parameters.query as string, - ).toString(); - } - if ( action.payload.backend === 'bigquery' && action.payload.configuration_method === @@ -288,6 +335,46 @@ function dbReducer( }; } + if ( + action.payload.backend === 'gsheets' && + action.payload.configuration_method === + CONFIGURATION_METHOD.DYNAMIC_FORM && + extra_json?.engine_params?.catalog !== undefined + ) { + // pull catalog from engine params + const engineParamsCatalog = extra_json?.engine_params?.catalog; + + return { + ...action.payload, + engine: action.payload.backend, + configuration_method: action.payload.configuration_method, + extra_json: deserializeExtraJSON, + catalog: Object.keys(engineParamsCatalog).map(e => ({ + name: e, + value: engineParamsCatalog[e], + })), + } as DatabaseObject; + } + + if (action.payload?.parameters?.query) { + // convert query into URI params string + query = new URLSearchParams( + action.payload.parameters.query as string, + ).toString(); + + return { + ...action.payload, + encrypted_extra: action.payload.encrypted_extra || '', + engine: action.payload.backend || trimmedState.engine, + configuration_method: action.payload.configuration_method, + extra_json: deserializeExtraJSON, + parameters: { + ...action.payload.parameters, + query, + }, + }; + } + return { ...action.payload, encrypted_extra: action.payload.encrypted_extra || '', @@ -296,9 +383,9 @@ function dbReducer( extra_json: deserializeExtraJSON, parameters: { ...action.payload.parameters, - query, }, }; + case ActionType.dbSelected: return { ...action.payload, @@ -319,7 +406,9 @@ const serializeExtra = (extraJson: DatabaseObject['extra_json']) => JSON.stringify({ ...extraJson, metadata_params: JSON.parse((extraJson?.metadata_params as string) || '{}'), - engine_params: JSON.parse((extraJson?.engine_params as string) || '{}'), + engine_params: JSON.parse( + ((extraJson?.engine_params as unknown) as string) || '{}', + ), schemas_allowed_for_csv_upload: (extraJson?.schemas_allowed_for_csv_upload as string) || '[]', }); @@ -369,7 +458,6 @@ const DatabaseModal: FunctionComponent = ({ t('database'), addDangerToast, ); - const isDynamic = (engine: string | undefined) => availableDbs?.databases.filter( (DB: DatabaseObject) => DB.backend === engine || DB.engine === engine, @@ -435,7 +523,10 @@ const DatabaseModal: FunctionComponent = ({ .replace(/&/g, '","') .replace(/=/g, '":"')}"}`, ); - } else if (dbToUpdate?.parameters?.query === '') { + } else if ( + dbToUpdate?.parameters?.query === '' && + 'query' in dbModel.parameters + ) { dbToUpdate.parameters.query = {}; } @@ -466,6 +557,15 @@ const DatabaseModal: FunctionComponent = ({ } } + if (dbToUpdate.parameters.catalog) { + // need to stringify gsheets catalog to allow it to be seralized + dbToUpdate.extra_json = { + engine_params: JSON.stringify({ + catalog: dbToUpdate.parameters.catalog, + }), + }; + } + if (dbToUpdate?.extra_json) { // convert extra_json to back to string dbToUpdate.extra = serializeExtra(dbToUpdate?.extra_json); @@ -545,6 +645,7 @@ const DatabaseModal: FunctionComponent = ({ engine, }, }); + setDB({ type: ActionType.addTableCatalogSheet }); }; const renderAvailableSelector = () => ( @@ -816,6 +917,15 @@ const DatabaseModal: FunctionComponent = ({ value: target.value, }) } + onAddTableCatalog={() => + setDB({ type: ActionType.addTableCatalogSheet }) + } + onRemoveTableCatalog={(idx: number) => + setDB({ + type: ActionType.removeTableCatalogSheet, + payload: { indexToDelete: idx }, + }) + } getValidation={() => getValidation(db)} validationErrors={validationErrors} /> @@ -928,6 +1038,15 @@ const DatabaseModal: FunctionComponent = ({ value: target.value, }) } + onAddTableCatalog={() => + setDB({ type: ActionType.addTableCatalogSheet }) + } + onRemoveTableCatalog={(idx: number) => + setDB({ + type: ActionType.removeTableCatalogSheet, + payload: { indexToDelete: idx }, + }) + } getValidation={() => getValidation(db)} validationErrors={validationErrors} /> @@ -1030,7 +1149,7 @@ const DatabaseModal: FunctionComponent = ({ ) : ( <> - {/* Step 1 */} + {/* Dyanmic Form Step 1 */} {!isLoading && (!db ? ( @@ -1073,6 +1192,15 @@ const DatabaseModal: FunctionComponent = ({ db={db} sslForced={sslForced} dbModel={dbModel} + onAddTableCatalog={() => { + setDB({ type: ActionType.addTableCatalogSheet }); + }} + onRemoveTableCatalog={(idx: number) => { + setDB({ + type: ActionType.removeTableCatalogSheet, + payload: { indexToDelete: idx }, + }); + }} onParametersChange={({ target, }: { diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/styles.ts b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/styles.ts index de6562845cdf..9a4a2a1237bb 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/styles.ts +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/styles.ts @@ -537,3 +537,43 @@ export const StyledStickyHeader = styled.div` z-index: ${({ theme }) => theme.zIndex.max}; background: ${({ theme }) => theme.colors.grayscale.light5}; `; + +export const StyledCatalogTable = styled.div` + margin-bottom: 16px; + + .catalog-type-select { + margin: 0 0 40px; + } + + .gsheet-title { + font-size: ${({ theme }) => theme.typography.sizes.l * 1.1}px; + font-weight: bold; + margin: ${({ theme }) => theme.gridUnit * 6}px 0 16px; + } + + .catalog-label { + margin: 0 0 8px; + } + + .catalog-name { + display: flex; + .catalog-name-input { + width: 95%; + } + } + + .catalog-name-url { + margin: 4px 0; + width: 95%; + } + + .catalog-delete { + align-self: center; + background: ${({ theme }) => theme.colors.grayscale.light4}; + margin: 5px; + } + + .catalog-add-btn { + width: 95%; + } +`; diff --git a/superset-frontend/src/views/CRUD/data/database/types.ts b/superset-frontend/src/views/CRUD/data/database/types.ts index 3e87c6bbb7bf..4cf4b8475a35 100644 --- a/superset-frontend/src/views/CRUD/data/database/types.ts +++ b/superset-frontend/src/views/CRUD/data/database/types.ts @@ -21,6 +21,11 @@ type DatabaseUser = { last_name: string; }; +export type CatalogObject = { + name: string; + value: string; +}; + export type DatabaseObject = { // Connection + general id?: number; @@ -41,10 +46,14 @@ export type DatabaseObject = { encryption?: boolean; credentials_info?: string; query?: string | object; + catalog?: {}; }; configuration_method: CONFIGURATION_METHOD; engine?: string; + // Gsheets temporary storage + catalog?: Array; + // Performance cache_timeout?: string; allow_run_async?: boolean; @@ -65,7 +74,9 @@ export type DatabaseObject = { // Extra extra_json?: { - engine_params?: {} | string; + engine_params?: { + catalog: Record | string; + }; metadata_params?: {} | string; metadata_cache_timeout?: { schema_cache_timeout?: number; // in Performance diff --git a/superset-frontend/src/views/CRUD/hooks.ts b/superset-frontend/src/views/CRUD/hooks.ts index 33c1fe353a45..c1ee1f52c69d 100644 --- a/superset-frontend/src/views/CRUD/hooks.ts +++ b/superset-frontend/src/views/CRUD/hooks.ts @@ -674,7 +674,11 @@ export function useDatabaseValidation() { message, }: { error_type: string; - extra: { invalid?: string[]; missing?: string[] }; + extra: { + invalid?: string[]; + missing?: string[]; + name: string; + }; message: string; }, ) => { @@ -682,6 +686,13 @@ export function useDatabaseValidation() { // error can't be mapped to a parameter // so leave it alone if (extra.invalid) { + if (extra.invalid[0] === 'catalog') { + return { + ...obj, + [extra.name]: message, + error_type, + }; + } return { ...obj, [extra.invalid[0]]: message, diff --git a/superset/databases/commands/validate.py b/superset/databases/commands/validate.py index e59cab570f36..fcbff1fed70f 100644 --- a/superset/databases/commands/validate.py +++ b/superset/databases/commands/validate.py @@ -64,7 +64,7 @@ def run(self) -> None: ), ) engine_spec = engine_specs[engine] - if not issubclass(engine_spec, BasicParametersMixin): + if not hasattr(engine_spec, "parameters_schema"): raise InvalidEngineError( SupersetError( message=__( @@ -85,7 +85,9 @@ def run(self) -> None: ) # perform initial validation - errors = engine_spec.validate_parameters(self._properties.get("parameters", {})) + errors = engine_spec.validate_parameters( # type: ignore + self._properties.get("parameters", {}) + ) if errors: raise InvalidParametersError(errors) @@ -96,9 +98,8 @@ def run(self) -> None: encrypted_extra = {} # try to connect - sqlalchemy_uri = engine_spec.build_sqlalchemy_uri( - self._properties.get("parameters"), # type: ignore - encrypted_extra, + sqlalchemy_uri = engine_spec.build_sqlalchemy_uri( # type: ignore + self._properties.get("parameters"), encrypted_extra, ) if self._model and sqlalchemy_uri == self._model.safe_sqlalchemy_uri(): sqlalchemy_uri = self._model.sqlalchemy_uri_decrypted diff --git a/superset/db_engine_specs/gsheets.py b/superset/db_engine_specs/gsheets.py index 0becf050647e..774924fde71e 100644 --- a/superset/db_engine_specs/gsheets.py +++ b/superset/db_engine_specs/gsheets.py @@ -19,13 +19,18 @@ from contextlib import closing from typing import Any, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING +from apispec import APISpec +from apispec.ext.marshmallow import MarshmallowPlugin from flask import g from flask_babel import gettext as __ +from marshmallow import fields, Schema +from marshmallow.exceptions import ValidationError from sqlalchemy.engine import create_engine from sqlalchemy.engine.url import URL from typing_extensions import TypedDict from superset import security_manager +from superset.databases.schemas import encrypted_field_properties from superset.db_engine_specs.sqlite import SqliteEngineSpec from superset.errors import ErrorLevel, SupersetError, SupersetErrorType @@ -35,11 +40,16 @@ SYNTAX_ERROR_REGEX = re.compile('SQLError: near "(?P.*?)": syntax error') +ma_plugin = MarshmallowPlugin() + + +class GSheetsParametersSchema(Schema): + catalog = fields.Dict() + class GSheetsParametersType(TypedDict): credentials_info: Dict[str, Any] - query: Dict[str, Any] - table_catalog: Dict[str, str] + catalog: Dict[str, str] class GSheetsEngineSpec(SqliteEngineSpec): @@ -50,6 +60,10 @@ class GSheetsEngineSpec(SqliteEngineSpec): allows_joins = True allows_subqueries = True + parameters_schema = GSheetsParametersSchema() + default_driver = "apsw" + sqlalchemy_uri_placeholder = "gsheets://" + custom_errors: Dict[Pattern[str], Tuple[str, SupersetErrorType, Dict[str, Any]]] = { SYNTAX_ERROR_REGEX: ( __( @@ -87,16 +101,64 @@ def extra_table_metadata( return {"metadata": metadata["extra"]} + @classmethod + def build_sqlalchemy_uri( + cls, + _: GSheetsParametersType, + encrypted_extra: Optional[ # pylint: disable=unused-argument + Dict[str, Any] + ] = None, + ) -> str: # pylint: disable=unused-variable + + return "gsheets://" + + @classmethod + def get_parameters_from_uri( + cls, encrypted_extra: Optional[Dict[str, str]] = None, + ) -> Any: + # Building parameters from encrypted_extra and uri + if encrypted_extra: + return {**encrypted_extra} + + raise ValidationError("Invalid service credentials") + + @classmethod + def parameters_json_schema(cls) -> Any: + """ + Return configuration parameters as OpenAPI. + """ + if not cls.parameters_schema: + return None + + spec = APISpec( + title="Database Parameters", + version="1.0.0", + openapi_version="3.0.0", + plugins=[ma_plugin], + ) + + ma_plugin.init_spec(spec) + ma_plugin.converter.add_attribute_function(encrypted_field_properties) + spec.components.schema(cls.__name__, schema=cls.parameters_schema) + return spec.to_dict()["components"]["schemas"][cls.__name__] + @classmethod def validate_parameters( cls, parameters: GSheetsParametersType, ) -> List[SupersetError]: errors: List[SupersetError] = [] - credentials_info = parameters.get("credentials_info") - table_catalog = parameters.get("table_catalog", {}) + table_catalog = parameters.get("catalog", {}) if not table_catalog: + errors.append( + SupersetError( + message="URL is required", + error_type=SupersetErrorType.CONNECTION_MISSING_PARAMETERS_ERROR, + level=ErrorLevel.WARNING, + extra={"invalid": ["catalog"], "name": "", "url": ""}, + ), + ) return errors # We need a subject in case domain wide delegation is set, otherwise the @@ -110,17 +172,27 @@ def validate_parameters( ) conn = engine.connect() for name, url in table_catalog.items(): + + if not name: + errors.append( + SupersetError( + message="Sheet name is required", + error_type=SupersetErrorType.CONNECTION_MISSING_PARAMETERS_ERROR, + level=ErrorLevel.WARNING, + extra={"invalid": [], "name": name, "url": url}, + ), + ) + try: results = conn.execute(f'SELECT * FROM "{url}" LIMIT 1') results.fetchall() except Exception: # pylint: disable=broad-except errors.append( SupersetError( - message=f"Unable to connect to spreadsheet {name} at {url}", + message="URL could not be identified", error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR, level=ErrorLevel.WARNING, - extra={"name": name, "url": url}, + extra={"invalid": ["catalog"], "name": name, "url": url}, ), ) - return errors diff --git a/tests/integration_tests/databases/api_tests.py b/tests/integration_tests/databases/api_tests.py index d0d3d94e0413..7c3406bf11ec 100644 --- a/tests/integration_tests/databases/api_tests.py +++ b/tests/integration_tests/databases/api_tests.py @@ -39,6 +39,7 @@ from superset.db_engine_specs.postgres import PostgresEngineSpec from superset.db_engine_specs.redshift import RedshiftEngineSpec from superset.db_engine_specs.bigquery import BigQueryEngineSpec +from superset.db_engine_specs.gsheets import GSheetsEngineSpec from superset.db_engine_specs.hana import HanaEngineSpec from superset.errors import SupersetError from superset.models.core import Database, ConfigurationMethod @@ -1438,6 +1439,7 @@ def test_available(self, app, get_available_engine_specs): PostgresEngineSpec: {"psycopg2"}, BigQueryEngineSpec: {"bigquery"}, MySQLEngineSpec: {"mysqlconnector", "mysqldb"}, + GSheetsEngineSpec: {"apsw"}, RedshiftEngineSpec: {"psycopg2"}, HanaEngineSpec: {""}, } @@ -1565,6 +1567,18 @@ def test_available(self, app, get_available_engine_specs): "preferred": False, "sqlalchemy_uri_placeholder": "redshift+psycopg2://user:password@host:port/dbname[?key=value&key=value...]", }, + { + "available_drivers": ["apsw"], + "default_driver": "apsw", + "engine": "gsheets", + "name": "Google Sheets", + "parameters": { + "properties": {"catalog": {"type": "object"},}, + "type": "object", + }, + "preferred": False, + "sqlalchemy_uri_placeholder": "gsheets://", + }, { "available_drivers": ["mysqlconnector", "mysqldb"], "default_driver": "mysqldb", diff --git a/tests/unit_tests/db_engine_specs/test_gsheets.py b/tests/unit_tests/db_engine_specs/test_gsheets.py index ba0155d4359f..bd11375bb68b 100644 --- a/tests/unit_tests/db_engine_specs/test_gsheets.py +++ b/tests/unit_tests/db_engine_specs/test_gsheets.py @@ -37,11 +37,27 @@ def test_validate_parameters_simple( parameters: GSheetsParametersType = { "credentials_info": {}, - "query": {}, - "table_catalog": {}, + "catalog": {}, } errors = GSheetsEngineSpec.validate_parameters(parameters) - assert errors == [] + assert errors == [ + SupersetError( + message="URL is required", + error_type=SupersetErrorType.CONNECTION_MISSING_PARAMETERS_ERROR, + level=ErrorLevel.WARNING, + extra={ + "invalid": ["catalog"], + "name": "", + "url": "", + "issue_codes": [ + { + "code": 1018, + "message": "Issue 1018 - One or more parameters needed to configure a database are missing.", + } + ], + }, + ) + ] def test_validate_parameters_catalog( @@ -66,72 +82,57 @@ def test_validate_parameters_catalog( parameters: GSheetsParametersType = { "credentials_info": {}, - "query": {}, - "table_catalog": { + "catalog": { "private_sheet": "https://docs.google.com/spreadsheets/d/1/edit", "public_sheet": "https://docs.google.com/spreadsheets/d/1/edit#gid=1", "not_a_sheet": "https://www.google.com/", }, } - errors = GSheetsEngineSpec.validate_parameters(parameters) + errors = GSheetsEngineSpec.validate_parameters(parameters) # ignore: type + assert errors == [ SupersetError( - message=( - "Unable to connect to spreadsheet private_sheet at " - "https://docs.google.com/spreadsheets/d/1/edit" - ), + message="URL could not be identified", error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR, level=ErrorLevel.WARNING, extra={ + "invalid": ["catalog"], "name": "private_sheet", "url": "https://docs.google.com/spreadsheets/d/1/edit", "issue_codes": [ { "code": 1003, - "message": ( - "Issue 1003 - There is a syntax error in the SQL query. " - "Perhaps there was a misspelling or a typo." - ), + "message": "Issue 1003 - There is a syntax error in the SQL query. Perhaps there was a misspelling or a typo.", }, { "code": 1005, - "message": ( - "Issue 1005 - The table was deleted or renamed in the " - "database." - ), + "message": "Issue 1005 - The table was deleted or renamed in the database.", }, ], }, ), SupersetError( - message=( - "Unable to connect to spreadsheet not_a_sheet at " - "https://www.google.com/" - ), + message="URL could not be identified", error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR, level=ErrorLevel.WARNING, extra={ + "invalid": ["catalog"], "name": "not_a_sheet", "url": "https://www.google.com/", "issue_codes": [ { "code": 1003, - "message": ( - "Issue 1003 - There is a syntax error in the SQL query. " - "Perhaps there was a misspelling or a typo." - ), + "message": "Issue 1003 - There is a syntax error in the SQL query. Perhaps there was a misspelling or a typo.", }, { "code": 1005, - "message": ( - "Issue 1005 - The table was deleted or renamed in the " - "database.", - ), + "message": "Issue 1005 - The table was deleted or renamed in the database.", }, ], }, ), ] + create_engine.assert_called_with( "gsheets://", service_account_info={}, subject="admin@example.com", ) @@ -159,44 +160,36 @@ def test_validate_parameters_catalog_and_credentials( parameters: GSheetsParametersType = { "credentials_info": {}, - "query": {}, - "table_catalog": { + "catalog": { "private_sheet": "https://docs.google.com/spreadsheets/d/1/edit", "public_sheet": "https://docs.google.com/spreadsheets/d/1/edit#gid=1", "not_a_sheet": "https://www.google.com/", }, } - errors = GSheetsEngineSpec.validate_parameters(parameters) + errors = GSheetsEngineSpec.validate_parameters(parameters) # ignore: type assert errors == [ SupersetError( - message=( - "Unable to connect to spreadsheet not_a_sheet at " - "https://www.google.com/" - ), + message="URL could not be identified", error_type=SupersetErrorType.TABLE_DOES_NOT_EXIST_ERROR, level=ErrorLevel.WARNING, extra={ + "invalid": ["catalog"], "name": "not_a_sheet", "url": "https://www.google.com/", "issue_codes": [ { "code": 1003, - "message": ( - "Issue 1003 - There is a syntax error in the SQL query. " - "Perhaps there was a misspelling or a typo." - ), + "message": "Issue 1003 - There is a syntax error in the SQL query. Perhaps there was a misspelling or a typo.", }, { "code": 1005, - "message": ( - "Issue 1005 - The table was deleted or renamed in the " - "database.", - ), + "message": "Issue 1005 - The table was deleted or renamed in the database.", }, ], }, - ), + ) ] + create_engine.assert_called_with( "gsheets://", service_account_info={}, subject="admin@example.com", )