Skip to content

Commit

Permalink
feat: filter parameters from DB API (apache#21248)
Browse files Browse the repository at this point in the history
  • Loading branch information
betodealmeida committed Sep 2, 2022
1 parent 9af8a50 commit 358550f
Show file tree
Hide file tree
Showing 29 changed files with 445 additions and 81 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ describe('Nativefilters initial state not required', () => {
validateFilterContentOnDashboard(testItems.filterTimeGrain);
});

it('User can create a time range filter', () => {
xit('User can create a time range filter', () => {
enterNativeFilterEditModal();
fillNativeFilterForm(
testItems.filterType.timeRange,
Expand All @@ -414,7 +414,7 @@ describe('Nativefilters initial state not required', () => {
.should('be.visible');
});

it('User can create a time column filter', () => {
xit('User can create a time column filter', () => {
enterNativeFilterEditModal();
fillNativeFilterForm(
testItems.filterType.timeColumn,
Expand Down
2 changes: 1 addition & 1 deletion superset-frontend/src/types/Database.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export default interface Database {
id: number;
allow_run_async: boolean;
database_name: string;
encrypted_extra: string;
masked_encrypted_extra: string;
extra: string;
impersonate_user: boolean;
server_cert: string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,7 @@ export const EncryptedField = ({
const [isPublic, setIsPublic] = useState<boolean>(true);
const showCredentialsInfo =
db?.engine === 'gsheets' ? !isEditMode && !isPublic : !isEditMode;
// a database that has an optional encrypted field has an encrypted_extra that is an empty object, this checks for that.
const isEncrypted = isEditMode && db?.encrypted_extra !== '{}';
const isEncrypted = isEditMode && db?.masked_encrypted_extra !== '{}';
const encryptedField = db?.engine && encryptedCredentialsMap[db.engine];
const encryptedValue =
typeof db?.parameters?.[encryptedField] === 'object'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -344,11 +344,11 @@ const ExtraOptions = ({
<div className="control-label">{t('Secure extra')}</div>
<div className="input-container">
<StyledJsonEditor
name="encrypted_extra"
value={db?.encrypted_extra || ''}
name="masked_encrypted_extra"
value={db?.masked_encrypted_extra || ''}
placeholder={t('Secure extra')}
onChange={(json: string) =>
onEditorChange({ json, name: 'encrypted_extra' })
onEditorChange({ json, name: 'masked_encrypted_extra' })
}
width="100%"
height="160px"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ function dbReducer(
.join('&');

if (
action.payload.encrypted_extra &&
action.payload.masked_encrypted_extra &&
action.payload.configuration_method ===
CONFIGURATION_METHOD.DYNAMIC_FORM
) {
Expand All @@ -375,7 +375,7 @@ function dbReducer(
}
return {
...action.payload,
encrypted_extra: action.payload.encrypted_extra || '',
masked_encrypted_extra: action.payload.masked_encrypted_extra || '',
engine: action.payload.backend || trimmedState.engine,
configuration_method: action.payload.configuration_method,
extra_json: deserializeExtraJSON,
Expand Down Expand Up @@ -492,7 +492,7 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
database_name: db?.database_name?.trim() || undefined,
impersonate_user: db?.impersonate_user || undefined,
extra: serializeExtra(db?.extra_json) || undefined,
encrypted_extra: db?.encrypted_extra || '',
masked_encrypted_extra: db?.masked_encrypted_extra || '',
server_cert: db?.server_cert || undefined,
};
setTestInProgress(true);
Expand Down Expand Up @@ -559,10 +559,8 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
};

const onSave = async () => {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { id, ...update } = db || {};
// Clone DB object
const dbToUpdate = JSON.parse(JSON.stringify(update));
const dbToUpdate = JSON.parse(JSON.stringify(db || {}));

if (dbToUpdate.configuration_method === CONFIGURATION_METHOD.DYNAMIC_FORM) {
// Validate DB before saving
Expand All @@ -574,25 +572,26 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
? dbToUpdate.parameters_schema.properties
: dbModel?.parameters.properties;
const additionalEncryptedExtra = JSON.parse(
dbToUpdate.encrypted_extra || '{}',
dbToUpdate.masked_encrypted_extra || '{}',
);
const paramConfigArray = Object.keys(parameters_schema || {});

paramConfigArray.forEach(paramConfig => {
/*
* Parameters that are annotated with the `x-encrypted-extra` properties should be moved to
* `encrypted_extra`, so that they are stored encrypted in the backend when the database is
* created or edited.
* Parameters that are annotated with the `x-encrypted-extra` properties should be
* moved to `masked_encrypted_extra`, so that they are stored encrypted in the
* backend when the database is created or edited.
*/
if (
parameters_schema[paramConfig]['x-encrypted-extra'] &&
dbToUpdate.parameters?.[paramConfig]
) {
if (typeof dbToUpdate.parameters?.[paramConfig] === 'object') {
// add new encrypted extra to encrypted_extra object
// add new encrypted extra to masked_encrypted_extra object
additionalEncryptedExtra[paramConfig] =
dbToUpdate.parameters?.[paramConfig];
// The backend expects `encrypted_extra` as a string for historical reasons.
// The backend expects `masked_encrypted_extra` as a string for historical
// reasons.
dbToUpdate.parameters[paramConfig] = JSON.stringify(
dbToUpdate.parameters[paramConfig],
);
Expand All @@ -604,7 +603,9 @@ const DatabaseModal: FunctionComponent<DatabaseModalProps> = ({
}
});
// cast the new encrypted extra object into a string
dbToUpdate.encrypted_extra = JSON.stringify(additionalEncryptedExtra);
dbToUpdate.masked_encrypted_extra = JSON.stringify(
additionalEncryptedExtra,
);
// this needs to be added by default to gsheets
if (dbToUpdate.engine === Engines.GSheet) {
dbToUpdate.impersonate_user = true;
Expand Down
2 changes: 1 addition & 1 deletion superset-frontend/src/views/CRUD/data/database/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ export type DatabaseObject = {
force_ctas_schema?: string;

// Security
encrypted_extra?: string;
masked_encrypted_extra?: string;
server_cert?: string;
allow_file_upload?: boolean;
impersonate_user?: boolean;
Expand Down
2 changes: 2 additions & 0 deletions superset/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
# UUID for the examples database
EXAMPLES_DB_UUID = "a2dc77af-e654-49bb-b321-40f6b559a1ee"

PASSWORD_MASK = "X" * 10


class RouteMethod: # pylint: disable=too-few-public-methods
"""
Expand Down
2 changes: 1 addition & 1 deletion superset/databases/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
"force_ctas_schema",
"allow_multi_schema_metadata_fetch",
"impersonate_user",
"encrypted_extra",
"masked_encrypted_extra",
"extra",
"parameters",
"parameters_schema",
Expand Down
6 changes: 6 additions & 0 deletions superset/databases/commands/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,12 @@ def run(self) -> Model:
)
raise DatabaseConnectionFailedError() from ex

# when creating a new database we don't need to unmask encrypted extra
self._properties["encrypted_extra"] = self._properties.pop(
"masked_encrypted_extra",
"{}",
)

try:
database = DatabaseDAO.create(self._properties, commit=False)
database.set_sqlalchemy_uri(database.sqlalchemy_uri)
Expand Down
16 changes: 14 additions & 2 deletions superset/databases/commands/test_connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ def __init__(self, data: Dict[str, Any]):
self._properties = data.copy()
self._model: Optional[Database] = None

def run(self) -> None:
def run(self) -> None: # pylint: disable=too-many-statements
self.validate()
uri = self._properties.get("sqlalchemy_uri", "")
if self._model and uri == self._model.safe_sqlalchemy_uri():
Expand All @@ -63,12 +63,24 @@ def run(self) -> None:
"database": url.database,
}

serialized_encrypted_extra = self._properties.get(
"masked_encrypted_extra",
"{}",
)
if self._model:
serialized_encrypted_extra = (
self._model.db_engine_spec.unmask_encrypted_extra(
self._model.encrypted_extra,
serialized_encrypted_extra,
)
)

try:
database = DatabaseDAO.build_db_for_connection_test(
server_cert=self._properties.get("server_cert", ""),
extra=self._properties.get("extra", "{}"),
impersonate_user=self._properties.get("impersonate_user", False),
encrypted_extra=self._properties.get("encrypted_extra", "{}"),
encrypted_extra=serialized_encrypted_extra,
)

database.set_sqlalchemy_uri(uri)
Expand Down
8 changes: 8 additions & 0 deletions superset/databases/commands/update.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,14 @@ def run(self) -> Model:
raise DatabaseNotFoundError()
old_database_name = self._model.database_name

# unmask ``encrypted_extra``
self._properties[
"encrypted_extra"
] = self._model.db_engine_spec.unmask_encrypted_extra(
self._model.encrypted_extra,
self._properties.pop("masked_encrypted_extra", "{}"),
)

try:
database = DatabaseDAO.update(self._model, self._properties, commit=False)
database.set_sqlalchemy_uri(database.sqlalchemy_uri)
Expand Down
18 changes: 14 additions & 4 deletions superset/databases/commands/validate.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ def __init__(self, parameters: Dict[str, Any]):
self._model: Optional[Database] = None

def run(self) -> None:
self.validate()

engine = self._properties["engine"]
engine_specs = get_engine_specs()

Expand Down Expand Up @@ -92,7 +94,15 @@ def run(self) -> None:
event_logger.log_with_context(action="validation_error", engine=engine)
raise InvalidParametersError(errors)

serialized_encrypted_extra = self._properties.get("encrypted_extra", "{}")
serialized_encrypted_extra = self._properties.get(
"masked_encrypted_extra",
"{}",
)
if self._model:
serialized_encrypted_extra = engine_spec.unmask_encrypted_extra(
self._model.encrypted_extra,
serialized_encrypted_extra,
)
try:
encrypted_extra = json.loads(serialized_encrypted_extra)
except json.decoder.JSONDecodeError:
Expand Down Expand Up @@ -140,6 +150,6 @@ def run(self) -> None:
)

def validate(self) -> None:
database_name = self._properties.get("database_name")
if database_name is not None:
self._model = DatabaseDAO.get_database_by_name(database_name)
database_id = self._properties.get("id")
if database_id is not None:
self._model = DatabaseDAO.find_by_id(database_id)
24 changes: 24 additions & 0 deletions superset/databases/dao.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,30 @@ class DatabaseDAO(BaseDAO):
model_cls = Database
base_filter = DatabaseFilter

@classmethod
def update(
cls,
model: Database,
properties: Dict[str, Any],
commit: bool = True,
) -> Database:
"""
Unmask ``encrypted_extra`` before updating.
When a database is edited the user sees a masked version of ``encrypted_extra``,
depending on the engine spec. Eg, BigQuery will mask the ``private_key`` attribute
of the credentials.
The masked values should be unmasked before the database is updated.
"""
if "encrypted_extra" in properties:
properties["encrypted_extra"] = model.db_engine_spec.unmask_encrypted_extra(
model.encrypted_extra,
properties["encrypted_extra"],
)

return super().update(model, properties, commit)

@staticmethod
def validate_uniqueness(database_name: str) -> bool:
database_query = db.session.query(Database).filter(
Expand Down
17 changes: 10 additions & 7 deletions superset/databases/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,12 @@
from sqlalchemy import MetaData

from superset import db
from superset.constants import PASSWORD_MASK
from superset.databases.commands.exceptions import DatabaseInvalidError
from superset.databases.utils import make_url_safe
from superset.db_engine_specs import BaseEngineSpec, get_engine_specs
from superset.exceptions import CertificateException, SupersetSecurityException
from superset.models.core import ConfigurationMethod, Database, PASSWORD_MASK
from superset.models.core import ConfigurationMethod, Database
from superset.security.analytics_db_safety import check_sqlalchemy_uri
from superset.utils.core import markdown, parse_ssl_cert

Expand Down Expand Up @@ -282,14 +283,15 @@ def build_sqlalchemy_uri(
# validate parameters
parameters = engine_spec.parameters_schema.load(parameters) # type: ignore

serialized_encrypted_extra = data.get("encrypted_extra") or "{}"
serialized_encrypted_extra = data.get("masked_encrypted_extra") or "{}"
try:
encrypted_extra = json.loads(serialized_encrypted_extra)
except json.decoder.JSONDecodeError:
encrypted_extra = {}

data["sqlalchemy_uri"] = engine_spec.build_sqlalchemy_uri( # type: ignore
parameters, encrypted_extra
parameters,
encrypted_extra,
)

return data
Expand Down Expand Up @@ -322,6 +324,7 @@ class DatabaseValidateParametersSchema(Schema):
class Meta: # pylint: disable=too-few-public-methods
unknown = EXCLUDE

id = fields.Integer(allow_none=True, description="Database ID (for updates)")
engine = fields.String(required=True, description="SQLAlchemy engine to use")
parameters = fields.Dict(
keys=fields.String(),
Expand All @@ -335,7 +338,7 @@ class Meta: # pylint: disable=too-few-public-methods
)
impersonate_user = fields.Boolean(description=impersonate_user_description)
extra = fields.String(description=extra_description, validate=extra_validator)
encrypted_extra = fields.String(
masked_encrypted_extra = fields.String(
description=encrypted_extra_description,
validate=encrypted_extra_validator,
allow_none=True,
Expand Down Expand Up @@ -380,7 +383,7 @@ class Meta: # pylint: disable=too-few-public-methods
description=allow_multi_schema_metadata_fetch_description,
)
impersonate_user = fields.Boolean(description=impersonate_user_description)
encrypted_extra = fields.String(
masked_encrypted_extra = fields.String(
description=encrypted_extra_description,
validate=encrypted_extra_validator,
allow_none=True,
Expand Down Expand Up @@ -427,7 +430,7 @@ class Meta: # pylint: disable=too-few-public-methods
description=allow_multi_schema_metadata_fetch_description
)
impersonate_user = fields.Boolean(description=impersonate_user_description)
encrypted_extra = fields.String(
masked_encrypted_extra = fields.String(
description=encrypted_extra_description,
allow_none=True,
validate=encrypted_extra_validator,
Expand All @@ -454,7 +457,7 @@ class DatabaseTestConnectionSchema(Schema, DatabaseParametersSchemaMixin):
)
impersonate_user = fields.Boolean(description=impersonate_user_description)
extra = fields.String(description=extra_description, validate=extra_validator)
encrypted_extra = fields.String(
masked_encrypted_extra = fields.String(
description=encrypted_extra_description,
validate=encrypted_extra_validator,
allow_none=True,
Expand Down
Loading

0 comments on commit 358550f

Please sign in to comment.