From 6fb80f143a4cab273df0b278b0e70da04e472584 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Tue, 30 Jan 2024 15:10:15 -0800 Subject: [PATCH] finish array work --- .../warning-checklist/warning-checklist.tsx | 2 +- .../external-config/external-config.ts | 7 ++- .../ingest-query-pattern.ts | 3 +- .../ingestion-spec/ingestion-spec.tsx | 18 ++++++++ .../workbench-query/workbench-query.ts | 8 +++- web-console/src/entry.scss | 20 +++++++++ .../views/load-data-view/load-data-view.scss | 4 -- .../views/load-data-view/load-data-view.tsx | 7 ++- .../sql-data-loader-view.tsx | 6 ++- .../connect-external-data-dialog.tsx | 4 +- .../input-format-step/input-format-step.tsx | 45 ++++++++++++++++++- .../views/workbench-view/workbench-view.tsx | 8 +++- 12 files changed, 116 insertions(+), 16 deletions(-) diff --git a/web-console/src/components/warning-checklist/warning-checklist.tsx b/web-console/src/components/warning-checklist/warning-checklist.tsx index 351203fe7272..fa496b26a532 100644 --- a/web-console/src/components/warning-checklist/warning-checklist.tsx +++ b/web-console/src/components/warning-checklist/warning-checklist.tsx @@ -40,7 +40,7 @@ export const WarningChecklist = React.memo(function WarningChecklist(props: Warn return (
{checks.map((check, i) => ( - doCheck(i)}> + doCheck(i)}> {check} ))} diff --git a/web-console/src/druid-models/external-config/external-config.ts b/web-console/src/druid-models/external-config/external-config.ts index 2012891f1441..253bc4e12099 100644 --- a/web-console/src/druid-models/external-config/external-config.ts +++ b/web-console/src/druid-models/external-config/external-config.ts @@ -19,6 +19,7 @@ import type { SqlQuery } from '@druid-toolkit/query'; import { C, + F, filterMap, L, SqlColumnDeclaration, @@ -127,13 +128,17 @@ export function externalConfigToTableExpression(config: ExternalConfig): SqlExpr export function externalConfigToInitDimensions( config: ExternalConfig, timeExpression: SqlExpression | undefined, + forceMultiValue: boolean, ): SqlExpression[] { return (timeExpression ? [timeExpression.as('__time')] : []) .concat( filterMap(config.signature, columnDeclaration => { const columnName = columnDeclaration.getColumnName(); if (timeExpression && timeExpression.containsColumnName(columnName)) return; - return C(columnName); + return C(columnName).applyIf( + forceMultiValue && columnDeclaration.columnType.isArray(), + ex => F('ARRAY_TO_MV', ex).as(columnName), + ); }), ) .slice(0, MULTI_STAGE_QUERY_MAX_COLUMNS); diff --git a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts index 6c199f65bc6f..f77d8d0f09a7 100644 --- a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts +++ b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts @@ -62,6 +62,7 @@ export function externalConfigToIngestQueryPattern( config: ExternalConfig, timeExpression: SqlExpression | undefined, partitionedByHint: string | undefined, + forceMultiValue: boolean, ): IngestQueryPattern { return { destinationTableName: guessDataSourceNameFromInputSource(config.inputSource) || 'data', @@ -69,7 +70,7 @@ export function externalConfigToIngestQueryPattern( mainExternalName: 'ext', mainExternalConfig: config, filters: [], - dimensions: externalConfigToInitDimensions(config, timeExpression), + dimensions: externalConfigToInitDimensions(config, timeExpression, forceMultiValue), partitionedBy: partitionedByHint || (timeExpression ? 'day' : 'all'), clusteredBy: [], }; diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx index b599eb19f6a3..174a91308597 100644 --- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx +++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx @@ -302,6 +302,7 @@ export function getArrayMode(spec: Partial): ArrayMode { spec, 'spec.dataSchema.dimensionsSpec.dimensions', ); + if ( dimensions.some( d => @@ -327,6 +328,23 @@ export function getArrayMode(spec: Partial): ArrayMode { } } +export function showArrayModeToggle(spec: Partial): boolean { + const schemaMode = getSchemaMode(spec); + if (schemaMode !== 'fixed') return false; + + const dimensions: (DimensionSpec | string)[] = deepGet( + spec, + 'spec.dataSchema.dimensionsSpec.dimensions', + ); + + return dimensions.some( + d => + typeof d === 'object' && + ((d.type === 'auto' && String(d.castToType).startsWith('ARRAY')) || + (d.type === 'string' && typeof d.multiValueHandling === 'string')), + ); +} + export function getRollup(spec: Partial, valueIfUnset = true): boolean { const specRollup = deepGet(spec, 'spec.dataSchema.granularitySpec.rollup'); return typeof specRollup === 'boolean' ? specRollup : valueIfUnset; diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index 81b35bac580a..cb2ce8e0d370 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -91,10 +91,16 @@ export class WorkbenchQuery { externalConfig: ExternalConfig, timeExpression: SqlExpression | undefined, partitionedByHint: string | undefined, + forceMultiValue: boolean, ): WorkbenchQuery { return new WorkbenchQuery({ queryString: ingestQueryPatternToQuery( - externalConfigToIngestQueryPattern(externalConfig, timeExpression, partitionedByHint), + externalConfigToIngestQueryPattern( + externalConfig, + timeExpression, + partitionedByHint, + forceMultiValue, + ), ).toString(), queryContext: { arrayIngestMode: 'array', diff --git a/web-console/src/entry.scss b/web-console/src/entry.scss index a32e7de82c1d..5f231bc5d152 100644 --- a/web-console/src/entry.scss +++ b/web-console/src/entry.scss @@ -62,3 +62,23 @@ body { width: 100%; } } + +.legacy-switch.#{$bp-ns}-control.#{$bp-ns}-switch { + input:checked ~ .#{$bp-ns}-control-indicator { + background: $orange5; + } + + &:hover input:checked ~ .#{$bp-ns}-control-indicator { + background: $orange2; + } +} + +.danger-switch.#{$bp-ns}-control.#{$bp-ns}-switch { + input:checked ~ .#{$bp-ns}-control-indicator { + background: $red5; + } + + &:hover input:checked ~ .#{$bp-ns}-control-indicator { + background: $red2; + } +} diff --git a/web-console/src/views/load-data-view/load-data-view.scss b/web-console/src/views/load-data-view/load-data-view.scss index b30b2b3c801d..8532313b10bd 100644 --- a/web-console/src/views/load-data-view/load-data-view.scss +++ b/web-console/src/views/load-data-view/load-data-view.scss @@ -320,8 +320,4 @@ $actual-icon-height: 400px; .parse-metadata { border-top: 1px solid $gray1; } - - .legacy-switch.bp4-control.bp4-switch input:checked ~ .bp4-control-indicator { - background: $orange5; - } } diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx index 86cf102abdc0..cc82c3536643 100644 --- a/web-console/src/views/load-data-view/load-data-view.tsx +++ b/web-console/src/views/load-data-view/load-data-view.tsx @@ -120,6 +120,7 @@ import { possibleDruidFormatForValues, PRIMARY_PARTITION_RELATED_FORM_FIELDS, removeTimestampTransform, + showArrayModeToggle, splitFilter, STREAMING_INPUT_FORMAT_FIELDS, TIME_COLUMN, @@ -2375,7 +2376,7 @@ export class LoadDataView extends React.PureComponent - {schemaMode === 'fixed' ? ( + {showArrayModeToggle(spec) && ( this.setState({ newArrayMode: arrayMode === 'arrays' ? 'multi-values' : 'arrays', @@ -2408,7 +2410,8 @@ export class LoadDataView extends React.PureComponent - ) : ( + )} + {schemaMode !== 'fixed' && ( { + onSet={({ inputFormat, signature, timeExpression, forceMultiValue }) => { setContent({ queryString: ingestQueryPatternToQuery( externalConfigToIngestQueryPattern( { inputSource, inputFormat, signature }, timeExpression, undefined, + forceMultiValue, ), ).toString(), queryContext: INITIAL_QUERY_CONTEXT, }); }} altText="Skip the wizard and continue with custom SQL" - onAltSet={({ inputFormat, signature, timeExpression }) => { + onAltSet={({ inputFormat, signature, timeExpression, forceMultiValue }) => { goToQuery({ queryString: ingestQueryPatternToQuery( externalConfigToIngestQueryPattern( { inputSource, inputFormat, signature }, timeExpression, undefined, + forceMultiValue, ), ).toString(), }); diff --git a/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx b/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx index 9bb75e752e63..78c3a30efdd0 100644 --- a/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx +++ b/web-console/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx @@ -32,6 +32,7 @@ export interface ConnectExternalDataDialogProps { config: ExternalConfig, timeExpression: SqlExpression | undefined, partitionedByHint: string | undefined, + forceMultiValue: boolean, ): void; onClose(): void; } @@ -66,11 +67,12 @@ export const ConnectExternalDataDialog = React.memo(function ConnectExternalData inputSource={inputSource} initInputFormat={inputFormat} doneButton - onSet={({ inputFormat, signature, timeExpression }) => { + onSet={({ inputFormat, signature, timeExpression, forceMultiValue }) => { onSetExternalConfig( { inputSource, inputFormat, signature }, timeExpression, partitionedByHint, + forceMultiValue, ); onClose(); }} diff --git a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx index f319b83c6e29..0940b9a8e185 100644 --- a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx +++ b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx @@ -16,13 +16,21 @@ * limitations under the License. */ -import { Button, Callout, FormGroup, Icon, Intent, Tag } from '@blueprintjs/core'; +import { Button, Callout, FormGroup, Icon, Intent, Switch, Tag } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import type { SqlExpression } from '@druid-toolkit/query'; import { C, SqlColumnDeclaration, SqlType } from '@druid-toolkit/query'; import React, { useState } from 'react'; -import { AutoForm, CenterMessage, LearnMore, Loader } from '../../../components'; +import { + AutoForm, + CenterMessage, + ExternalLink, + FormGroupWithInfo, + LearnMore, + Loader, + PopoverText, +} from '../../../components'; import type { InputFormat, InputSource } from '../../../druid-models'; import { BATCH_INPUT_FORMAT_FIELDS, @@ -52,6 +60,7 @@ export interface InputFormatAndMore { inputFormat: InputFormat; signature: SqlColumnDeclaration[]; timeExpression: SqlExpression | undefined; + forceMultiValue: boolean; } interface PossibleTimeExpression { @@ -78,6 +87,7 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF AutoForm.isValidModel(initInputFormat, BATCH_INPUT_FORMAT_FIELDS) ? initInputFormat : undefined, ); const [selectTimestamp, setSelectTimestamp] = useState(true); + const [forceMultiValue, setForceMultiValue] = useState(false); const [previewState] = useQueryManager({ query: inputFormatToSample, @@ -165,9 +175,12 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF ), ), timeExpression: selectTimestamp ? possibleTimeExpression?.timeExpression : undefined, + forceMultiValue, } : undefined; + const hasArrays = inputFormatAndMore?.signature.some(d => d.columnType.isArray()); + return (
@@ -219,6 +232,34 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF )}
+ {hasArrays && ( + +

+ Store arrays as multi-value string columns instead of arrays. Note that all + detected array elements will be coerced to strings if you choose this option, + and data will behave more like a string than an array at query time. See{' '} + + array docs + {' '} + and{' '} + + mvd docs + {' '} + for more details about the differences between arrays and multi-value strings. +

+ + } + > + setForceMultiValue(!forceMultiValue)} + /> + + )} {possibleTimeExpression && ( diff --git a/web-console/src/views/workbench-view/workbench-view.tsx b/web-console/src/views/workbench-view/workbench-view.tsx index 00ab17d34012..7d721da413f2 100644 --- a/web-console/src/views/workbench-view/workbench-view.tsx +++ b/web-console/src/views/workbench-view/workbench-view.tsx @@ -320,12 +320,18 @@ export class WorkbenchView extends React.PureComponent { + onSetExternalConfig={( + externalConfig, + timeExpression, + partitionedByHint, + forceMultiValue, + ) => { this.handleNewTab( WorkbenchQuery.fromInitExternalConfig( externalConfig, timeExpression, partitionedByHint, + forceMultiValue, ), 'Ext ' + guessDataSourceNameFromInputSource(externalConfig.inputSource), );