finish array work

apache · Jan 30, 2024 · 6fb80f1 · 6fb80f1
1 parent 49d1a01
commit 6fb80f1
Show file tree

Hide file tree

Showing 12 changed files with 116 additions and 16 deletions.
diff --git a/web-console/src/components/warning-checklist/warning-checklist.tsx b/web-console/src/components/warning-checklist/warning-checklist.tsx
@@ -40,7 +40,7 @@ export const WarningChecklist = React.memo(function WarningChecklist(props: Warn
   return (
     <div className="warning-checklist">
       {checks.map((check, i) => (
-        <Switch key={i} onChange={() => doCheck(i)}>
+        <Switch key={i} className="danger-switch" onChange={() => doCheck(i)}>
           {check}
         </Switch>
       ))}

diff --git a/web-console/src/druid-models/external-config/external-config.ts b/web-console/src/druid-models/external-config/external-config.ts
@@ -19,6 +19,7 @@
 import type { SqlQuery } from '@druid-toolkit/query';
 import {
   C,
+  F,
   filterMap,
   L,
   SqlColumnDeclaration,
@@ -127,13 +128,17 @@ export function externalConfigToTableExpression(config: ExternalConfig): SqlExpr
 export function externalConfigToInitDimensions(
   config: ExternalConfig,
   timeExpression: SqlExpression | undefined,
+  forceMultiValue: boolean,
 ): SqlExpression[] {
   return (timeExpression ? [timeExpression.as('__time')] : [])
     .concat(
       filterMap(config.signature, columnDeclaration => {
         const columnName = columnDeclaration.getColumnName();
         if (timeExpression && timeExpression.containsColumnName(columnName)) return;
-        return C(columnName);
+        return C(columnName).applyIf(
+          forceMultiValue && columnDeclaration.columnType.isArray(),
+          ex => F('ARRAY_TO_MV', ex).as(columnName),
+        );
       }),
     )
     .slice(0, MULTI_STAGE_QUERY_MAX_COLUMNS);

diff --git a/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts b/web-console/src/druid-models/ingest-query-pattern/ingest-query-pattern.ts
@@ -62,14 +62,15 @@ export function externalConfigToIngestQueryPattern(
   config: ExternalConfig,
   timeExpression: SqlExpression | undefined,
   partitionedByHint: string | undefined,
+  forceMultiValue: boolean,
 ): IngestQueryPattern {
   return {
     destinationTableName: guessDataSourceNameFromInputSource(config.inputSource) || 'data',
     mode: 'replace',
     mainExternalName: 'ext',
     mainExternalConfig: config,
     filters: [],
-    dimensions: externalConfigToInitDimensions(config, timeExpression),
+    dimensions: externalConfigToInitDimensions(config, timeExpression, forceMultiValue),
     partitionedBy: partitionedByHint || (timeExpression ? 'day' : 'all'),
     clusteredBy: [],
   };

diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -302,6 +302,7 @@ export function getArrayMode(spec: Partial<IngestionSpec>): ArrayMode {
         spec,
         'spec.dataSchema.dimensionsSpec.dimensions',
       );
+
       if (
         dimensions.some(
           d =>
@@ -327,6 +328,23 @@ export function getArrayMode(spec: Partial<IngestionSpec>): ArrayMode {
   }
 }
 
+export function showArrayModeToggle(spec: Partial<IngestionSpec>): boolean {
+  const schemaMode = getSchemaMode(spec);
+  if (schemaMode !== 'fixed') return false;
+
+  const dimensions: (DimensionSpec | string)[] = deepGet(
+    spec,
+    'spec.dataSchema.dimensionsSpec.dimensions',
+  );
+
+  return dimensions.some(
+    d =>
+      typeof d === 'object' &&
+      ((d.type === 'auto' && String(d.castToType).startsWith('ARRAY')) ||
+        (d.type === 'string' && typeof d.multiValueHandling === 'string')),
+  );
+}
+
 export function getRollup(spec: Partial<IngestionSpec>, valueIfUnset = true): boolean {
   const specRollup = deepGet(spec, 'spec.dataSchema.granularitySpec.rollup');
   return typeof specRollup === 'boolean' ? specRollup : valueIfUnset;

diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts
@@ -91,10 +91,16 @@ export class WorkbenchQuery {
     externalConfig: ExternalConfig,
     timeExpression: SqlExpression | undefined,
     partitionedByHint: string | undefined,
+    forceMultiValue: boolean,
   ): WorkbenchQuery {
     return new WorkbenchQuery({
       queryString: ingestQueryPatternToQuery(
-        externalConfigToIngestQueryPattern(externalConfig, timeExpression, partitionedByHint),
+        externalConfigToIngestQueryPattern(
+          externalConfig,
+          timeExpression,
+          partitionedByHint,
+          forceMultiValue,
+        ),
       ).toString(),
       queryContext: {
         arrayIngestMode: 'array',

diff --git a/web-console/src/entry.scss b/web-console/src/entry.scss
@@ -62,3 +62,23 @@ body {
     width: 100%;
   }
 }
+
+.legacy-switch.#{$bp-ns}-control.#{$bp-ns}-switch {
+  input:checked ~ .#{$bp-ns}-control-indicator {
+    background: $orange5;
+  }
+
+  &:hover input:checked ~ .#{$bp-ns}-control-indicator {
+    background: $orange2;
+  }
+}
+
+.danger-switch.#{$bp-ns}-control.#{$bp-ns}-switch {
+  input:checked ~ .#{$bp-ns}-control-indicator {
+    background: $red5;
+  }
+
+  &:hover input:checked ~ .#{$bp-ns}-control-indicator {
+    background: $red2;
+  }
+}
diff --git a/web-console/src/views/load-data-view/load-data-view.scss b/web-console/src/views/load-data-view/load-data-view.scss
@@ -320,8 +320,4 @@ $actual-icon-height: 400px;
   .parse-metadata {
     border-top: 1px solid $gray1;
   }
-
-  .legacy-switch.bp4-control.bp4-switch input:checked ~ .bp4-control-indicator {
-    background: $orange5;
-  }
 }
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -120,6 +120,7 @@ import {
   possibleDruidFormatForValues,
   PRIMARY_PARTITION_RELATED_FORM_FIELDS,
   removeTimestampTransform,
+  showArrayModeToggle,
   splitFilter,
   STREAMING_INPUT_FORMAT_FIELDS,
   TIME_COLUMN,
@@ -2375,7 +2376,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                   label="Explicitly specify schema"
                 />
               </FormGroupWithInfo>
-              {schemaMode === 'fixed' ? (
+              {showArrayModeToggle(spec) && (
                 <FormGroupWithInfo
                   inlineInfo
                   info={
@@ -2401,14 +2402,16 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
                   <Switch
                     label="Store ARRAYs as MVDs"
                     className="legacy-switch"
+                    checked={arrayMode === 'multi-values'}
                     onChange={() =>
                       this.setState({
                         newArrayMode: arrayMode === 'arrays' ? 'multi-values' : 'arrays',
                       })
                     }
                   />
                 </FormGroupWithInfo>
-              ) : (
+              )}
+              {schemaMode !== 'fixed' && (
                 <AutoForm
                   fields={[
                     {

diff --git a/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx b/web-console/src/views/sql-data-loader-view/sql-data-loader-view.tsx
@@ -189,26 +189,28 @@ export const SqlDataLoaderView = React.memo(function SqlDataLoaderView(
             inputSource={inputSource}
             initInputFormat={inputFormat}
             doneButton={false}
-            onSet={({ inputFormat, signature, timeExpression }) => {
+            onSet={({ inputFormat, signature, timeExpression, forceMultiValue }) => {
               setContent({
                 queryString: ingestQueryPatternToQuery(
                   externalConfigToIngestQueryPattern(
                     { inputSource, inputFormat, signature },
                     timeExpression,
                     undefined,
+                    forceMultiValue,
                   ),
                 ).toString(),
                 queryContext: INITIAL_QUERY_CONTEXT,
               });
             }}
             altText="Skip the wizard and continue with custom SQL"
-            onAltSet={({ inputFormat, signature, timeExpression }) => {
+            onAltSet={({ inputFormat, signature, timeExpression, forceMultiValue }) => {
               goToQuery({
                 queryString: ingestQueryPatternToQuery(
                   externalConfigToIngestQueryPattern(
                     { inputSource, inputFormat, signature },
                     timeExpression,
                     undefined,
+                    forceMultiValue,
                   ),
                 ).toString(),
               });

diff --git a/...le/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx b/...le/src/views/workbench-view/connect-external-data-dialog/connect-external-data-dialog.tsx
@@ -32,6 +32,7 @@ export interface ConnectExternalDataDialogProps {
     config: ExternalConfig,
     timeExpression: SqlExpression | undefined,
     partitionedByHint: string | undefined,
+    forceMultiValue: boolean,
   ): void;
   onClose(): void;
 }
@@ -66,11 +67,12 @@ export const ConnectExternalDataDialog = React.memo(function ConnectExternalData
             inputSource={inputSource}
             initInputFormat={inputFormat}
             doneButton
-            onSet={({ inputFormat, signature, timeExpression }) => {
+            onSet={({ inputFormat, signature, timeExpression, forceMultiValue }) => {
               onSetExternalConfig(
                 { inputSource, inputFormat, signature },
                 timeExpression,
                 partitionedByHint,
+                forceMultiValue,
               );
               onClose();
             }}

diff --git a/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx b/web-console/src/views/workbench-view/input-format-step/input-format-step.tsx
@@ -16,13 +16,21 @@
  * limitations under the License.
  */
 
-import { Button, Callout, FormGroup, Icon, Intent, Tag } from '@blueprintjs/core';
+import { Button, Callout, FormGroup, Icon, Intent, Switch, Tag } from '@blueprintjs/core';
 import { IconNames } from '@blueprintjs/icons';
 import type { SqlExpression } from '@druid-toolkit/query';
 import { C, SqlColumnDeclaration, SqlType } from '@druid-toolkit/query';
 import React, { useState } from 'react';
 
-import { AutoForm, CenterMessage, LearnMore, Loader } from '../../../components';
+import {
+  AutoForm,
+  CenterMessage,
+  ExternalLink,
+  FormGroupWithInfo,
+  LearnMore,
+  Loader,
+  PopoverText,
+} from '../../../components';
 import type { InputFormat, InputSource } from '../../../druid-models';
 import {
   BATCH_INPUT_FORMAT_FIELDS,
@@ -52,6 +60,7 @@ export interface InputFormatAndMore {
   inputFormat: InputFormat;
   signature: SqlColumnDeclaration[];
   timeExpression: SqlExpression | undefined;
+  forceMultiValue: boolean;
 }
 
 interface PossibleTimeExpression {
@@ -78,6 +87,7 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
     AutoForm.isValidModel(initInputFormat, BATCH_INPUT_FORMAT_FIELDS) ? initInputFormat : undefined,
   );
   const [selectTimestamp, setSelectTimestamp] = useState(true);
+  const [forceMultiValue, setForceMultiValue] = useState(false);
 
   const [previewState] = useQueryManager<InputFormat, SampleResponse>({
     query: inputFormatToSample,
@@ -165,9 +175,12 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
             ),
           ),
           timeExpression: selectTimestamp ? possibleTimeExpression?.timeExpression : undefined,
+          forceMultiValue,
         }
       : undefined;
 
+  const hasArrays = inputFormatAndMore?.signature.some(d => d.columnType.isArray());
+
   return (
     <div className="input-format-step">
       <div className="preview">
@@ -219,6 +232,34 @@ export const InputFormatStep = React.memo(function InputFormatStep(props: InputF
           )}
         </div>
         <div className="bottom-controls">
+          {hasArrays && (
+            <FormGroupWithInfo
+              inlineInfo
+              info={
+                <PopoverText>
+                  <p>
+                    Store arrays as multi-value string columns instead of arrays. Note that all
+                    detected array elements will be coerced to strings if you choose this option,
+                    and data will behave more like a string than an array at query time. See{' '}
+                    <ExternalLink href={`${getLink('DOCS')}/querying/arrays`}>
+                      array docs
+                    </ExternalLink>{' '}
+                    and{' '}
+                    <ExternalLink href={`${getLink('DOCS')}/querying/multi-value-dimensions`}>
+                      mvd docs
+                    </ExternalLink>{' '}
+                    for more details about the differences between arrays and multi-value strings.
+                  </p>
+                </PopoverText>
+              }
+            >
+              <Switch
+                label="Store ARRAYs as MVDs"
+                className="legacy-switch"
+                onChange={() => setForceMultiValue(!forceMultiValue)}
+              />
+            </FormGroupWithInfo>
+          )}
           {possibleTimeExpression && (
             <FormGroup>
               <Callout>

diff --git a/web-console/src/views/workbench-view/workbench-view.tsx b/web-console/src/views/workbench-view/workbench-view.tsx
@@ -320,12 +320,18 @@ export class WorkbenchView extends React.PureComponent<WorkbenchViewProps, Workb
 
     return (
       <ConnectExternalDataDialog
-        onSetExternalConfig={(externalConfig, timeExpression, partitionedByHint) => {
+        onSetExternalConfig={(
+          externalConfig,
+          timeExpression,
+          partitionedByHint,
+          forceMultiValue,
+        ) => {
           this.handleNewTab(
             WorkbenchQuery.fromInitExternalConfig(
               externalConfig,
               timeExpression,
               partitionedByHint,
+              forceMultiValue,
             ),
             'Ext ' + guessDataSourceNameFromInputSource(externalConfig.inputSource),
           );