From 575c9c85d5e53245b8bb317ec9b17ca5cc0e7b52 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 15 May 2024 11:35:00 -0700 Subject: [PATCH 1/9] copy as values --- web-console/src/utils/download.ts | 56 ++++++++++--------- web-console/src/utils/index.tsx | 2 +- web-console/src/utils/types.ts | 2 + ...e-query.spec.tsx => values-query.spec.tsx} | 32 ++++++----- .../{sample-query.tsx => values-query.tsx} | 49 ++++++++++++---- .../schema-step/schema-step.tsx | 4 +- .../execution-summary-panel.tsx | 27 +++++---- 7 files changed, 106 insertions(+), 66 deletions(-) rename web-console/src/utils/{sample-query.spec.tsx => values-query.spec.tsx} (64%) rename web-console/src/utils/{sample-query.tsx => values-query.tsx} (57%) diff --git a/web-console/src/utils/download.ts b/web-console/src/utils/download.ts index aebf5f14f637..aabe2fc1c66b 100644 --- a/web-console/src/utils/download.ts +++ b/web-console/src/utils/download.ts @@ -21,6 +21,9 @@ import FileSaver from 'file-saver'; import * as JSONBig from 'json-bigint-native'; import { copyAndAlert, stringifyValue } from './general'; +import { queryResultToValuesQuery } from './values-query'; + +export type Format = 'csv' | 'tsv' | 'json' | 'sql'; export function downloadUrl(url: string, filename: string) { // Create a link and set the URL using `createObjectURL` @@ -74,44 +77,43 @@ export function downloadFile(text: string, type: string, filename: string): void FileSaver.saveAs(blob, filename); } -function queryResultsToString(queryResult: QueryResult, format: string): string { - let lines: string[] = []; - let separator = ''; - - if (format === 'csv' || format === 'tsv') { - separator = format === 'csv' ? ',' : '\t'; - lines.push( - queryResult.header.map(column => formatForFormat(column.name, format)).join(separator), - ); - lines = lines.concat( - queryResult.rows.map(r => r.map(cell => formatForFormat(cell, format)).join(separator)), - ); - } else { - // json - lines = queryResult.rows.map(r => { - const outputObject: Record = {}; - for (let k = 0; k < r.length; k++) { - const newName = queryResult.header[k]; - if (newName) { - outputObject[newName.name] = r[k]; - } - } - return JSONBig.stringify(outputObject); - }); +function queryResultsToString(queryResult: QueryResult, format: Format): string { + const { header, rows } = queryResult; + + switch (format) { + case 'csv': + case 'tsv': { + const separator = format === 'csv' ? ',' : '\t'; + return [ + header.map(column => formatForFormat(column.name, format)).join(separator), + ...rows.map(r => r.map(cell => formatForFormat(cell, format)).join(separator)), + ].join('\n'); + } + + case 'sql': + return queryResultToValuesQuery(queryResult).toString(); + + case 'json': + return queryResult + .toObjectArray() + .map(r => JSONBig.stringify(r)) + .join('\n'); + + default: + throw new Error(`unknown format: ${format}`); } - return lines.join('\n'); } export function downloadQueryResults( queryResult: QueryResult, filename: string, - format: string, + format: Format, ): void { const resultString: string = queryResultsToString(queryResult, format); downloadFile(resultString, format, filename); } -export function copyQueryResultsToClipboard(queryResult: QueryResult, format: string): void { +export function copyQueryResultsToClipboard(queryResult: QueryResult, format: Format): void { const resultString: string = queryResultsToString(queryResult, format); copyAndAlert(resultString, 'Query results copied to clipboard'); } diff --git a/web-console/src/utils/index.tsx b/web-console/src/utils/index.tsx index 244eec5372cc..4daeefe61c13 100644 --- a/web-console/src/utils/index.tsx +++ b/web-console/src/utils/index.tsx @@ -32,8 +32,8 @@ export * from './object-change'; export * from './query-action'; export * from './query-manager'; export * from './query-state'; -export * from './sample-query'; export * from './sanitizers'; export * from './sql'; export * from './table-helpers'; export * from './types'; +export * from './values-query'; diff --git a/web-console/src/utils/types.ts b/web-console/src/utils/types.ts index 76437e58327b..937e45b191b7 100644 --- a/web-console/src/utils/types.ts +++ b/web-console/src/utils/types.ts @@ -75,7 +75,9 @@ export function dataTypeToIcon(dataType: string): IconName { return IconNames.DIAGRAM_TREE; case 'COMPLEX': + case 'COMPLEX': case 'COMPLEX': + case 'COMPLEX': case 'COMPLEX': return IconNames.SNOWFLAKE; diff --git a/web-console/src/utils/sample-query.spec.tsx b/web-console/src/utils/values-query.spec.tsx similarity index 64% rename from web-console/src/utils/sample-query.spec.tsx rename to web-console/src/utils/values-query.spec.tsx index 2864f0d28feb..85839aa33584 100644 --- a/web-console/src/utils/sample-query.spec.tsx +++ b/web-console/src/utils/values-query.spec.tsx @@ -18,26 +18,30 @@ import { QueryResult, sane } from '@druid-toolkit/query'; -import { sampleDataToQuery } from './sample-query'; +import { queryResultToValuesQuery } from './values-query'; -describe('sample-query', () => { +describe('queryResultToValuesQuery', () => { it('works', () => { const result = QueryResult.fromRawResult( [ - ['__time', 'host', 'service', 'msg'], - ['LONG', 'STRING', 'STRING', 'COMPLEX'], - ['TIMESTAMP', 'VARCHAR', 'VARCHAR', 'OTHER'], + ['__time', 'host', 'service', 'msg', 'language', 'nums'], + ['LONG', 'STRING', 'STRING', 'COMPLEX', 'ARRAY', 'ARRAY'], + ['TIMESTAMP', 'VARCHAR', 'VARCHAR', 'OTHER', 'ARRAY', 'ARRAY'], [ '2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '{"type":"sys","swap/free":1223334,"swap/max":3223334}', + ['es', 'es-419'], + [1], ], [ '2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '{"type":"query","time":1223,"bytes":2434234}', + ['en', 'es', 'es-419'], + [2, 3], ], ], false, @@ -46,17 +50,19 @@ describe('sample-query', () => { true, ); - expect(sampleDataToQuery(result).toString()).toEqual(sane` + expect(queryResultToValuesQuery(result).toString()).toEqual(sane` SELECT - CAST("c0" AS TIMESTAMP) AS "__time", - CAST("c1" AS VARCHAR) AS "host", - CAST("c2" AS VARCHAR) AS "service", - PARSE_JSON("c3") AS "msg" + CAST("c1" AS TIMESTAMP) AS "__time", + CAST("c2" AS VARCHAR) AS "host", + CAST("c3" AS VARCHAR) AS "service", + PARSE_JSON("c4") AS "msg", + STRING_TO_ARRAY("c5", '<#>') AS "language", + CAST(STRING_TO_ARRAY("c6", '<#>') AS BIGINT ARRAY) AS "nums" FROM ( VALUES - ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"sys\\",\\"swap/free\\":1223334,\\"swap/max\\":3223334}"'), - ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"query\\",\\"time\\":1223,\\"bytes\\":2434234}"') - ) AS "t" ("c0", "c1", "c2", "c3") + ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"sys\\",\\"swap/free\\":1223334,\\"swap/max\\":3223334}"', 'es<#>es-419', '1'), + ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"query\\",\\"time\\":1223,\\"bytes\\":2434234}"', 'en<#>es<#>es-419', '2<#>3') + ) AS "t" ("c1", "c2", "c3", "c4", "c5", "c6") `); }); }); diff --git a/web-console/src/utils/sample-query.tsx b/web-console/src/utils/values-query.tsx similarity index 57% rename from web-console/src/utils/sample-query.tsx rename to web-console/src/utils/values-query.tsx index 2c32618fac1a..3006d4ff49d3 100644 --- a/web-console/src/utils/sample-query.tsx +++ b/web-console/src/utils/values-query.tsx @@ -26,6 +26,7 @@ import { SqlColumnList, SqlQuery, SqlRecord, + SqlType, SqlValues, } from '@druid-toolkit/query'; @@ -33,11 +34,19 @@ import { oneOf } from './general'; const SAMPLE_ARRAY_SEPARATOR = '<#>'; // Note that this is a regexp so don't add anything that is a special regexp thing -function nullForColumn(column: Column): LiteralValue { - return oneOf(column.sqlType, 'BIGINT', 'DOUBLE', 'FLOAT') ? 0 : ''; +function getEffectiveSqlType(column: Column): string | undefined { + const sqlType = column.sqlType; + if (sqlType === 'ARRAY' && String(column.nativeType).startsWith('ARRAY<')) { + return `${SqlType.fromNativeType(String(column.nativeType).slice(6, -1))} ARRAY`; + } + return sqlType; } -export function sampleDataToQuery(sample: QueryResult): SqlQuery { +function nullForSqlType(sqlType: string | undefined): LiteralValue { + return oneOf(sqlType, 'BIGINT', 'DOUBLE', 'FLOAT') ? 0 : ''; +} + +export function queryResultToValuesQuery(sample: QueryResult): SqlQuery { const { header, rows } = sample; return SqlQuery.create( new SqlAlias({ @@ -45,14 +54,26 @@ export function sampleDataToQuery(sample: QueryResult): SqlQuery { rows.map(row => SqlRecord.create( row.map((r, i) => { - if (header[i].nativeType === 'COMPLEX') { + const column = header[i]; + const { nativeType } = column; + const sqlType = getEffectiveSqlType(column); + if (nativeType === 'COMPLEX') { return L(JSON.stringify(r)); - } else if (String(header[i].sqlType).endsWith(' ARRAY')) { + } else if (String(sqlType).endsWith(' ARRAY')) { return L(r.join(SAMPLE_ARRAY_SEPARATOR)); + } else if ( + sqlType === 'OTHER' && + String(nativeType).startsWith('COMPLEX<') && + typeof r === 'string' && + r.startsWith('"') && + r.endsWith('"') + ) { + // r is a JSON encoded base64 string + return L(r.slice(1, -1)); } else if (r == null || typeof r === 'object') { // Avoid actually using NULL literals as they create havoc in the VALUES type system and throw errors. // Also, cleanup array if it happens to get here, it shouldn't. - return L(nullForColumn(header[i])); + return L(nullForSqlType(sqlType)); } else { return L(r); } @@ -61,19 +82,23 @@ export function sampleDataToQuery(sample: QueryResult): SqlQuery { ), ), alias: RefName.alias('t'), - columns: SqlColumnList.create(header.map((_, i) => RefName.create(`c${i}`, true))), + columns: SqlColumnList.create(header.map((_, i) => RefName.create(`c${i + 1}`, true))), }), ).changeSelectExpressions( - header.map(({ name, nativeType, sqlType }, i) => { - let ex: SqlExpression = C(`c${i}`); + header.map((column, i) => { + const { name, nativeType } = column; + const sqlType = getEffectiveSqlType(column); + let ex: SqlExpression = C(`c${i + 1}`); if (nativeType === 'COMPLEX') { ex = F('PARSE_JSON', ex); - } else if (sqlType && sqlType.endsWith(' ARRAY')) { + } else if (String(sqlType).endsWith(' ARRAY')) { ex = F('STRING_TO_ARRAY', ex, SAMPLE_ARRAY_SEPARATOR); - if (sqlType !== 'VARCHAR ARRAY') { + if (sqlType && sqlType !== 'ARRAY' && sqlType !== 'VARCHAR ARRAY') { ex = ex.cast(sqlType); } - } else if (sqlType) { + } else if (sqlType === 'OTHER' && String(nativeType).startsWith('COMPLEX<')) { + ex = F('DECODE_BASE64_COMPLEX', String(nativeType).slice(8, -1), ex); + } else if (sqlType && sqlType !== 'OTHER') { ex = ex.cast(sqlType); } return ex.as(name, true); diff --git a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx index 149843adc0af..5f0d24f69b1d 100644 --- a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx +++ b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx @@ -79,7 +79,7 @@ import { filterMap, oneOf, queryDruidSql, - sampleDataToQuery, + queryResultToValuesQuery, tickIcon, timeFormatToSql, wait, @@ -479,7 +479,7 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) { const sampleDataQuery = useMemo(() => { if (!sampleState.data) return; - return sampleDataToQuery(sampleState.data); + return queryResultToValuesQuery(sampleState.data); }, [sampleState.data]); const previewQueryString = useLastDefined( diff --git a/web-console/src/views/workbench-view/execution-summary-panel/execution-summary-panel.tsx b/web-console/src/views/workbench-view/execution-summary-panel/execution-summary-panel.tsx index 75999b5ee15d..e64e05a703ba 100644 --- a/web-console/src/views/workbench-view/execution-summary-panel/execution-summary-panel.tsx +++ b/web-console/src/views/workbench-view/execution-summary-panel/execution-summary-panel.tsx @@ -16,13 +16,14 @@ * limitations under the License. */ -import { Button, ButtonGroup, Menu, MenuDivider, MenuItem, Position } from '@blueprintjs/core'; +import { Button, ButtonGroup, Menu, MenuItem, Position } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import { Popover2 } from '@blueprintjs/popover2'; import type { JSX } from 'react'; import React, { useState } from 'react'; import type { Execution } from '../../../druid-models'; +import type { Format } from '../../../utils'; import { copyQueryResultsToClipboard, downloadQueryResults, @@ -76,11 +77,11 @@ export const ExecutionSummaryPanel = React.memo(function ExecutionSummaryPanel( const warningCount = execution?.stages?.getWarningCount(); - const handleDownload = (format: string) => { + const handleDownload = (format: Format) => { downloadQueryResults(queryResult, `results-${execution.id}.${format}`, format); }; - const handleCopy = (format: string) => { + const handleCopy = (format: Format) => { copyQueryResultsToClipboard(queryResult, format); }; @@ -113,14 +114,18 @@ export const ExecutionSummaryPanel = React.memo(function ExecutionSummaryPanel( className="download-button" content={ - - handleDownload('csv')} /> - handleDownload('tsv')} /> - handleDownload('json')} /> - - handleCopy('csv')} /> - handleCopy('tsv')} /> - handleCopy('json')} /> + + handleDownload('csv')} /> + handleDownload('tsv')} /> + handleDownload('json')} /> + handleDownload('sql')} /> + + + handleCopy('csv')} /> + handleCopy('tsv')} /> + handleCopy('json')} /> + handleCopy('sql')} /> + } position={Position.BOTTOM_RIGHT} From 669d2b02e0ec3df508aa194c64765a3ba1734aca Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 15 May 2024 12:13:55 -0700 Subject: [PATCH 2/9] address NULL issue --- web-console/src/utils/values-query.tsx | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/web-console/src/utils/values-query.tsx b/web-console/src/utils/values-query.tsx index 3006d4ff49d3..91c8166af43f 100644 --- a/web-console/src/utils/values-query.tsx +++ b/web-console/src/utils/values-query.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import type { Column, LiteralValue, QueryResult, SqlExpression } from '@druid-toolkit/query'; +import type { Column, QueryResult, SqlExpression } from '@druid-toolkit/query'; import { C, F, @@ -30,8 +30,6 @@ import { SqlValues, } from '@druid-toolkit/query'; -import { oneOf } from './general'; - const SAMPLE_ARRAY_SEPARATOR = '<#>'; // Note that this is a regexp so don't add anything that is a special regexp thing function getEffectiveSqlType(column: Column): string | undefined { @@ -42,8 +40,8 @@ function getEffectiveSqlType(column: Column): string | undefined { return sqlType; } -function nullForSqlType(sqlType: string | undefined): LiteralValue { - return oneOf(sqlType, 'BIGINT', 'DOUBLE', 'FLOAT') ? 0 : ''; +function columnIsAllNulls(rows: readonly unknown[][], columnIndex: number): boolean { + return rows.every(row => row[columnIndex] === null); } export function queryResultToValuesQuery(sample: QueryResult): SqlQuery { @@ -70,10 +68,9 @@ export function queryResultToValuesQuery(sample: QueryResult): SqlQuery { ) { // r is a JSON encoded base64 string return L(r.slice(1, -1)); - } else if (r == null || typeof r === 'object') { - // Avoid actually using NULL literals as they create havoc in the VALUES type system and throw errors. - // Also, cleanup array if it happens to get here, it shouldn't. - return L(nullForSqlType(sqlType)); + } else if (typeof r === 'object') { + // Cleanup array if it happens to get here, it shouldn't. + return L.NULL; } else { return L(r); } @@ -88,7 +85,10 @@ export function queryResultToValuesQuery(sample: QueryResult): SqlQuery { header.map((column, i) => { const { name, nativeType } = column; const sqlType = getEffectiveSqlType(column); - let ex: SqlExpression = C(`c${i + 1}`); + + // The columnIsAllNulls check is needed due to https://github.com/apache/druid/issues/16456 + // Remove it when the issue above is resolved + let ex: SqlExpression = columnIsAllNulls(rows, i) ? L.NULL : C(`c${i + 1}`); if (nativeType === 'COMPLEX') { ex = F('PARSE_JSON', ex); } else if (String(sqlType).endsWith(' ARRAY')) { From 3b7b81f756f3463ac8ec042d3ca8e42b172fd0d2 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 15 May 2024 13:56:02 -0700 Subject: [PATCH 3/9] add decription --- web-console/src/utils/values-query.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web-console/src/utils/values-query.tsx b/web-console/src/utils/values-query.tsx index 91c8166af43f..f6adc03475e2 100644 --- a/web-console/src/utils/values-query.tsx +++ b/web-console/src/utils/values-query.tsx @@ -32,6 +32,13 @@ import { const SAMPLE_ARRAY_SEPARATOR = '<#>'; // Note that this is a regexp so don't add anything that is a special regexp thing +/** + This function corrects for the legacy behaviour where Druid sometimes returns array columns as + { sqlType: 'ARRAY', nativeType: 'ARRAY' } + instead of the more correct description of + { sqlType: 'VARCHAR ARRAY', nativeType: 'ARRAY' } + use this function to get the effective SQL type of `VARCHAR ARRAY` + */ function getEffectiveSqlType(column: Column): string | undefined { const sqlType = column.sqlType; if (sqlType === 'ARRAY' && String(column.nativeType).startsWith('ARRAY<')) { From 5dae74ac64cf60c4323b8cb5fe858468abd65978 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Wed, 15 May 2024 14:11:13 -0700 Subject: [PATCH 4/9] extend test --- web-console/src/utils/values-query.spec.tsx | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/web-console/src/utils/values-query.spec.tsx b/web-console/src/utils/values-query.spec.tsx index 85839aa33584..c71ab4860749 100644 --- a/web-console/src/utils/values-query.spec.tsx +++ b/web-console/src/utils/values-query.spec.tsx @@ -24,9 +24,9 @@ describe('queryResultToValuesQuery', () => { it('works', () => { const result = QueryResult.fromRawResult( [ - ['__time', 'host', 'service', 'msg', 'language', 'nums'], - ['LONG', 'STRING', 'STRING', 'COMPLEX', 'ARRAY', 'ARRAY'], - ['TIMESTAMP', 'VARCHAR', 'VARCHAR', 'OTHER', 'ARRAY', 'ARRAY'], + ['__time', 'host', 'service', 'msg', 'language', 'nums', 'nulls'], + ['LONG', 'STRING', 'STRING', 'COMPLEX', 'ARRAY', 'ARRAY', 'STRING'], + ['TIMESTAMP', 'VARCHAR', 'VARCHAR', 'OTHER', 'ARRAY', 'ARRAY', 'VARCHAR'], [ '2022-02-01T00:00:00.000Z', 'brokerA.internal', @@ -34,6 +34,7 @@ describe('queryResultToValuesQuery', () => { '{"type":"sys","swap/free":1223334,"swap/max":3223334}', ['es', 'es-419'], [1], + null, ], [ '2022-02-01T00:00:00.000Z', @@ -42,6 +43,7 @@ describe('queryResultToValuesQuery', () => { '{"type":"query","time":1223,"bytes":2434234}', ['en', 'es', 'es-419'], [2, 3], + null, ], ], false, @@ -57,12 +59,13 @@ describe('queryResultToValuesQuery', () => { CAST("c3" AS VARCHAR) AS "service", PARSE_JSON("c4") AS "msg", STRING_TO_ARRAY("c5", '<#>') AS "language", - CAST(STRING_TO_ARRAY("c6", '<#>') AS BIGINT ARRAY) AS "nums" + CAST(STRING_TO_ARRAY("c6", '<#>') AS BIGINT ARRAY) AS "nums", + CAST(NULL AS VARCHAR) AS "nulls" FROM ( VALUES - ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"sys\\",\\"swap/free\\":1223334,\\"swap/max\\":3223334}"', 'es<#>es-419', '1'), - ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"query\\",\\"time\\":1223,\\"bytes\\":2434234}"', 'en<#>es<#>es-419', '2<#>3') - ) AS "t" ("c1", "c2", "c3", "c4", "c5", "c6") + ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"sys\\",\\"swap/free\\":1223334,\\"swap/max\\":3223334}"', 'es<#>es-419', '1', NULL), + ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"query\\",\\"time\\":1223,\\"bytes\\":2434234}"', 'en<#>es<#>es-419', '2<#>3', NULL) + ) AS "t" ("c1", "c2", "c3", "c4", "c5", "c6", "c7") `); }); }); From 05260f6781b03b777bd4adab3b6d516f1a6ec46d Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Thu, 16 May 2024 09:24:15 -0700 Subject: [PATCH 5/9] fix json --- web-console/src/utils/values-query.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/web-console/src/utils/values-query.tsx b/web-console/src/utils/values-query.tsx index f6adc03475e2..2f1a5f699cae 100644 --- a/web-console/src/utils/values-query.tsx +++ b/web-console/src/utils/values-query.tsx @@ -29,6 +29,9 @@ import { SqlType, SqlValues, } from '@druid-toolkit/query'; +import * as JSONBig from 'json-bigint-native'; + +import { oneOf } from './general'; const SAMPLE_ARRAY_SEPARATOR = '<#>'; // Note that this is a regexp so don't add anything that is a special regexp thing @@ -51,6 +54,10 @@ function columnIsAllNulls(rows: readonly unknown[][], columnIndex: number): bool return rows.every(row => row[columnIndex] === null); } +function isJsonString(x: unknown): boolean { + return typeof x === 'string' && oneOf(x[0], '"', '{', '['); +} + export function queryResultToValuesQuery(sample: QueryResult): SqlQuery { const { header, rows } = sample; return SqlQuery.create( @@ -63,7 +70,7 @@ export function queryResultToValuesQuery(sample: QueryResult): SqlQuery { const { nativeType } = column; const sqlType = getEffectiveSqlType(column); if (nativeType === 'COMPLEX') { - return L(JSON.stringify(r)); + return L(isJsonString(r) ? r : JSONBig.stringify(r)); } else if (String(sqlType).endsWith(' ARRAY')) { return L(r.join(SAMPLE_ARRAY_SEPARATOR)); } else if ( From b718b1fad3885bd1cf4995cb752b57727a531792 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Thu, 16 May 2024 11:21:07 -0700 Subject: [PATCH 6/9] more types --- web-console/script/druid | 11 +++++++++-- web-console/src/utils/types.ts | 17 +++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/web-console/script/druid b/web-console/script/druid index 69346d4a2b1a..68f6e58f12a4 100755 --- a/web-console/script/druid +++ b/web-console/script/druid @@ -62,9 +62,16 @@ function _build_distribution() { && cd apache-druid-$(_get_druid_version) \ && mkdir -p extensions/druid-testing-tools \ && cp "$(_get_code_root)/extensions-core/testing-tools/target/druid-testing-tools-$(_get_druid_version).jar" extensions/druid-testing-tools/ \ - && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-datasketches\", \"druid-multi-stage-query\", \"druid-testing-tools\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ + && mkdir -p extensions/druid-compressed-bigdecimal \ + && cp "$(_get_code_root)/extensions-contrib/compressed-bigdecimal/target/druid-compressed-bigdecimal-$(_get_druid_version).jar" extensions/druid-compressed-bigdecimal/ \ + && mkdir -p extensions/druid-momentsketch \ + && cp "$(_get_code_root)/extensions-contrib/momentsketch/target/druid-momentsketch-$(_get_druid_version).jar" extensions/druid-momentsketch/ \ + && mkdir -p extensions/druid-tdigestsketch \ + && cp "$(_get_code_root)/extensions-contrib/tdigestsketch/target/druid-tdigestsketch-$(_get_druid_version).jar" extensions/druid-tdigestsketch/ \ + && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\", \"druid-momentsketch\", \"druid-tdigestsketch\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ + && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\", \"druid-momentsketch\", \"druid-tdigestsketch\"]" >> conf/druid/auto/_common/common.runtime.properties \ && echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ - && echo -e "\n\ndruid.generic.useDefaultValueForNull=false" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ + && echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/auto/_common/common.runtime.properties \ ) } diff --git a/web-console/src/utils/types.ts b/web-console/src/utils/types.ts index 937e45b191b7..8ebbd6938b3c 100644 --- a/web-console/src/utils/types.ts +++ b/web-console/src/utils/types.ts @@ -41,6 +41,9 @@ export function dataTypeToIcon(dataType: string): IconName { const typeUpper = dataType.toUpperCase(); switch (typeUpper) { + case 'NULL': + return IconNames.CIRCLE; + case 'TIMESTAMP': return IconNames.TIME; @@ -83,6 +86,9 @@ export function dataTypeToIcon(dataType: string): IconName { case 'COMPLEX': case 'COMPLEX': + case 'COMPLEX': + case 'COMPLEX': + case 'COMPLEX': return IconNames.HORIZONTAL_DISTRIBUTION; case 'COMPLEX': @@ -95,8 +101,15 @@ export function dataTypeToIcon(dataType: string): IconName { case 'COMPLEX': return IconNames.DOUBLE_CHEVRON_RIGHT; - case 'NULL': - return IconNames.CIRCLE; + case 'COMPLEX': + return IconNames.FILTER_LIST; + + case 'COMPLEX': + case 'COMPLEX': + return IconNames.HURRICANE; + + case 'COMPLEX': + return IconNames.SORT_NUMERICAL_DESC; default: if (typeUpper.startsWith('ARRAY')) return IconNames.ARRAY; From 7d4149c62b38d6969c8e1bef1ded6bf4ed6ad257 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Thu, 16 May 2024 14:18:25 -0700 Subject: [PATCH 7/9] fix braces with nulls --- web-console/src/utils/table-helpers.ts | 6 +++--- .../workbench-view/result-table-pane/result-table-pane.tsx | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/web-console/src/utils/table-helpers.ts b/web-console/src/utils/table-helpers.ts index a04635c61c50..a2b97350fa4e 100644 --- a/web-console/src/utils/table-helpers.ts +++ b/web-console/src/utils/table-helpers.ts @@ -55,11 +55,11 @@ export function getNumericColumnBraces( queryResult.header.forEach((column, i) => { if (!oneOf(column.nativeType, 'LONG', 'FLOAT', 'DOUBLE')) return; const formatter = columnHints?.get(column.name)?.formatter || formatNumber; - const brace = filterMap(rows, row => + const braces = filterMap(rows, row => oneOf(typeof row[i], 'number', 'bigint') ? formatter(row[i]) : undefined, ); - if (rows.length === brace.length) { - numericColumnBraces[i] = brace; + if (braces.length) { + numericColumnBraces[i] = braces; } }); } diff --git a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx index 294aeeeb7981..9b074937c5a1 100644 --- a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx +++ b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx @@ -620,7 +620,7 @@ export const ResultTablePane = React.memo(function ResultTablePane(props: Result {numericColumnBraces[i] ? ( From a241c8cd855feb92c3c0fe1060784a46dd56c438 Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Thu, 16 May 2024 16:09:21 -0700 Subject: [PATCH 8/9] fix test --- web-console/src/utils/values-query.spec.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web-console/src/utils/values-query.spec.tsx b/web-console/src/utils/values-query.spec.tsx index c71ab4860749..99884f382c1c 100644 --- a/web-console/src/utils/values-query.spec.tsx +++ b/web-console/src/utils/values-query.spec.tsx @@ -63,8 +63,8 @@ describe('queryResultToValuesQuery', () => { CAST(NULL AS VARCHAR) AS "nulls" FROM ( VALUES - ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"sys\\",\\"swap/free\\":1223334,\\"swap/max\\":3223334}"', 'es<#>es-419', '1', NULL), - ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"query\\",\\"time\\":1223,\\"bytes\\":2434234}"', 'en<#>es<#>es-419', '2<#>3', NULL) + ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '{"type":"sys","swap/free":1223334,"swap/max":3223334}', 'es<#>es-419', '1', NULL), + ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '{"type":"query","time":1223,"bytes":2434234}', 'en<#>es<#>es-419', '2<#>3', NULL) ) AS "t" ("c1", "c2", "c3", "c4", "c5", "c6", "c7") `); }); From c4515330e3a0e2d8a12312d4570ce1b4f955400f Mon Sep 17 00:00:00 2001 From: Vadim Ogievetsky Date: Tue, 21 May 2024 10:31:22 -0700 Subject: [PATCH 9/9] update functions to scan --- web-console/script/create-sql-docs.js | 5 +++-- web-console/script/druid | 8 ++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/web-console/script/create-sql-docs.js b/web-console/script/create-sql-docs.js index 82328fd74b57..869888377112 100755 --- a/web-console/script/create-sql-docs.js +++ b/web-console/script/create-sql-docs.js @@ -23,8 +23,8 @@ const snarkdown = require('snarkdown'); const writefile = 'lib/sql-docs.js'; -const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 167; -const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14; +const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 198; +const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 15; const initialFunctionDocs = { TABLE: [['external', convertMarkdownToHtml('Defines a logical table from an external.')]], @@ -78,6 +78,7 @@ const readDoc = async () => { await fs.readFile('../docs/querying/sql-array-functions.md', 'utf-8'), await fs.readFile('../docs/querying/sql-multivalue-string-functions.md', 'utf-8'), await fs.readFile('../docs/querying/sql-json-functions.md', 'utf-8'), + await fs.readFile('../docs/querying/sql-window-functions.md', 'utf-8'), await fs.readFile('../docs/querying/sql-operators.md', 'utf-8'), ].join('\n'); diff --git a/web-console/script/druid b/web-console/script/druid index 68f6e58f12a4..bff3cb490d92 100755 --- a/web-console/script/druid +++ b/web-console/script/druid @@ -64,12 +64,8 @@ function _build_distribution() { && cp "$(_get_code_root)/extensions-core/testing-tools/target/druid-testing-tools-$(_get_druid_version).jar" extensions/druid-testing-tools/ \ && mkdir -p extensions/druid-compressed-bigdecimal \ && cp "$(_get_code_root)/extensions-contrib/compressed-bigdecimal/target/druid-compressed-bigdecimal-$(_get_druid_version).jar" extensions/druid-compressed-bigdecimal/ \ - && mkdir -p extensions/druid-momentsketch \ - && cp "$(_get_code_root)/extensions-contrib/momentsketch/target/druid-momentsketch-$(_get_druid_version).jar" extensions/druid-momentsketch/ \ - && mkdir -p extensions/druid-tdigestsketch \ - && cp "$(_get_code_root)/extensions-contrib/tdigestsketch/target/druid-tdigestsketch-$(_get_druid_version).jar" extensions/druid-tdigestsketch/ \ - && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\", \"druid-momentsketch\", \"druid-tdigestsketch\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ - && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\", \"druid-momentsketch\", \"druid-tdigestsketch\"]" >> conf/druid/auto/_common/common.runtime.properties \ + && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ + && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\"]" >> conf/druid/auto/_common/common.runtime.properties \ && echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \ && echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/auto/_common/common.runtime.properties \ )