Merge remote-tracking branch 'apache/master' into catalog-validation-…

…control

web-console/script/create-sql-docs.js

@@ -23,8 +23,8 @@ const snarkdown = require('snarkdown');
 
 const writefile = 'lib/sql-docs.js';
 
-const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 167;
-const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 14;
+const MINIMUM_EXPECTED_NUMBER_OF_FUNCTIONS = 198;
+const MINIMUM_EXPECTED_NUMBER_OF_DATA_TYPES = 15;
 
 const initialFunctionDocs = {
   TABLE: [['external', convertMarkdownToHtml('Defines a logical table from an external.')]],
@@ -78,6 +78,7 @@ const readDoc = async () => {
     await fs.readFile('../docs/querying/sql-array-functions.md', 'utf-8'),
     await fs.readFile('../docs/querying/sql-multivalue-string-functions.md', 'utf-8'),
     await fs.readFile('../docs/querying/sql-json-functions.md', 'utf-8'),
+    await fs.readFile('../docs/querying/sql-window-functions.md', 'utf-8'),
     await fs.readFile('../docs/querying/sql-operators.md', 'utf-8'),
   ].join('\n');
 

web-console/script/druid

@@ -62,9 +62,12 @@ function _build_distribution() {
     && cd apache-druid-$(_get_druid_version) \
     && mkdir -p extensions/druid-testing-tools \
     && cp "$(_get_code_root)/extensions-core/testing-tools/target/druid-testing-tools-$(_get_druid_version).jar" extensions/druid-testing-tools/ \
-    && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-datasketches\", \"druid-multi-stage-query\", \"druid-testing-tools\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \
+    && mkdir -p extensions/druid-compressed-bigdecimal \
+    && cp "$(_get_code_root)/extensions-contrib/compressed-bigdecimal/target/druid-compressed-bigdecimal-$(_get_druid_version).jar" extensions/druid-compressed-bigdecimal/ \
+    && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \
+    && echo -e "\n\ndruid.extensions.loadList=[\"druid-hdfs-storage\", \"druid-kafka-indexing-service\", \"druid-multi-stage-query\", \"druid-testing-tools\", \"druid-bloom-filter\", \"druid-datasketches\", \"druid-histogram\", \"druid-stats\", \"druid-compressed-bigdecimal\"]" >> conf/druid/auto/_common/common.runtime.properties \
     && echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \
-    && echo -e "\n\ndruid.generic.useDefaultValueForNull=false" >> conf/druid/single-server/micro-quickstart/_common/common.runtime.properties \
+    && echo -e "\n\ndruid.server.http.allowedHttpMethods=[\"HEAD\"]" >> conf/druid/auto/_common/common.runtime.properties \
   )
 }
 

web-console/src/utils/download.ts

@@ -21,6 +21,9 @@ import FileSaver from 'file-saver';
 import * as JSONBig from 'json-bigint-native';
 
 import { copyAndAlert, stringifyValue } from './general';
+import { queryResultToValuesQuery } from './values-query';
+
+export type Format = 'csv' | 'tsv' | 'json' | 'sql';
 
 export function downloadUrl(url: string, filename: string) {
   // Create a link and set the URL using `createObjectURL`
@@ -74,44 +77,43 @@ export function downloadFile(text: string, type: string, filename: string): void
   FileSaver.saveAs(blob, filename);
 }
 
-function queryResultsToString(queryResult: QueryResult, format: string): string {
-  let lines: string[] = [];
-  let separator = '';
-
-  if (format === 'csv' || format === 'tsv') {
-    separator = format === 'csv' ? ',' : '\t';
-    lines.push(
-      queryResult.header.map(column => formatForFormat(column.name, format)).join(separator),
-    );
-    lines = lines.concat(
-      queryResult.rows.map(r => r.map(cell => formatForFormat(cell, format)).join(separator)),
-    );
-  } else {
-    // json
-    lines = queryResult.rows.map(r => {
-      const outputObject: Record<string, any> = {};
-      for (let k = 0; k < r.length; k++) {
-        const newName = queryResult.header[k];
-        if (newName) {
-          outputObject[newName.name] = r[k];
-        }
-      }
-      return JSONBig.stringify(outputObject);
-    });
+function queryResultsToString(queryResult: QueryResult, format: Format): string {
+  const { header, rows } = queryResult;
+
+  switch (format) {
+    case 'csv':
+    case 'tsv': {
+      const separator = format === 'csv' ? ',' : '\t';
+      return [
+        header.map(column => formatForFormat(column.name, format)).join(separator),
+        ...rows.map(r => r.map(cell => formatForFormat(cell, format)).join(separator)),
+      ].join('\n');
+    }
+
+    case 'sql':
+      return queryResultToValuesQuery(queryResult).toString();
+
+    case 'json':
+      return queryResult
+        .toObjectArray()
+        .map(r => JSONBig.stringify(r))
+        .join('\n');
+
+    default:
+      throw new Error(`unknown format: ${format}`);
   }
-  return lines.join('\n');
 }
 
 export function downloadQueryResults(
   queryResult: QueryResult,
   filename: string,
-  format: string,
+  format: Format,
 ): void {
   const resultString: string = queryResultsToString(queryResult, format);
   downloadFile(resultString, format, filename);
 }
 
-export function copyQueryResultsToClipboard(queryResult: QueryResult, format: string): void {
+export function copyQueryResultsToClipboard(queryResult: QueryResult, format: Format): void {
   const resultString: string = queryResultsToString(queryResult, format);
   copyAndAlert(resultString, 'Query results copied to clipboard');
 }

web-console/src/utils/index.tsx

@@ -32,8 +32,8 @@ export * from './object-change';
 export * from './query-action';
 export * from './query-manager';
 export * from './query-state';
-export * from './sample-query';
 export * from './sanitizers';
 export * from './sql';
 export * from './table-helpers';
 export * from './types';
+export * from './values-query';

web-console/src/utils/table-helpers.ts

@@ -55,11 +55,11 @@ export function getNumericColumnBraces(
     queryResult.header.forEach((column, i) => {
       if (!oneOf(column.nativeType, 'LONG', 'FLOAT', 'DOUBLE')) return;
       const formatter = columnHints?.get(column.name)?.formatter || formatNumber;
-      const brace = filterMap(rows, row =>
+      const braces = filterMap(rows, row =>
         oneOf(typeof row[i], 'number', 'bigint') ? formatter(row[i]) : undefined,
       );
-      if (rows.length === brace.length) {
-        numericColumnBraces[i] = brace;
+      if (braces.length) {
+        numericColumnBraces[i] = braces;
       }
     });
   }

web-console/src/utils/types.ts

@@ -41,6 +41,9 @@ export function dataTypeToIcon(dataType: string): IconName {
   const typeUpper = dataType.toUpperCase();
 
   switch (typeUpper) {
+    case 'NULL':
+      return IconNames.CIRCLE;
+
     case 'TIMESTAMP':
       return IconNames.TIME;
 
@@ -75,12 +78,17 @@ export function dataTypeToIcon(dataType: string): IconName {
       return IconNames.DIAGRAM_TREE;
 
     case 'COMPLEX<HYPERUNIQUE>':
+    case 'COMPLEX<HLLSKETCH>':
     case 'COMPLEX<HLLSKETCHBUILD>':
+    case 'COMPLEX<THETASKETCH>':
     case 'COMPLEX<THETASKETCHBUILD>':
       return IconNames.SNOWFLAKE;
 
     case 'COMPLEX<QUANTILESDOUBLESSKETCH>':
     case 'COMPLEX<APPROXIMATEHISTOGRAM>':
+    case 'COMPLEX<FIXEDBUCKETSHISTOGRAM>':
+    case 'COMPLEX<ARRAYOFDOUBLESSKETCH>':
+    case 'COMPLEX<MOMENTSKETCH>':
       return IconNames.HORIZONTAL_DISTRIBUTION;
 
     case 'COMPLEX<VARIANCE>':
@@ -93,8 +101,15 @@ export function dataTypeToIcon(dataType: string): IconName {
     case 'COMPLEX<SERIALIZABLEPAIRLONGSTRING>':
       return IconNames.DOUBLE_CHEVRON_RIGHT;
 
-    case 'NULL':
-      return IconNames.CIRCLE;
+    case 'COMPLEX<BLOOM>':
+      return IconNames.FILTER_LIST;
+
+    case 'COMPLEX<KLLDOUBLESSKETCH>':
+    case 'COMPLEX<KLLFLOATSSKETCH>':
+      return IconNames.HURRICANE;
+
+    case 'COMPLEX<COMPRESSEDBIGDECIMAL>':
+      return IconNames.SORT_NUMERICAL_DESC;
 
     default:
       if (typeUpper.startsWith('ARRAY')) return IconNames.ARRAY;

web-console/src/utils/sample-query.spec.tsx → web-console/src/utils/values-query.spec.tsx

@@ -18,26 +18,32 @@
 
 import { QueryResult, sane } from '@druid-toolkit/query';
 
-import { sampleDataToQuery } from './sample-query';
+import { queryResultToValuesQuery } from './values-query';
 
-describe('sample-query', () => {
+describe('queryResultToValuesQuery', () => {
   it('works', () => {
     const result = QueryResult.fromRawResult(
       [
-        ['__time', 'host', 'service', 'msg'],
-        ['LONG', 'STRING', 'STRING', 'COMPLEX<json>'],
-        ['TIMESTAMP', 'VARCHAR', 'VARCHAR', 'OTHER'],
+        ['__time', 'host', 'service', 'msg', 'language', 'nums', 'nulls'],
+        ['LONG', 'STRING', 'STRING', 'COMPLEX<json>', 'ARRAY<STRING>', 'ARRAY<LONG>', 'STRING'],
+        ['TIMESTAMP', 'VARCHAR', 'VARCHAR', 'OTHER', 'ARRAY', 'ARRAY', 'VARCHAR'],
         [
           '2022-02-01T00:00:00.000Z',
           'brokerA.internal',
           'broker',
           '{"type":"sys","swap/free":1223334,"swap/max":3223334}',
+          ['es', 'es-419'],
+          [1],
+          null,
         ],
         [
           '2022-02-01T00:00:00.000Z',
           'brokerA.internal',
           'broker',
           '{"type":"query","time":1223,"bytes":2434234}',
+          ['en', 'es', 'es-419'],
+          [2, 3],
+          null,
         ],
       ],
       false,
@@ -46,17 +52,20 @@ describe('sample-query', () => {
       true,
     );
 
-    expect(sampleDataToQuery(result).toString()).toEqual(sane`
+    expect(queryResultToValuesQuery(result).toString()).toEqual(sane`
       SELECT
-        CAST("c0" AS TIMESTAMP) AS "__time",
-        CAST("c1" AS VARCHAR) AS "host",
-        CAST("c2" AS VARCHAR) AS "service",
-        PARSE_JSON("c3") AS "msg"
+        CAST("c1" AS TIMESTAMP) AS "__time",
+        CAST("c2" AS VARCHAR) AS "host",
+        CAST("c3" AS VARCHAR) AS "service",
+        PARSE_JSON("c4") AS "msg",
+        STRING_TO_ARRAY("c5", '<#>') AS "language",
+        CAST(STRING_TO_ARRAY("c6", '<#>') AS BIGINT ARRAY) AS "nums",
+        CAST(NULL AS VARCHAR) AS "nulls"
       FROM (
         VALUES
-        ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"sys\\",\\"swap/free\\":1223334,\\"swap/max\\":3223334}"'),
-        ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '"{\\"type\\":\\"query\\",\\"time\\":1223,\\"bytes\\":2434234}"')
-      ) AS "t" ("c0", "c1", "c2", "c3")
+        ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '{"type":"sys","swap/free":1223334,"swap/max":3223334}', 'es<#>es-419', '1', NULL),
+        ('2022-02-01T00:00:00.000Z', 'brokerA.internal', 'broker', '{"type":"query","time":1223,"bytes":2434234}', 'en<#>es<#>es-419', '2<#>3', NULL)
+      ) AS "t" ("c1", "c2", "c3", "c4", "c5", "c6", "c7")
     `);
   });
 });