diff --git a/licenses.yaml b/licenses.yaml index da0ceb887d33..ad0f90afe86c 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -5094,7 +5094,7 @@ license_category: binary module: web-console license_name: Apache License version 2.0 copyright: Imply Data -version: 0.22.11 +version: 0.22.13 --- diff --git a/web-console/package-lock.json b/web-console/package-lock.json index 816f734948e0..9d835e3da043 100644 --- a/web-console/package-lock.json +++ b/web-console/package-lock.json @@ -14,7 +14,7 @@ "@blueprintjs/datetime2": "^0.9.35", "@blueprintjs/icons": "^4.16.0", "@blueprintjs/popover2": "^1.14.9", - "@druid-toolkit/query": "^0.22.11", + "@druid-toolkit/query": "^0.22.13", "@druid-toolkit/visuals-core": "^0.3.3", "@druid-toolkit/visuals-react": "^0.3.3", "ace-builds": "~1.4.14", @@ -1004,9 +1004,9 @@ } }, "node_modules/@druid-toolkit/query": { - "version": "0.22.11", - "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.11.tgz", - "integrity": "sha512-VVEn/tsEr9fb+8eKc+nu3/YH7l+LZ1vd0D32UDo66GLS3cI+EKOCM7VYC8lTvB1tAS+98w/EzfbdlRPlkSeOoQ==", + "version": "0.22.13", + "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.13.tgz", + "integrity": "sha512-p0Cmmbk55vLaYs2WWcUr09qDRU2IrkXOxGgUG+wS6Uuq/ALBqSmUDlbMSxB3vJjMvegiwgJ8+n7VfVpO0t/bJg==", "dependencies": { "tslib": "^2.5.2" } @@ -19146,9 +19146,9 @@ "dev": true }, "@druid-toolkit/query": { - "version": "0.22.11", - "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.11.tgz", - "integrity": "sha512-VVEn/tsEr9fb+8eKc+nu3/YH7l+LZ1vd0D32UDo66GLS3cI+EKOCM7VYC8lTvB1tAS+98w/EzfbdlRPlkSeOoQ==", + "version": "0.22.13", + "resolved": "https://registry.npmjs.org/@druid-toolkit/query/-/query-0.22.13.tgz", + "integrity": "sha512-p0Cmmbk55vLaYs2WWcUr09qDRU2IrkXOxGgUG+wS6Uuq/ALBqSmUDlbMSxB3vJjMvegiwgJ8+n7VfVpO0t/bJg==", "requires": { "tslib": "^2.5.2" } diff --git a/web-console/package.json b/web-console/package.json index 8cd0985f644a..3e713fc2f68a 100644 --- a/web-console/package.json +++ b/web-console/package.json @@ -68,7 +68,7 @@ "@blueprintjs/datetime2": "^0.9.35", "@blueprintjs/icons": "^4.16.0", "@blueprintjs/popover2": "^1.14.9", - "@druid-toolkit/query": "^0.22.11", + "@druid-toolkit/query": "^0.22.13", "@druid-toolkit/visuals-core": "^0.3.3", "@druid-toolkit/visuals-react": "^0.3.3", "ace-builds": "~1.4.14", diff --git a/web-console/src/components/record-table-pane/record-table-pane.tsx b/web-console/src/components/record-table-pane/record-table-pane.tsx index a2849ed0d5c3..bfd9b644de91 100644 --- a/web-console/src/components/record-table-pane/record-table-pane.tsx +++ b/web-console/src/components/record-table-pane/record-table-pane.tsx @@ -104,7 +104,7 @@ export const RecordTablePane = React.memo(function RecordTablePane(props: Record const finalPage = hasMoreResults && Math.floor(queryResult.rows.length / pagination.pageSize) === pagination.page; // on the last page - const numericColumnBraces = getNumericColumnBraces(queryResult, pagination); + const numericColumnBraces = getNumericColumnBraces(queryResult, undefined, pagination); return (
{finalPage ? ( diff --git a/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx b/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx index e36ae5112713..b8816fd493a6 100644 --- a/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx +++ b/web-console/src/dialogs/async-action-dialog/async-action-dialog.tsx @@ -38,8 +38,8 @@ export interface AsyncActionDialogProps { className?: string; icon?: IconName; intent?: Intent; - successText: string; - failText: string; + successText: ReactNode; + failText: ReactNode; warningChecks?: ReactNode[]; children?: ReactNode; } diff --git a/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx b/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx index dba85268d000..f5e2ca8add60 100644 --- a/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx +++ b/web-console/src/dialogs/kill-datasource-dialog/kill-datasource-dialog.tsx @@ -66,7 +66,12 @@ export const KillDatasourceDialog = function KillDatasourceDialog( return resp.data; }} confirmButtonText="Permanently delete unused segments" - successText="Kill task was issued. Unused segments in datasource will be deleted" + successText={ + <> + Kill task was issued. Unused segments in datasource {datasource} will + be deleted + + } failText="Failed submit kill task" intent={Intent.DANGER} onClose={onClose} diff --git a/web-console/src/druid-models/execution/execution.ts b/web-console/src/druid-models/execution/execution.ts index 0cf8d5d0ed37..799de6f9c511 100644 --- a/web-console/src/druid-models/execution/execution.ts +++ b/web-console/src/druid-models/execution/execution.ts @@ -440,7 +440,10 @@ export class Execution { value.queryContext = queryContext; const parsedQuery = parseSqlQuery(sqlQuery); if (value.result && (parsedQuery || queryContext)) { - value.result = value.result.attachQuery({ context: queryContext }, parsedQuery); + value.result = value.result.attachQuery( + { ...this.nativeQuery, context: queryContext }, + parsedQuery, + ); } return new Execution(value); @@ -463,7 +466,10 @@ export class Execution { public changeResult(result: QueryResult): Execution { return new Execution({ ...this.valueOf(), - result: result.attachQuery({}, this.sqlQuery ? parseSqlQuery(this.sqlQuery) : undefined), + result: result.attachQuery( + this.nativeQuery, + this.sqlQuery ? parseSqlQuery(this.sqlQuery) : undefined, + ), }); } diff --git a/web-console/src/druid-models/workbench-query/workbench-query.ts b/web-console/src/druid-models/workbench-query/workbench-query.ts index d59cdbbe92ef..e912e61ed57b 100644 --- a/web-console/src/druid-models/workbench-query/workbench-query.ts +++ b/web-console/src/druid-models/workbench-query/workbench-query.ts @@ -18,6 +18,7 @@ import type { QueryParameter, + QueryPayload, SqlClusteredByClause, SqlExpression, SqlPartitionedByClause, @@ -446,7 +447,7 @@ export class WorkbenchQuery { public getApiQuery(makeQueryId: () => string = uuidv4): { engine: DruidEngine; - query: Record; + query: QueryPayload; prefixLines: number; cancelQueryId?: string; } { @@ -478,7 +479,7 @@ export class WorkbenchQuery { }; } - let apiQuery: Record = {}; + let apiQuery: QueryPayload; if (this.isJsonLike()) { try { apiQuery = Hjson.parse(queryString); diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index b4537a63e08b..1ea872b68f28 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -338,6 +338,22 @@ export function pluralIfNeeded(n: NumberLike, singular: string, plural?: string) // ---------------------------- +export function partition(xs: T[], predicate: (x: T, i: number) => boolean): [T[], T[]] { + const match: T[] = []; + const nonMatch: T[] = []; + + for (let i = 0; i < xs.length; i++) { + const x = xs[i]; + if (predicate(x, i)) { + match.push(x); + } else { + nonMatch.push(x); + } + } + + return [match, nonMatch]; +} + export function filterMap(xs: readonly T[], f: (x: T, i: number) => Q | undefined): Q[] { return xs.map(f).filter((x: Q | undefined) => typeof x !== 'undefined') as Q[]; } diff --git a/web-console/src/utils/table-helpers.ts b/web-console/src/utils/table-helpers.ts index 7eedd1acaab9..a04635c61c50 100644 --- a/web-console/src/utils/table-helpers.ts +++ b/web-console/src/utils/table-helpers.ts @@ -32,9 +32,16 @@ export function changePage(pagination: Pagination, page: number): Pagination { return deepSet(pagination, 'page', page); } +export interface ColumnHint { + displayName?: string; + group?: string; + formatter?: (x: any) => string; +} + export function getNumericColumnBraces( queryResult: QueryResult, - pagination?: Pagination, + columnHints: Map | undefined, + pagination: Pagination | undefined, ): Record { let rows = queryResult.rows; @@ -47,8 +54,9 @@ export function getNumericColumnBraces( if (rows.length) { queryResult.header.forEach((column, i) => { if (!oneOf(column.nativeType, 'LONG', 'FLOAT', 'DOUBLE')) return; + const formatter = columnHints?.get(column.name)?.formatter || formatNumber; const brace = filterMap(rows, row => - oneOf(typeof row[i], 'number', 'bigint') ? formatNumber(row[i]) : undefined, + oneOf(typeof row[i], 'number', 'bigint') ? formatter(row[i]) : undefined, ); if (rows.length === brace.length) { numericColumnBraces[i] = brace; diff --git a/web-console/src/views/datasources-view/datasources-view.tsx b/web-console/src/views/datasources-view/datasources-view.tsx index 713df9b18b1c..54b11a5a0cbd 100644 --- a/web-console/src/views/datasources-view/datasources-view.tsx +++ b/web-console/src/views/datasources-view/datasources-view.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import { FormGroup, InputGroup, Intent, MenuItem, Switch } from '@blueprintjs/core'; +import { FormGroup, InputGroup, Intent, MenuItem, Switch, Tag } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import { SqlQuery, T } from '@druid-toolkit/query'; import classNames from 'classnames'; @@ -651,8 +651,18 @@ GROUP BY 1, 2`; return resp.data; }} confirmButtonText="Mark as unused all segments" - successText="All segments in datasource have been marked as unused" - failText="Failed to mark as unused all segments in datasource" + successText={ + <> + All segments in datasource {datasourceToMarkAsUnusedAllSegmentsIn}{' '} + have been marked as unused + + } + failText={ + <> + Failed to mark as unused all segments in datasource{' '} + {datasourceToMarkAsUnusedAllSegmentsIn} + + } intent={Intent.DANGER} onClose={() => { this.setState({ datasourceToMarkAsUnusedAllSegmentsIn: undefined }); @@ -684,8 +694,19 @@ GROUP BY 1, 2`; return resp.data; }} confirmButtonText="Mark as used all segments" - successText="All non-overshadowed segments in datasource have been marked as used" - failText="Failed to mark as used all non-overshadowed segments in datasource" + successText={ + <> + All non-overshadowed segments in datasource{' '} + {datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn} have been marked + as used + + } + failText={ + <> + Failed to mark as used all non-overshadowed segments in datasource{' '} + {datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn} + + } intent={Intent.PRIMARY} onClose={() => { this.setState({ datasourceToMarkAllNonOvershadowedSegmentsAsUsedIn: undefined }); diff --git a/web-console/src/views/explore-view/modules/components/generic-output-table/generic-output-table.tsx b/web-console/src/views/explore-view/modules/components/generic-output-table/generic-output-table.tsx index 255a3a9b6fa8..4f79156175be 100644 --- a/web-console/src/views/explore-view/modules/components/generic-output-table/generic-output-table.tsx +++ b/web-console/src/views/explore-view/modules/components/generic-output-table/generic-output-table.tsx @@ -31,7 +31,7 @@ import ReactTable from 'react-table'; import { BracedText, Deferred, TableCell } from '../../../../../components'; import { possibleDruidFormatForValues, TIME_COLUMN } from '../../../../../druid-models'; import { SMALL_TABLE_PAGE_SIZE, SMALL_TABLE_PAGE_SIZE_OPTIONS } from '../../../../../react-table'; -import type { Pagination, QueryAction } from '../../../../../utils'; +import type { ColumnHint, Pagination, QueryAction } from '../../../../../utils'; import { columnToIcon, columnToWidth, @@ -60,30 +60,34 @@ function isComparable(x: unknown): boolean { return x !== null && x !== ''; } -function columnNester(columns: TableColumn[], groupHints: string[] | undefined): TableColumn[] { - if (!groupHints) return columns; +function columnNester( + tableColumns: TableColumn[], + resultColumns: readonly Column[], + columnHints: Map | undefined, +): TableColumn[] { + if (!columnHints) return tableColumns; const ret: TableColumn[] = []; - let currentGroupHint: string | null = null; + let currentGroupName: string | null = null; let currentColumnGroup: TableColumn | null = null; - for (let i = 0; i < columns.length; i++) { - const column = columns[i]; - const groupHint = groupHints[i]; - if (groupHint) { - if (currentGroupHint === groupHint) { - currentColumnGroup!.columns!.push(column); + for (let i = 0; i < tableColumns.length; i++) { + const tableColumn = tableColumns[i]; + const group = columnHints.get(resultColumns[i].name)?.group; + if (group) { + if (currentGroupName === group) { + currentColumnGroup!.columns!.push(tableColumn); } else { - currentGroupHint = groupHint; + currentGroupName = group; ret.push( (currentColumnGroup = { - Header:
{currentGroupHint}
, - columns: [column], + Header:
{currentGroupName}
, + columns: [tableColumn], }), ); } } else { - ret.push(column); - currentGroupHint = null; + ret.push(tableColumn); + currentGroupName = null; currentColumnGroup = null; } } @@ -94,12 +98,12 @@ function columnNester(columns: TableColumn[], groupHints: string[] | undefined): export interface GenericOutputTableProps { queryResult: QueryResult; onQueryAction(action: QueryAction): void; - onOrderByChange?(columnIndex: number, desc: boolean): void; + onOrderByChange?(columnName: string, desc: boolean): void; onExport?(): void; runeMode: boolean; showTypeIcons: boolean; initPageSize?: number; - groupHints?: string[]; + columnHints?: Map; } export const GenericOutputTable = React.memo(function GenericOutputTable( @@ -113,7 +117,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable( runeMode, showTypeIcons, initPageSize, - groupHints, + columnHints, } = props; const parsedQuery = queryResult.sqlQuery; const [pagination, setPagination] = useState({ @@ -159,7 +163,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable( icon={reverseOrderByDirection === 'ASC' ? IconNames.SORT_ASC : IconNames.SORT_DESC} text={`Order ${reverseOrderByDirection === 'ASC' ? 'ascending' : 'descending'}`} onClick={() => { - onOrderByChange(headerIndex, reverseOrderByDirection !== 'ASC'); + onOrderByChange(header, reverseOrderByDirection !== 'ASC'); }} />, ); @@ -170,7 +174,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable( icon={IconNames.SORT_DESC} text="Order descending" onClick={() => { - onOrderByChange(headerIndex, true); + onOrderByChange(header, true); }} />, { - onOrderByChange(headerIndex, false); + onOrderByChange(header, false); }} />, ); @@ -426,7 +430,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable( const finalPage = hasMoreResults && Math.floor(queryResult.rows.length / pagination.pageSize) === pagination.page; // on the last page - const numericColumnBraces = getNumericColumnBraces(queryResult, pagination); + const numericColumnBraces = getNumericColumnBraces(queryResult, columnHints, pagination); return (
{finalPage ? ( @@ -479,7 +483,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable(
{icon && } - {h} + {columnHints?.get(h)?.displayName ?? h} {hasFilterOnHeader(h, i) && }
@@ -490,6 +494,7 @@ export const GenericOutputTable = React.memo(function GenericOutputTable( accessor: String(i), Cell(row) { const value = row.value; + const formatter = columnHints?.get(h)?.formatter || formatNumber; return (
@@ -516,7 +521,8 @@ export const GenericOutputTable = React.memo(function GenericOutputTable( : undefined, }; }), - groupHints, + queryResult.header, + columnHints, )} /> )} diff --git a/web-console/src/views/explore-view/modules/table-react-module.tsx b/web-console/src/views/explore-view/modules/table-react-module.tsx index 2f20a44b1127..fcc06a09784f 100644 --- a/web-console/src/views/explore-view/modules/table-react-module.tsx +++ b/web-console/src/views/explore-view/modules/table-react-module.tsx @@ -16,11 +16,13 @@ * limitations under the License. */ -import type { SqlOrderByExpression } from '@druid-toolkit/query'; +import { Button } from '@blueprintjs/core'; +import type { SqlOrderByExpression, SqlTable } from '@druid-toolkit/query'; import { C, F, SqlCase, + SqlColumn, SqlExpression, SqlFunction, SqlLiteral, @@ -35,15 +37,23 @@ import ReactDOM from 'react-dom'; import { Loader } from '../../../components'; import { useQueryManager } from '../../../hooks'; +import type { ColumnHint } from '../../../utils'; +import { formatInteger, formatPercent } from '../../../utils'; import { getInitQuery } from '../utils'; import { GenericOutputTable } from './components'; -import { shiftTimeInWhere } from './utils/utils'; +import { getWhereForCompares, shiftTimeInExpression } from './utils/utils'; import './table-react-module.scss'; type MultipleValueMode = 'null' | 'empty' | 'latest' | 'latestNonNull' | 'count'; +type CompareType = 'value' | 'delta' | 'absDelta' | 'percent' | 'absPercent'; + +// As of this writing ordering the outer query on something other than __time sometimes throws an error, set this to false / remove it +// when ordering on non __time is more robust +const NEEDS_GROUPING_TO_ORDER = true; + const KNOWN_AGGREGATIONS = [ 'COUNT', 'SUM', @@ -73,13 +83,37 @@ const KNOWN_AGGREGATIONS = [ 'ANY_VALUE', ]; +const TOP_VALUES_NAME = 'top_values'; +const TOP_VALUES_K = 5000; + +function coalesce0(ex: SqlExpression) { + return F('COALESCE', ex, SqlLiteral.ZERO); +} + +function safeDivide0(a: SqlExpression, b: SqlExpression) { + return coalesce0(F('SAFE_DIVIDE', a, b)); +} + +function anyValue(ex: SqlExpression) { + return F('ANY_VALUE', ex); +} + +function addTableScope(expression: SqlExpression, newTableScope: string): SqlExpression { + return expression.walk(ex => { + if (ex instanceof SqlColumn && !ex.getTableName()) { + return ex.changeTableName(newTableScope); + } + return ex; + }) as SqlExpression; +} + function toGroupByExpression( splitColumn: ExpressionMeta, timeBucket: string, compareShiftDuration?: string, ) { const { expression, sqlType, name } = splitColumn; - return expression + return addTableScope(expression, 't') .applyIf(sqlType === 'TIMESTAMP' && compareShiftDuration, e => F.timeShift(e, compareShiftDuration!, 1), ) @@ -131,9 +165,21 @@ function toShowColumnExpression( return ex.as(showColumn.name); } +function getJoinCondition( + splitColumns: ExpressionMeta[], + table1: SqlTable, + table2: SqlTable, +): SqlExpression { + return SqlExpression.and( + ...splitColumns.map(splitColumn => + table1.column(splitColumn.name).isNotDistinctFrom(table2.column(splitColumn.name)), + ), + ); +} + interface QueryAndHints { query: SqlQuery; - groupHints: string[]; + columnHints: Map; } export default typedVisualModule({ @@ -200,13 +246,14 @@ export default typedVisualModule({ compares: { type: 'options', - options: ['PT1M', 'PT5M', 'PT1H', 'P1D', 'P1M'], + options: ['PT1M', 'PT5M', 'PT1H', 'PT6H', 'P1D', 'P1M'], control: { label: 'Compares', optionLabels: { PT1M: '1 minute', PT5M: '5 minutes', PT1H: '1 hour', + PT6H: '6 hours', P1D: '1 day', P1M: '1 month', }, @@ -214,12 +261,31 @@ export default typedVisualModule({ }, }, - showDelta: { + compareTypes: { + type: 'options', + options: ['value', 'delta', 'absDelta', 'percent', 'absPercent'], + default: ['value', 'delta'], + control: { + label: 'Compare types', + visible: ({ params }) => Boolean((params.compares || []).length) && !params.pivotColumn, + optionLabels: { + value: 'Value', + delta: 'Delta', + absDelta: 'Abs. delta', + percent: 'Percent', + absPercent: 'Abs. percent', + }, + }, + }, + restrictTop: { type: 'boolean', + default: true, control: { - visible: ({ params }) => Boolean((params.compares || []).length), + label: `Restrict to top ${formatInteger(TOP_VALUES_K)} when ordering on delta`, + visible: ({ params }) => Boolean((params.compares || []).length) && !params.pivotColumn, }, }, + maxRows: { type: 'number', default: 200, @@ -287,7 +353,7 @@ function TableModule(props: TableModuleProps) { }, }); - const queryAndHints = useMemo(() => { + const queryAndHints = useMemo((): QueryAndHints | undefined => { const splitColumns: ExpressionMeta[] = parameterValues.splitColumns; const timeBucket: string = parameterValues.timeBucket || 'PT1H'; const showColumns: ExpressionMeta[] = parameterValues.showColumns; @@ -295,26 +361,71 @@ function TableModule(props: TableModuleProps) { const pivotColumn: ExpressionMeta = parameterValues.pivotColumn; const metrics: ExpressionMeta[] = parameterValues.metrics; const compares: string[] = parameterValues.compares || []; - const showDelta: boolean = parameterValues.showDelta; + const compareTypes: CompareType[] = parameterValues.compareTypes; + const restrictTop: boolean = parameterValues.restrictTop; const maxRows: number = parameterValues.maxRows; const pivotValues = pivotColumn ? pivotValueState.data : undefined; if (pivotColumn && !pivotValues) return; - const hasCompare = Boolean(compares.length); + const effectiveOrderBy = + orderBy || C(metrics[0]?.name || splitColumns[0]?.name).toOrderByExpression('DESC'); + const hasCompare = !pivotColumn && Boolean(compares.length) && Boolean(compareTypes.length); + + const orderByColumnName = (effectiveOrderBy.expression as SqlColumn).getName(); + let orderByCompareMeasure: string | undefined; + let orderByCompareDuration: string | undefined; + let orderByCompareType: CompareType | undefined; + if (hasCompare) { + const m = orderByColumnName.match( + /^(.+):cmp:([^:]+):(value|delta|absDelta|percent|absPercent)$/, + ); + if (m) { + orderByCompareMeasure = m[1]; + orderByCompareDuration = m[2]; + orderByCompareType = m[3] as CompareType; + } + } + + const metricExpression = metrics.find(m => m.name === orderByCompareMeasure)?.expression; + const topValuesQuery = + restrictTop && metricExpression && orderByCompareType !== 'value' && splitColumns.length + ? getInitQuery(table, getWhereForCompares(where, compares)) + .applyForEach(splitColumns, (q, splitColumn) => + q.addSelect(toGroupByExpression(splitColumn, timeBucket), { + addToGroupBy: 'end', + }), + ) + .changeOrderByExpression(metricExpression.toOrderByExpression('DESC')) + .changeLimitValue(TOP_VALUES_K) + : undefined; + + const columnHints = new Map(); const mainQuery = getInitQuery(table, where) + .applyIf(topValuesQuery, q => + q.addInnerJoin( + T(TOP_VALUES_NAME), + getJoinCondition(splitColumns, T('t'), T(TOP_VALUES_NAME)), + ), + ) .applyForEach(splitColumns, (q, splitColumn) => q.addSelect(toGroupByExpression(splitColumn, timeBucket), { addToGroupBy: 'end', }), ) - .applyForEach(showColumns, (q, showColumn) => - q.addSelect(toShowColumnExpression(showColumn, multipleValueMode)), + .applyIf(!orderByCompareDuration, q => + q.applyForEach(showColumns, (q, showColumn) => + q.addSelect(toShowColumnExpression(showColumn, multipleValueMode)), + ), ) .applyForEach(pivotValues || [''], (q, pivotValue, i) => - q.applyForEach(metrics, (q, metric) => - q.addSelect( + q.applyForEach(metrics, (q, metric) => { + const alias = `${metric.name}${pivotColumn && i > 0 ? `:${pivotValue}` : ''}`; + if (pivotColumn) { + columnHints.set(alias, { displayName: metric.name, group: pivotValue }); + } + return q.addSelect( metric.expression .as(metric.name) .applyIf(pivotColumn, q => @@ -323,115 +434,204 @@ function TableModule(props: TableModuleProps) { pivotColumn.expression.equal(pivotValue), KNOWN_AGGREGATIONS, ) - .as(`${metric.name}${i > 0 ? ` [${pivotValue}]` : ''}`), + .as(alias), ), - ), - ), - ) - .applyIf(metrics.length > 0 || splitColumns.length > 0, q => - q.changeOrderByExpression( - orderBy || C(metrics[0]?.name || splitColumns[0]?.name).toOrderByExpression('DESC'), - ), + ); + }), ) - .changeLimitValue(maxRows); + .applyIf(!orderByCompareDuration, q => + q + .applyIf(metrics.length > 0 || splitColumns.length > 0, q => + q.changeOrderByExpression(effectiveOrderBy), + ) + .changeLimitValue(maxRows), + ); if (!hasCompare) { return { query: mainQuery, - groupHints: pivotColumn - ? splitColumns - .map(() => '') - .concat( - showColumns.map(() => ''), - (pivotValues || []).flatMap(v => metrics.map(() => v)), - ) - : [], + columnHints, }; } const main = T('main'); - return { - query: SqlQuery.from(main) - .changeWithParts( - [SqlWithPart.simple('main', mainQuery)].concat( - compares.map((comparePeriod, i) => - SqlWithPart.simple( - `compare${i}`, - getInitQuery(table, shiftTimeInWhere(where, comparePeriod)) - .applyForEach(splitColumns, (q, splitColumn) => - q.addSelect(toGroupByExpression(splitColumn, timeBucket, comparePeriod), { - addToGroupBy: 'end', - }), - ) - .applyForEach(metrics, (q, metric) => - q.addSelect(metric.expression.as(metric.name)), + const leader = T(orderByCompareDuration ? `compare_${orderByCompareDuration}` : 'main'); + const query = SqlQuery.from(leader) + .changeWithParts( + ( + (topValuesQuery + ? [SqlWithPart.simple(TOP_VALUES_NAME, topValuesQuery)] + : []) as SqlWithPart[] + ).concat( + SqlWithPart.simple('main', mainQuery), + compares.map(compare => + SqlWithPart.simple( + `compare_${compare}`, + getInitQuery(table, shiftTimeInExpression(where, compare)) + .applyIf(topValuesQuery, q => + q.addInnerJoin( + T(TOP_VALUES_NAME), + getJoinCondition(splitColumns, T('t'), T(TOP_VALUES_NAME)), ), - ), + ) + .applyForEach(splitColumns, (q, splitColumn) => + q.addSelect(toGroupByExpression(splitColumn, timeBucket, compare), { + addToGroupBy: 'end', + }), + ) + .applyIf(orderByCompareDuration === compare, q => + q.applyForEach(showColumns, (q, showColumn) => + q.addSelect(toShowColumnExpression(showColumn, multipleValueMode)), + ), + ) + .applyForEach(metrics, (q, metric) => + q.addSelect(metric.expression.as(metric.name)), + ) + .applyIf(compare === orderByCompareDuration && orderByCompareType === 'value', q => + q + .changeOrderByExpression( + effectiveOrderBy.changeExpression(C(orderByCompareMeasure!)), + ) + .changeLimitValue(maxRows), + ), ), ), - ) - .changeSelectExpressions( - splitColumns - .map(splitColumn => main.column(splitColumn.name).as(splitColumn.name)) - .concat( - showColumns.map(showColumn => main.column(showColumn.name).as(showColumn.name)), - metrics.map(metric => main.column(metric.name).as(metric.name)), - compares.flatMap((_, i) => - metrics.flatMap(metric => { - const c = T(`compare${i}`).column(metric.name); - - const ret = [SqlFunction.simple('COALESCE', [c, 0]).as(`#prev: ${metric.name}`)]; - - if (showDelta) { - ret.push( - F.stringFormat( - '%.1f%%', - SqlFunction.simple('SAFE_DIVIDE', [ - SqlExpression.parse(`(${main.column(metric.name)} - ${c}) * 100.0`), - c, - ]), - ).as(`%chg: ${metric.name}`), - ); - } - - return ret; - }), - ), + ), + ) + .changeSelectExpressions( + splitColumns + .map(splitColumn => main.column(splitColumn.name).as(splitColumn.name)) + .concat( + showColumns.map(showColumn => + leader + .column(showColumn.name) + .applyIf(NEEDS_GROUPING_TO_ORDER, anyValue) + .as(showColumn.name), ), - ) - .applyForEach(compares, (q, _comparePeriod, i) => - q.addLeftJoin( - T(`compare${i}`), - SqlExpression.and( - ...splitColumns.map(splitColumn => - main - .column(splitColumn.name) - .isNotDistinctFrom(T(`compare${i}`).column(splitColumn.name)), - ), + metrics.map(metric => + main + .column(metric.name) + .applyIf(NEEDS_GROUPING_TO_ORDER, anyValue) + .applyIf(orderByCompareDuration, coalesce0) + .as(metric.name), + ), + compares.flatMap(compare => + metrics.flatMap(metric => { + const c = T(`compare_${compare}`) + .column(metric.name) + .applyIf(NEEDS_GROUPING_TO_ORDER, anyValue) + .applyIf(compare !== orderByCompareDuration, coalesce0); + + const mainMetric = main + .column(metric.name) + .applyIf(NEEDS_GROUPING_TO_ORDER, anyValue) + .applyIf(orderByCompareDuration, coalesce0); + + const diff = mainMetric.subtract(c); + + const ret: SqlExpression[] = []; + + if (compareTypes.includes('value')) { + const valueName = `${metric.name}:cmp:${compare}:value`; + columnHints.set(valueName, { + group: `Comparison to ${compare}`, + displayName: `${metric.name} (value)`, + }); + ret.push(c.as(valueName)); + } + + if (compareTypes.includes('delta')) { + const deltaName = `${metric.name}:cmp:${compare}:delta`; + columnHints.set(deltaName, { + group: `Comparison to ${compare}`, + displayName: `${metric.name} (delta)`, + }); + ret.push(diff.as(deltaName)); + } + + if (compareTypes.includes('absDelta')) { + const deltaName = `${metric.name}:cmp:${compare}:absDelta`; + columnHints.set(deltaName, { + group: `Comparison to ${compare}`, + displayName: `${metric.name} (Abs. delta)`, + }); + ret.push(F('ABS', diff).as(deltaName)); + } + + if (compareTypes.includes('percent')) { + const percentName = `${metric.name}:cmp:${compare}:percent`; + columnHints.set(percentName, { + group: `Comparison to ${compare}`, + displayName: `${metric.name} (%)`, + formatter: formatPercent, + }); + ret.push( + safeDivide0(diff.multiply(SqlLiteral.ONE_POINT_ZERO), c).as(percentName), + ); + } + + if (compareTypes.includes('absPercent')) { + const percentName = `${metric.name}:cmp:${compare}:absPercent`; + columnHints.set(percentName, { + group: `Comparison to ${compare}`, + displayName: `${metric.name} (abs. %)`, + formatter: formatPercent, + }); + ret.push( + F('ABS', safeDivide0(diff.multiply(SqlLiteral.ONE_POINT_ZERO), c)).as( + percentName, + ), + ); + } + + return ret; + }), ), ), + ) + .applyIf(orderByCompareDuration, q => + q.addLeftJoin( + main, + getJoinCondition(splitColumns, main, T(`compare_${orderByCompareDuration}`)), ), - groupHints: splitColumns - .map(() => 'Current') - .concat( - showColumns.map(() => 'Current'), - metrics.map(() => 'Current'), - compares.flatMap(comparePeriod => - metrics - .flatMap(() => (showDelta ? ['', ''] : [''])) - .map(() => `Comparison to ${comparePeriod}`), + ) + .applyForEach( + compares.filter(c => c !== orderByCompareDuration), + (q, compare) => + q.addLeftJoin( + T(`compare_${compare}`), + getJoinCondition(splitColumns, main, T(`compare_${compare}`)), ), - ), + ) + .applyIf(NEEDS_GROUPING_TO_ORDER, q => + q.changeGroupByExpressions(splitColumns.map((_, i) => SqlLiteral.index(i))), + ) + .addOrderBy(effectiveOrderBy) + .changeLimitValue(maxRows); + + for (const splitColumn of splitColumns) { + columnHints.set(splitColumn.name, { group: 'Current' }); + } + for (const showColumn of showColumns) { + columnHints.set(showColumn.name, { group: 'Current' }); + } + for (const metric of metrics) { + columnHints.set(metric.name, { group: 'Current' }); + } + + return { + query, + columnHints, }; }, [table, where, parameterValues, orderBy, pivotValueState.data]); const [resultState] = useQueryManager({ query: queryAndHints, processQuery: async (queryAndHints: QueryAndHints) => { - const { query, groupHints } = queryAndHints; + const { query, columnHints } = queryAndHints; return { result: await sqlQuery(query), - groupHints, + columnHints, }; }, }); @@ -440,19 +640,24 @@ function TableModule(props: TableModuleProps) { return (
{resultState.error ? ( - resultState.getErrorMessage() +
+
{resultState.getErrorMessage()}
+ {resultState.getErrorMessage()?.includes('not found in any table') && orderBy && ( +
) : resultData ? ( { - const idx = SqlLiteral.index(headerIndex); - if (orderBy && String(orderBy.expression) === String(idx)) { + onOrderByChange={(columnName, desc) => { + const column = C(columnName); + if (orderBy && orderBy.expression.equals(column)) { setOrderBy(orderBy.reverseDirection()); } else { - setOrderBy(idx.toOrderByExpression(desc ? 'DESC' : 'ASC')); + setOrderBy(column.toOrderByExpression(desc ? 'DESC' : 'ASC')); } }} onQueryAction={action => { diff --git a/web-console/src/views/explore-view/modules/utils/utils.spec.ts b/web-console/src/views/explore-view/modules/utils/utils.spec.ts index 9db276e6a0b6..022d0f4a829a 100644 --- a/web-console/src/views/explore-view/modules/utils/utils.spec.ts +++ b/web-console/src/views/explore-view/modules/utils/utils.spec.ts @@ -18,21 +18,64 @@ import { SqlExpression } from '@druid-toolkit/query'; -import { shiftTimeInWhere } from './utils'; +import { getWhereForCompares, shiftTimeInExpression } from './utils'; -describe('shiftTimeInWhere', () => { - it('works with TIME_IN_INTERVAL', () => { +describe('getWhereForCompares', () => { + it('works', () => { expect( - shiftTimeInWhere( + getWhereForCompares( + SqlExpression.parse( + `TIME_IN_INTERVAL("__time", '2016-06-27/2016-06-28') AND "country" = 'United States'`, + ), + ['PT1H', 'P1D'], + ).toString(), + ).toEqual( + `(TIME_IN_INTERVAL("__time", '2016-06-27/2016-06-28') OR (TIME_SHIFT(TIMESTAMP '2016-06-27', 'PT1H', -1) <= "__time" AND "__time" < TIME_SHIFT(TIMESTAMP '2016-06-28', 'PT1H', -1)) OR (TIME_SHIFT(TIMESTAMP '2016-06-27', 'P1D', -1) <= "__time" AND "__time" < TIME_SHIFT(TIMESTAMP '2016-06-28', 'P1D', -1))) AND "country" = 'United States'`, + ); + }); +}); + +describe('shiftTimeInExpression', () => { + it('works with TIME_IN_INTERVAL (date)', () => { + expect( + shiftTimeInExpression( SqlExpression.parse(`TIME_IN_INTERVAL("__time", '2016-06-27/2016-06-28')`), 'P1D', ).toString(), - ).toEqual(`TIME_IN_INTERVAL(TIME_SHIFT("__time", 'P1D', 1), '2016-06-27/2016-06-28')`); + ).toEqual( + `TIME_SHIFT(TIMESTAMP '2016-06-27', 'P1D', -1) <= "__time" AND "__time" < TIME_SHIFT(TIMESTAMP '2016-06-28', 'P1D', -1)`, + ); + }); + + it('works with TIME_IN_INTERVAL (date and time)', () => { + expect( + shiftTimeInExpression( + SqlExpression.parse( + `TIME_IN_INTERVAL("__time", '2016-06-27T12:34:56/2016-06-28T12:34:56')`, + ), + 'P1D', + ).toString(), + ).toEqual( + `TIME_SHIFT(TIMESTAMP '2016-06-27 12:34:56', 'P1D', -1) <= "__time" AND "__time" < TIME_SHIFT(TIMESTAMP '2016-06-28 12:34:56', 'P1D', -1)`, + ); + }); + + it('works with TIME_IN_INTERVAL (date and time, zulu)', () => { + expect( + shiftTimeInExpression( + SqlExpression.parse( + `TIME_IN_INTERVAL("__time", '2016-06-27T12:34:56Z/2016-06-28T12:34:56Z')`, + ), + 'P1D', + ).toString(), + ).toEqual( + `TIME_SHIFT(TIME_PARSE('2016-06-27 12:34:56', NULL, 'Etc/UTC'), 'P1D', -1) <= "__time" AND "__time" < TIME_SHIFT(TIME_PARSE('2016-06-28 12:34:56', NULL, 'Etc/UTC'), 'P1D', -1)`, + ); }); it('works with relative time', () => { expect( - shiftTimeInWhere( + shiftTimeInExpression( SqlExpression.parse( `(TIME_SHIFT(MAX_DATA_TIME(), 'PT1H', -1) <= "__time" AND "__time" < MAX_DATA_TIME())`, ), @@ -45,7 +88,7 @@ describe('shiftTimeInWhere', () => { it('works with relative time (specific timestamps)', () => { expect( - shiftTimeInWhere( + shiftTimeInExpression( SqlExpression.parse( `TIMESTAMP '2016-06-27 20:31:02.498' <= "__time" AND "__time" < TIMESTAMP '2016-06-27 21:31:02.498'`, ), diff --git a/web-console/src/views/explore-view/modules/utils/utils.ts b/web-console/src/views/explore-view/modules/utils/utils.ts index 7320ca52404c..2783585d4fe2 100644 --- a/web-console/src/views/explore-view/modules/utils/utils.ts +++ b/web-console/src/views/explore-view/modules/utils/utils.ts @@ -16,28 +16,77 @@ * limitations under the License. */ -import type { SqlExpression } from '@druid-toolkit/query'; -import { F, SqlFunction, SqlLiteral } from '@druid-toolkit/query'; +import { F, SqlExpression, SqlFunction, SqlLiteral } from '@druid-toolkit/query'; -export function shiftTimeInWhere(where: SqlExpression, period: string): SqlExpression { - return where.walk(ex => { +import { partition } from '../../../../utils'; + +const IS_DATE_LIKE = /^[+-]?\d\d\d\d[^']+$/; + +function isoStringToTimestampLiteral(iso: string): SqlExpression { + const zulu = iso.endsWith('Z'); + const cleanIso = iso.replace('T', ' ').replace('Z', ''); + let sql: string; + if (zulu) { + sql = `TIME_PARSE('${cleanIso}', NULL, 'Etc/UTC')`; + } else { + sql = `TIMESTAMP '${cleanIso}'`; + } + return SqlExpression.parse(sql); +} + +export function getWhereForCompares(where: SqlExpression, compares: string[]): SqlExpression { + const whereParts = where.decomposeViaAnd({ flatten: true }); + const [timeExpressions, timelessExpressions] = partition(whereParts, expressionUsesTime); + return SqlExpression.and( + SqlExpression.or( + SqlExpression.and(...timeExpressions), + ...compares.map(compare => + SqlExpression.and( + ...timeExpressions.map(timeExpression => shiftTimeInExpression(timeExpression, compare)), + ), + ), + ), + ...timelessExpressions, + ); +} + +function expressionUsesTime(expression: SqlExpression): boolean { + return shiftTimeInExpression(expression, 'P1D') !== expression; +} + +export function shiftTimeInExpression(expression: SqlExpression, compare: string): SqlExpression { + return expression.walk(ex => { if (ex instanceof SqlLiteral) { // Works with: __time < TIMESTAMP '2022-01-02 03:04:05' if (ex.isDate()) { - return F('TIME_SHIFT', ex, period, -1); + return F.timeShift(ex, compare, -1); } } else if (ex instanceof SqlFunction) { const effectiveFunctionName = ex.getEffectiveFunctionName(); // Works with: TIME_IN_INTERVAL(__time, '') if (effectiveFunctionName === 'TIME_IN_INTERVAL') { - return ex.changeArgs(ex.args!.change(0, F('TIME_SHIFT', ex.getArg(0), period, 1))); + // Ideally we could rewrite it to TIME_IN_INTERVAL(TIME_SHIFT(__time, period, 1), '') but that would be slow in the current Druid + // return ex.changeArgs(ex.args!.change(0, F('TIME_SHIFT', ex.getArg(0), period, 1)));a + + const interval = ex.getArgAsString(1); + if (!interval) return ex; + + const [start, end] = interval.split('/'); + if (!IS_DATE_LIKE.test(start) || !IS_DATE_LIKE.test(end)) return ex; + + const t = ex.getArg(0); + if (!t) return ex; + + return F.timeShift(isoStringToTimestampLiteral(start), compare, -1) + .lessThanOrEqual(t) + .and(t.lessThan(F.timeShift(isoStringToTimestampLiteral(end), compare, -1))); } // Works with: TIME_SHIFT(...) <= __time // and: __time < MAX_DATA_TIME() if (effectiveFunctionName === 'TIME_SHIFT' || effectiveFunctionName === 'MAX_DATA_TIME') { - return F('TIME_SHIFT', ex, period, -1); + return F.timeShift(ex, compare, -1); } } diff --git a/web-console/src/views/explore-view/utils/misc.ts b/web-console/src/views/explore-view/utils/misc.ts index 69946dd99d5f..05185264387b 100644 --- a/web-console/src/views/explore-view/utils/misc.ts +++ b/web-console/src/views/explore-view/utils/misc.ts @@ -34,7 +34,7 @@ export function toggle(xs: readonly T[], x: T, eq?: (a: T, b: T) => boolean): } export function getInitQuery(table: SqlExpression, where: SqlExpression): SqlQuery { - return SqlQuery.from(table).applyIf(String(where) !== 'TRUE', q => + return SqlQuery.from(table.as('t')).applyIf(String(where) !== 'TRUE', q => q.changeWhereExpression(where), ); } diff --git a/web-console/src/views/lookups-view/lookups-view.tsx b/web-console/src/views/lookups-view/lookups-view.tsx index af8207f6ab11..caeee7d34663 100644 --- a/web-console/src/views/lookups-view/lookups-view.tsx +++ b/web-console/src/views/lookups-view/lookups-view.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import { Button, Icon, Intent } from '@blueprintjs/core'; +import { Button, Icon, Intent, Tag } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import React from 'react'; import type { Filter } from 'react-table'; @@ -295,8 +295,16 @@ export class LookupsView extends React.PureComponent + Lookup {deleteLookupName} was deleted + + } + failText={ + <> + Could not delete lookup {deleteLookupName} + + } intent={Intent.DANGER} onClose={() => { this.setState({ deleteLookupTier: undefined, deleteLookupName: undefined }); diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index 3ff6eead2764..52de53f80de6 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -16,7 +16,7 @@ * limitations under the License. */ -import { Button, ButtonGroup, Intent, Label, MenuItem } from '@blueprintjs/core'; +import { Button, ButtonGroup, Intent, Label, MenuItem, Tag } from '@blueprintjs/core'; import { IconNames } from '@blueprintjs/icons'; import { sum } from 'd3-array'; import React from 'react'; @@ -699,8 +699,16 @@ ORDER BY return resp.data; }} confirmButtonText="Disable worker" - successText="Worker has been disabled" - failText="Could not disable worker" + successText={ + <> + Worker {middleManagerDisableWorkerHost} has been disabled + + } + failText={ + <> + Could not disable worker {middleManagerDisableWorkerHost} + + } intent={Intent.DANGER} onClose={() => { this.setState({ middleManagerDisableWorkerHost: undefined }); diff --git a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx index a674a3e6028a..12e127164269 100644 --- a/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx +++ b/web-console/src/views/sql-data-loader-view/schema-step/preview-table/preview-table.tsx @@ -103,7 +103,7 @@ export const PreviewTable = React.memo(function PreviewTable(props: PreviewTable ); } - const numericColumnBraces = getNumericColumnBraces(queryResult); + const numericColumnBraces = getNumericColumnBraces(queryResult, undefined, undefined); return (
String(startRowColumn.row), ); - if (found.length <= 1) return []; // Do not highlight a single query or no queries + if (!found.length) return []; // Do not report the first query if it is basically the main query minus whitespace const firstQuery = found[0].sql; diff --git a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx index 492767b3eaf8..294aeeeb7981 100644 --- a/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx +++ b/web-console/src/views/workbench-view/result-table-pane/result-table-pane.tsx @@ -546,7 +546,7 @@ export const ResultTablePane = React.memo(function ResultTablePane(props: Result ? parsedQuery.getSelectExpressionForIndex(editingColumn) : undefined; - const numericColumnBraces = getNumericColumnBraces(queryResult, pagination); + const numericColumnBraces = getNumericColumnBraces(queryResult, undefined, pagination); return (
{finalPage ? (