diff --git a/.changeset/honest-mice-applaud.md b/.changeset/honest-mice-applaud.md new file mode 100644 index 000000000..6d0dbb5e5 --- /dev/null +++ b/.changeset/honest-mice-applaud.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Add custom trace-level attributes above trace waterfall diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 078e63b37..a1f5c3ded 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -66,6 +66,9 @@ export const Source = mongoose.model( statusCodeExpression: String, statusMessageExpression: String, spanEventsValueExpression: String, + highlightedTraceAttributeExpressions: { + type: mongoose.Schema.Types.Array, + }, metricTables: { type: { diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 07a76ebb6..80ce1ce08 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -34,6 +34,7 @@ import { DisplayType, Filter, SourceKind, + TSource, } from '@hyperdx/common-utils/dist/types'; import { ActionIcon, @@ -1091,21 +1092,32 @@ function DBSearchPage() { ({ where, whereLanguage, + source, }: { where: SearchConfig['where']; whereLanguage: SearchConfig['whereLanguage']; + source?: TSource; }) => { const qParams = new URLSearchParams({ - where: where || searchedConfig.where || '', whereLanguage: whereLanguage || 'sql', from: searchedTimeRange[0].getTime().toString(), to: searchedTimeRange[1].getTime().toString(), - select: searchedConfig.select || '', - source: searchedSource?.id || '', - filters: JSON.stringify(searchedConfig.filters ?? []), isLive: 'false', liveInterval: interval.toString(), }); + + // When generating a search based on a different source, + // filters and select for the current source are not preserved. + if (source && source.id !== searchedSource?.id) { + qParams.append('where', where || ''); + qParams.append('source', source.id); + } else { + qParams.append('select', searchedConfig.select || ''); + qParams.append('where', where || searchedConfig.where || ''); + qParams.append('filters', JSON.stringify(searchedConfig.filters ?? [])); + qParams.append('source', searchedSource?.id || ''); + } + return `/search?${qParams.toString()}`; }, [ @@ -1829,6 +1841,7 @@ function DBSearchPage() { dbSqlRowTableConfig, isChildModalOpen: isDrawerChildModalOpen, setChildModalOpen: setDrawerChildModalOpen, + source: searchedSource, }} config={dbSqlRowTableConfig} sourceId={searchedConfig.source} diff --git a/packages/app/src/components/DBHighlightedAttributesList.tsx b/packages/app/src/components/DBHighlightedAttributesList.tsx new file mode 100644 index 000000000..92f36fe9e --- /dev/null +++ b/packages/app/src/components/DBHighlightedAttributesList.tsx @@ -0,0 +1,67 @@ +import { useContext, useMemo } from 'react'; +import { TSource } from '@hyperdx/common-utils/dist/types'; +import { Flex } from '@mantine/core'; + +import { RowSidePanelContext } from './DBRowSidePanel'; +import EventTag from './EventTag'; + +export type HighlightedAttribute = { + source: TSource; + displayedKey: string; + value: string; + sql: string; + lucene?: string; +}; + +export function DBHighlightedAttributesList({ + attributes = [], +}: { + attributes: HighlightedAttribute[]; +}) { + const { + onPropertyAddClick, + generateSearchUrl, + source: contextSource, + } = useContext(RowSidePanelContext); + + const sortedAttributes = useMemo(() => { + return attributes.sort( + (a, b) => + a.displayedKey.localeCompare(b.displayedKey) || + a.value.localeCompare(b.value), + ); + }, [attributes]); + + return ( + + {sortedAttributes.map(({ displayedKey, value, sql, lucene, source }) => ( + + generateSearchUrl({ + where: query || '', + whereLanguage: queryLanguage ?? 'lucene', + source, + }) + : undefined + } + /> + ))} + + ); +} diff --git a/packages/app/src/components/DBRowOverviewPanel.tsx b/packages/app/src/components/DBRowOverviewPanel.tsx index 4b57956ec..e9199b223 100644 --- a/packages/app/src/components/DBRowOverviewPanel.tsx +++ b/packages/app/src/components/DBRowOverviewPanel.tsx @@ -81,11 +81,11 @@ export function RowOverviewPanel({ : {}; const _generateSearchUrl = useCallback( - (query?: string, timeRange?: [Date, Date]) => { + (query?: string, queryLanguage?: 'sql' | 'lucene') => { return ( generateSearchUrl?.({ where: query, - whereLanguage: 'lucene', + whereLanguage: queryLanguage, }) ?? '/' ); }, @@ -195,8 +195,6 @@ export function RowOverviewPanel({ diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx index 575b8922e..6940d4cef 100644 --- a/packages/app/src/components/DBRowSidePanel.tsx +++ b/packages/app/src/components/DBRowSidePanel.tsx @@ -46,9 +46,11 @@ export type RowSidePanelContextProps = { generateSearchUrl?: ({ where, whereLanguage, + source, }: { where: SearchConfig['where']; whereLanguage: SearchConfig['whereLanguage']; + source?: TSource; }) => string; generateChartUrl?: (config: { aggFn: string; @@ -61,6 +63,7 @@ export type RowSidePanelContextProps = { dbSqlRowTableConfig?: ChartConfigWithDateRange; isChildModalOpen?: boolean; setChildModalOpen?: (open: boolean) => void; + source?: TSource; }; export const RowSidePanelContext = createContext({}); diff --git a/packages/app/src/components/DBRowSidePanelHeader.tsx b/packages/app/src/components/DBRowSidePanelHeader.tsx index 6b9c13102..10c488513 100644 --- a/packages/app/src/components/DBRowSidePanelHeader.tsx +++ b/packages/app/src/components/DBRowSidePanelHeader.tsx @@ -175,11 +175,11 @@ export default function DBRowSidePanelHeader({ const maxBoxHeight = 120; const _generateSearchUrl = useCallback( - (query?: string, timeRange?: [Date, Date]) => { + (query?: string, queryLanguage?: 'sql' | 'lucene') => { return ( generateSearchUrl?.({ where: query, - whereLanguage: 'lucene', + whereLanguage: queryLanguage, }) ?? '/' ); }, diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx index ebae6f6e7..b101caa90 100644 --- a/packages/app/src/components/DBTraceWaterfallChart.tsx +++ b/packages/app/src/components/DBTraceWaterfallChart.tsx @@ -19,6 +19,12 @@ import { getSpanEventBody, } from '@/source'; import TimelineChart from '@/TimelineChart'; +import { + getHighlightedAttributesFromData, + getSelectExpressionsForHighlightedAttributes, +} from '@/utils/highlightedAttributes'; + +import { DBHighlightedAttributesList } from './DBHighlightedAttributesList'; import styles from '@/../styles/LogSidePanel.module.scss'; import resizeStyles from '@/../styles/ResizablePanel.module.scss'; @@ -68,7 +74,7 @@ function getTableBody(tableModel: TSource) { } function getConfig(source: TSource, traceId: string) { - const alias = { + const alias: Record = { Body: getTableBody(source), Timestamp: getDisplayedTimestampValueExpression(source), Duration: source.durationExpression @@ -82,6 +88,17 @@ function getConfig(source: TSource, traceId: string) { SeverityText: source.severityTextExpression ?? '', SpanAttributes: source.eventAttributesExpression ?? '', }; + + // Aliases for trace attributes must be added here to ensure + // the returned `alias` object includes them and useRowWhere works. + if (source.highlightedTraceAttributeExpressions) { + for (const expr of source.highlightedTraceAttributeExpressions) { + if (expr.alias) { + alias[expr.alias] = expr.sqlExpression; + } + } + } + const select = [ { valueExpression: alias.Body, @@ -105,6 +122,14 @@ function getConfig(source: TSource, traceId: string) { : []), ]; + if (source.kind === SourceKind.Trace || source.kind === SourceKind.Log) { + select.push( + ...getSelectExpressionsForHighlightedAttributes( + source.highlightedTraceAttributeExpressions, + ), + ); + } + if (source.kind === SourceKind.Trace) { select.push( ...[ @@ -177,7 +202,7 @@ export function useEventsData({ dateRange, dateRangeStartInclusive, }; - }, [config, dateRange]); + }, [config, dateRange, dateRangeStartInclusive]); return useOffsetPaginatedQuery(query, { enabled }); } @@ -237,6 +262,7 @@ export function useEventsAroundFocus({ return { rows, + meta, isFetching, }; } @@ -267,28 +293,36 @@ export function DBTraceWaterfallChartContainer({ }) { const { size, startResize } = useResizable(30, 'bottom'); - const { rows: traceRowsData, isFetching: traceIsFetching } = - useEventsAroundFocus({ - tableSource: traceTableSource, - focusDate, - dateRange, - traceId, - enabled: true, - }); - const { rows: logRowsData, isFetching: logIsFetching } = useEventsAroundFocus( - { - // search data if logTableModel exist - // search invalid date range if no logTableModel(react hook need execute no matter what) - tableSource: logTableSource ? logTableSource : traceTableSource, - focusDate, - dateRange: logTableSource ? dateRange : [dateRange[1], dateRange[0]], // different query to prevent cache - traceId, - enabled: logTableSource ? true : false, // disable fire query if logSource is not exist - }, - ); + const { + rows: traceRowsData, + isFetching: traceIsFetching, + meta: traceRowsMeta, + } = useEventsAroundFocus({ + tableSource: traceTableSource, + focusDate, + dateRange, + traceId, + enabled: true, + }); + const { + rows: logRowsData, + isFetching: logIsFetching, + meta: logRowsMeta, + } = useEventsAroundFocus({ + // search data if logTableModel exist + // search invalid date range if no logTableModel(react hook need execute no matter what) + tableSource: logTableSource ? logTableSource : traceTableSource, + focusDate, + dateRange: logTableSource ? dateRange : [dateRange[1], dateRange[0]], // different query to prevent cache + traceId, + enabled: logTableSource ? true : false, // disable fire query if logSource is not exist + }); const isFetching = traceIsFetching || logIsFetching; - const rows: any[] = [...traceRowsData, ...logRowsData]; + const rows: any[] = useMemo( + () => [...traceRowsData, ...logRowsData], + [traceRowsData, logRowsData], + ); rows.sort((a, b) => { const aDate = TimestampNano.fromString(a.Timestamp); @@ -301,6 +335,39 @@ export function DBTraceWaterfallChartContainer({ } }); + const highlightedAttributeValues = useMemo(() => { + const attributes = getHighlightedAttributesFromData( + traceTableSource, + traceTableSource.highlightedTraceAttributeExpressions, + traceRowsData, + traceRowsMeta, + ); + + if (logTableSource && logRowsData && logRowsMeta) { + attributes.push( + ...getHighlightedAttributesFromData( + logTableSource, + logTableSource.highlightedTraceAttributeExpressions, + logRowsData, + logRowsMeta, + ), + ); + } + + return attributes.sort( + (a, b) => + a.displayedKey.localeCompare(b.displayedKey) || + a.value.localeCompare(b.value), + ); + }, [ + traceTableSource, + traceRowsData, + traceRowsMeta, + logTableSource, + logRowsData, + logRowsMeta, + ]); + useEffect(() => { if (initialRowHighlightHint && onClick && highlightedRowWhere == null) { const initialRowHighlightIndex = rows.findIndex(row => { @@ -572,25 +639,30 @@ export function DBTraceWaterfallChartContainer({ An unknown error occurred. ) : ( - {}} - rowHeight={22} - labelWidth={300} - onClick={ts => { - // onTimeClick(ts + startedAt); - }} - onEventClick={event => { - onClick?.({ id: event.id, type: event.type ?? '' }); - }} - cursors={[]} - rows={timelineRows} - initialScrollRowIndex={initialScrollRowIndex} - /> + <> + + {}} + rowHeight={22} + labelWidth={300} + onClick={ts => { + // onTimeClick(ts + startedAt); + }} + onEventClick={event => { + onClick?.({ id: event.id, type: event.type ?? '' }); + }} + cursors={[]} + rows={timelineRows} + initialScrollRowIndex={initialScrollRowIndex} + /> + )} string; + generateSearchUrl?: ( + query?: string, + queryLanguage?: SearchConditionLanguage, + ) => string; } & ( | { sqlExpression: undefined; @@ -35,6 +44,11 @@ export default function EventTag({ ); } + const searchCondition = + nameLanguage === 'sql' + ? SqlString.format('? = ?', [SqlString.raw(name), value]) + : `${name}:${typeof value === 'string' ? `"${value}"` : value}`; + return ( diff --git a/packages/app/src/components/NetworkPropertyPanel.tsx b/packages/app/src/components/NetworkPropertyPanel.tsx index 4d41a1bb9..47870407a 100644 --- a/packages/app/src/components/NetworkPropertyPanel.tsx +++ b/packages/app/src/components/NetworkPropertyPanel.tsx @@ -25,8 +25,6 @@ import { CurlGenerator } from '@/utils/curlGenerator'; interface NetworkPropertyPanelProps { eventAttributes: Record; - onPropertyAddClick?: (key: string, value: string) => void; - generateSearchUrl: (query?: string, timeRange?: [Date, Date]) => string; } // https://github.com/reduxjs/redux-devtools/blob/f11383d294c1139081f119ef08aa1169bd2ad5ff/packages/react-json-tree/src/createStylingFromTheme.ts @@ -189,8 +187,6 @@ export const NetworkBody = ({ export function NetworkPropertySubpanel({ eventAttributes, - onPropertyAddClick, - generateSearchUrl, }: NetworkPropertyPanelProps) { const requestHeaders = useMemo( () => parseHeaders('http.request.header.', eventAttributes), diff --git a/packages/app/src/components/SourceForm.tsx b/packages/app/src/components/SourceForm.tsx index c5cad69c7..1bd103437 100644 --- a/packages/app/src/components/SourceForm.tsx +++ b/packages/app/src/components/SourceForm.tsx @@ -2,6 +2,7 @@ import React, { useCallback, useEffect, useState } from 'react'; import { Control, Controller, + useFieldArray, useForm, UseFormSetValue, UseFormWatch, @@ -15,11 +16,13 @@ import { TSourceUnion, } from '@hyperdx/common-utils/dist/types'; import { + ActionIcon, Anchor, Box, Button, Divider, Flex, + Grid, Group, Radio, Slider, @@ -28,10 +31,12 @@ import { Tooltip, } from '@mantine/core'; import { notifications } from '@mantine/notifications'; +import { IconTrash } from '@tabler/icons-react'; import { SourceSelectControlled } from '@/components/SourceSelect'; import { IS_METRICS_ENABLED, IS_SESSIONS_ENABLED } from '@/config'; import { useConnections } from '@/connection'; +import SearchInputV2 from '@/SearchInputV2'; import { inferTableSourceConfig, isValidMetricTable, @@ -102,33 +107,35 @@ function FormRow({ }) { return ( // - - - {typeof label === 'string' ? ( - - {label} - - ) : ( - label - )} - - - - - - + + + + {typeof label === 'string' ? ( + + {label} + + ) : ( + label + )} + + + + + + + + + {highlightedAttributes.map((field, index) => ( + + + + + + + AS + + removeHighlightedAttribute(index)} + > + + + + + + + + + + + + + + + + ))} + + + + ); +} + // traceModel= ... // logModel=.... // traceModel.logModel = 'custom' @@ -149,7 +259,8 @@ function FormRow({ // OR traceModel.logModel = 'log_id_blah' // custom always points towards the url param -export function LogTableModelForm({ control, watch }: TableModelProps) { +export function LogTableModelForm(props: TableModelProps) { + const { control, watch } = props; const databaseName = watch(`from.databaseName`, DEFAULT_DATABASE); const tableName = watch(`from.tableName`); const connectionId = watch(`connection`); @@ -377,12 +488,15 @@ export function LogTableModelForm({ control, watch }: TableModelProps) { placeholder="Body" /> + + ); } -export function TraceTableModelForm({ control, watch }: TableModelProps) { +export function TraceTableModelForm(props: TableModelProps) { + const { control, watch } = props; const databaseName = watch(`from.databaseName`, DEFAULT_DATABASE); const tableName = watch(`from.tableName`); const connectionId = watch(`connection`); @@ -643,6 +757,8 @@ export function TraceTableModelForm({ control, watch }: TableModelProps) { disableKeywordAutocomplete /> + + ); } diff --git a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx index 25b58235d..facb351db 100644 --- a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx +++ b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx @@ -7,6 +7,7 @@ import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; import useRowWhere from '@/hooks/useRowWhere'; import TimelineChart from '@/TimelineChart'; +import { RowSidePanelContext } from '../DBRowSidePanel'; import { DBTraceWaterfallChartContainer, SpanRow, @@ -25,6 +26,9 @@ jest.mock('@/TimelineChart', () => { jest.mock('@/hooks/useOffsetPaginatedQuery'); jest.mock('@/hooks/useRowWhere'); +jest.mock('../DBRowDataPanel', () => ({ + getJSONColumnNames: jest.fn().mockReturnValue([]), +})); const mockUseOffsetPaginatedQuery = useOffsetPaginatedQuery as jest.Mock; const mockUseRowWhere = useRowWhere as jest.Mock; @@ -112,13 +116,15 @@ describe('DBTraceWaterfallChartContainer', () => { logTableSource: typeof mockLogTableSource | null = mockLogTableSource, ) => { return renderWithMantine( - , + + + , ); }; diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 5ce1e649d..59021ead2 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -174,7 +174,7 @@ describe('processRowToWhereClause', () => { const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - "toJSONString(dynamic_field) = toJSONString(JSONExtract('\\\"quoted_value\\\"', 'Dynamic'))", + "toJSONString(dynamic_field) = coalesce(toJSONString(JSONExtract('\\\"quoted_value\\\"', 'Dynamic')), toJSONString('\\\"quoted_value\\\"'))", ); }); @@ -194,7 +194,7 @@ describe('processRowToWhereClause', () => { const row = { dynamic_field: '{\\"took\\":7, not a valid json' }; const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - "toJSONString(dynamic_field) = toJSONString(JSONExtract('{\\\\\\\"took\\\\\\\":7, not a valid json', 'Dynamic'))", + "toJSONString(dynamic_field) = coalesce(toJSONString(JSONExtract('{\\\\\\\"took\\\\\\\":7, not a valid json', 'Dynamic')), toJSONString('{\\\\\\\"took\\\\\\\":7, not a valid json'))", ); }); @@ -214,7 +214,7 @@ describe('processRowToWhereClause', () => { const row = { dynamic_field: "{'foo': {'bar': 'baz'}}" }; const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - "toJSONString(dynamic_field) = toJSONString(JSONExtract('{\\'foo\\': {\\'bar\\': \\'baz\\'}}', 'Dynamic'))", + "toJSONString(dynamic_field) = coalesce(toJSONString(JSONExtract('{\\'foo\\': {\\'bar\\': \\'baz\\'}}', 'Dynamic')), toJSONString('{\\'foo\\': {\\'bar\\': \\'baz\\'}}'))", ); }); @@ -234,7 +234,7 @@ describe('processRowToWhereClause', () => { const row = { dynamic_field: "['foo', 'bar']" }; const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - "toJSONString(dynamic_field) = toJSONString(JSONExtract('[\\'foo\\', \\'bar\\']', 'Dynamic'))", + "toJSONString(dynamic_field) = coalesce(toJSONString(JSONExtract('[\\'foo\\', \\'bar\\']', 'Dynamic')), toJSONString('[\\'foo\\', \\'bar\\']'))", ); }); diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index 20338c72e..3bfe503cc 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -65,7 +65,7 @@ export function processRowToWhereClause( // Handle case for json element, ex: json.c // Currently we can't distinguish null or 'null' - if (value === 'null') { + if (value == null || value === 'null') { return SqlString.format(`isNull(??)`, [valueExpr]); } if (value.length > 1000 || column.length > 1000) { @@ -75,10 +75,11 @@ export function processRowToWhereClause( // escaped strings needs raw, because sqlString will add another layer of escaping // data other than array/object will always return with double quote(because of CH) - // remove double quote to search correctly + // remove double quote to search correctly. + // The coalesce is to handle the case when JSONExtract returns null due to the value being a string. return SqlString.format( - "toJSONString(?) = toJSONString(JSONExtract(?, 'Dynamic'))", - [SqlString.raw(valueExpr), value], + "toJSONString(?) = coalesce(toJSONString(JSONExtract(?, 'Dynamic')), toJSONString(?))", + [SqlString.raw(valueExpr), value, value], ); default: diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index e62af7bd7..74f6f8965 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -127,10 +127,7 @@ export const useSearchPageFilterState = ({ const [filters, setFilters] = React.useState({}); React.useEffect(() => { - if ( - !areFiltersEqual(filters, parsedQuery.filters) && - Object.values(parsedQuery.filters).length > 0 - ) { + if (!areFiltersEqual(filters, parsedQuery.filters)) { setFilters(parsedQuery.filters); } // only react to changes in parsed query diff --git a/packages/app/src/utils/__tests__/highlightedAttributes.test.ts b/packages/app/src/utils/__tests__/highlightedAttributes.test.ts new file mode 100644 index 000000000..236cde572 --- /dev/null +++ b/packages/app/src/utils/__tests__/highlightedAttributes.test.ts @@ -0,0 +1,480 @@ +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; + +import { getHighlightedAttributesFromData } from '../highlightedAttributes'; + +describe('getHighlightedAttributesFromData', () => { + const createBasicSource = ( + highlightedTraceAttributeExpressions: any[] = [], + ): TSource => ({ + kind: SourceKind.Trace, + from: { + databaseName: 'default', + tableName: 'otel_traces', + }, + timestampValueExpression: 'Timestamp', + connection: 'test-connection', + name: 'Traces', + highlightedTraceAttributeExpressions, + id: 'test-source-id', + }); + + const basicMeta = [ + { name: 'Body', type: 'String' }, + { name: 'Timestamp', type: 'DateTime64(9)' }, + { name: 'method', type: 'String' }, + ]; + + it('extracts attributes from data correctly', () => { + const data: Record[] = [ + { + Body: 'POST', + Timestamp: '2025-11-12T21:27:00.053000000Z', + SpanId: 'a51d12055f2058b9', + ServiceName: 'hdx-oss-dev-api', + method: 'POST', + "SpanAttributes['http.host']": 'localhost:8123', + Duration: 0.020954166, + ParentSpanId: '013cca18a6e626a6', + StatusCode: 'Unset', + SpanAttributes: { + 'http.flavor': '1.1', + 'http.host': 'localhost:8123', + 'http.method': 'POST', + }, + type: 'trace', + }, + { + Body: 'POST', + Timestamp: '2025-11-12T21:27:00.053000000Z', + SpanId: 'a51d12055f2058b9', + ServiceName: 'hdx-oss-dev-api', + method: 'GET', + "SpanAttributes['http.host']": 'localhost:8123', + Duration: 0.020954166, + ParentSpanId: '013cca18a6e626a6', + StatusCode: 'Unset', + SpanAttributes: { + 'http.flavor': '1.1', + 'http.host': 'localhost:8123', + 'http.method': 'POST', + }, + type: 'trace', + }, + ]; + + const meta = [ + { + name: 'Body', + type: 'LowCardinality(String)', + }, + { + name: 'Timestamp', + type: 'DateTime64(9)', + }, + { + name: 'SpanId', + type: 'String', + }, + { + name: 'ServiceName', + type: 'LowCardinality(String)', + }, + { + name: 'method', + type: 'String', + }, + { + name: "SpanAttributes['http.host']", + type: 'String', + }, + { + name: 'Duration', + type: 'Float64', + }, + { + name: 'ParentSpanId', + type: 'String', + }, + { + name: 'StatusCode', + type: 'LowCardinality(String)', + }, + { + name: 'SpanAttributes', + type: 'Map(LowCardinality(String), String)', + }, + ]; + + const source: TSource = { + kind: SourceKind.Trace, + from: { + databaseName: 'default', + tableName: 'otel_traces', + }, + timestampValueExpression: 'Timestamp', + connection: '68dd82484f54641b08667893', + name: 'Traces', + displayedTimestampValueExpression: 'Timestamp', + implicitColumnExpression: 'SpanName', + serviceNameExpression: 'ServiceName', + bodyExpression: 'SpanName', + eventAttributesExpression: 'SpanAttributes', + resourceAttributesExpression: 'ResourceAttributes', + defaultTableSelectExpression: + 'Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName', + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + durationExpression: 'Duration', + durationPrecision: 9, + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + statusCodeExpression: 'StatusCode', + statusMessageExpression: 'StatusMessage', + sessionSourceId: '68dd82484f54641b0866789e', + logSourceId: '6900eed982d3b3dfeff12a29', + highlightedTraceAttributeExpressions: [ + { + sqlExpression: "SpanAttributes['http.method']", + alias: 'method', + }, + { + sqlExpression: "SpanAttributes['http.host']", + luceneExpression: 'SpanAttributes.http.host', + alias: '', + }, + ], + id: '68dd82484f54641b08667899', + }; + + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + meta, + ); + + expect(attributes).toHaveLength(3); + expect(attributes).toContainEqual({ + sql: "SpanAttributes['http.method']", + displayedKey: 'method', + value: 'POST', + source, + }); + expect(attributes).toContainEqual({ + sql: "SpanAttributes['http.method']", + displayedKey: 'method', + value: 'GET', + source, + }); + expect(attributes).toContainEqual({ + sql: "SpanAttributes['http.host']", + displayedKey: "SpanAttributes['http.host']", + value: 'localhost:8123', + lucene: 'SpanAttributes.http.host', + source, + }); + }); + + it('returns empty array when data is empty', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + [], + basicMeta, + ); + expect(attributes).toEqual([]); + }); + + it('returns empty array when highlightedTraceAttributeExpressions is undefined', () => { + const source = createBasicSource(); + const data = [{ method: 'POST' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + expect(attributes).toEqual([]); + }); + + it('returns empty array when highlightedTraceAttributeExpressions is empty', () => { + const source = createBasicSource([]); + const data = [{ method: 'POST' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + expect(attributes).toEqual([]); + }); + + it('filters out non-string values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + { sqlExpression: 'count', alias: 'count' }, + { sqlExpression: 'isActive', alias: 'isActive' }, + ]); + const data = [ + { + method: 'POST', // string - should be included + count: 123, // number - should be filtered out + isActive: true, // boolean - should be filtered out + }, + ]; + const meta = [ + { name: 'method', type: 'String' }, + { name: 'count', type: 'Int32' }, + { name: 'isActive', type: 'Bool' }, + ]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + meta, + ); + + expect(attributes).toHaveLength(1); + expect(attributes[0]).toEqual({ + displayedKey: 'method', + value: 'POST', + sql: 'method', + lucene: undefined, + source, + }); + }); + + it('deduplicates values from multiple rows', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [ + { method: 'POST' }, + { method: 'POST' }, // duplicate + { method: 'GET' }, + { method: 'POST' }, // duplicate + ]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('uses sqlExpression as displayedKey when alias is empty string', () => { + const source = createBasicSource([ + { sqlExpression: "SpanAttributes['http.host']", alias: '' }, + ]); + const data = [{ "SpanAttributes['http.host']": 'localhost:8080' }]; + const meta = [ + { name: "SpanAttributes['http.host']", type: 'String' }, + { name: 'SpanAttributes', type: 'JSON' }, + ]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + meta, + ); + + expect(attributes).toHaveLength(1); + expect(attributes[0].displayedKey).toBe("SpanAttributes['http.host']"); + }); + + it('uses sqlExpression as displayedKey when alias is not provided', () => { + const source = createBasicSource([ + { sqlExpression: 'ServiceName' } as any, // No alias + ]); + const data = [{ ServiceName: 'api-service' }]; + const meta = [{ name: 'ServiceName', type: 'String' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + meta, + ); + + expect(attributes).toHaveLength(1); + expect(attributes[0].displayedKey).toBe('ServiceName'); + }); + + it('includes lucene expression when provided', () => { + const source = createBasicSource([ + { + sqlExpression: "SpanAttributes['http.method']", + alias: 'method', + luceneExpression: 'http.method', + }, + ]); + const data = [{ method: 'POST' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(1); + expect(attributes[0].lucene).toBe('http.method'); + }); + + it('omits lucene when not provided', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(1); + expect(attributes[0].lucene).toBeUndefined(); + }); + + it('handles multiple attributes with different values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + { sqlExpression: 'status', alias: 'status' }, + ]); + const data = [ + { method: 'POST', status: '200' }, + { method: 'GET', status: '404' }, + ]; + const meta = [ + { name: 'method', type: 'String' }, + { name: 'status', type: 'String' }, + ]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + meta, + ); + + expect(attributes).toHaveLength(4); + expect( + attributes.filter(a => a.displayedKey === 'method').map(a => a.value), + ).toEqual(['POST', 'GET']); + expect( + attributes.filter(a => a.displayedKey === 'status').map(a => a.value), + ).toEqual(['200', '404']); + }); + + it('ignores rows with null attribute values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: null }, { method: 'GET' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('ignores rows with undefined attribute values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: undefined }, { method: 'GET' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('ignores rows with empty string values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: '' }, { method: 'GET' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('handles errors gracefully and returns empty array', () => { + // Create a source that will cause an error during iteration + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + + // Mock console.error to suppress error output during test + const consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => {}); + + // Create data that throws when accessed + const data = [ + Object.create( + {}, + { + method: { + get() { + throw new Error('Test error'); + }, + }, + }, + ), + ]; + + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toEqual([]); + expect(consoleErrorSpy).toHaveBeenCalledWith( + 'Error extracting attributes from data', + expect.any(Error), + ); + + consoleErrorSpy.mockRestore(); + }); + + it('includes source reference in each attribute', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: 'GET' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(2); + attributes.forEach(attr => { + expect(attr.source).toBe(source); + }); + }); +}); diff --git a/packages/app/src/utils/highlightedAttributes.ts b/packages/app/src/utils/highlightedAttributes.ts new file mode 100644 index 000000000..a0deff516 --- /dev/null +++ b/packages/app/src/utils/highlightedAttributes.ts @@ -0,0 +1,69 @@ +import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; +import { TSource } from '@hyperdx/common-utils/dist/types'; + +import { getJSONColumnNames } from '@/components/DBRowDataPanel'; + +export function getSelectExpressionsForHighlightedAttributes( + expressions: TSource['highlightedTraceAttributeExpressions'] = [], +) { + return expressions.map(({ sqlExpression, alias }) => ({ + valueExpression: sqlExpression, + alias: alias || sqlExpression, + })); +} + +export function getHighlightedAttributesFromData( + source: TSource, + attributes: TSource['highlightedTraceAttributeExpressions'] = [], + data: Record[], + meta: ResponseJSON['meta'], +) { + const attributeValuesByDisplayKey = new Map>(); + const sqlExpressionsByDisplayKey = new Map(); + const luceneExpressionsByDisplayKey = new Map(); + const jsonColumns = getJSONColumnNames(meta); + + try { + for (const row of data) { + for (const { sqlExpression, luceneExpression, alias } of attributes) { + const displayName = alias || sqlExpression; + + const isJsonExpression = jsonColumns.includes( + sqlExpression.split('.')[0], + ); + const sqlExpressionWithJSONSupport = isJsonExpression + ? `toString(${sqlExpression})` + : sqlExpression; + + sqlExpressionsByDisplayKey.set( + displayName, + sqlExpressionWithJSONSupport, + ); + if (luceneExpression) { + luceneExpressionsByDisplayKey.set(displayName, luceneExpression); + } + + const value = row[displayName]; + if (value && typeof value === 'string') { + if (!attributeValuesByDisplayKey.has(displayName)) { + attributeValuesByDisplayKey.set(displayName, new Set()); + } + attributeValuesByDisplayKey.get(displayName)!.add(value); + } + } + } + } catch (e) { + console.error('Error extracting attributes from data', e); + } + + return Array.from(attributeValuesByDisplayKey.entries()).flatMap( + ([key, values]) => + [...values].map(value => ({ + displayedKey: key, + value, + sql: sqlExpressionsByDisplayKey.get(key)!, + lucene: luceneExpressionsByDisplayKey.get(key), + source, + })), + ); +} diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index ef0d96c7f..fbbc78f33 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -548,6 +548,14 @@ const RequiredTimestampColumnSchema = z .string() .min(1, 'Timestamp Column is required'); +const HighlightedAttributeExpressionsSchema = z.array( + z.object({ + sqlExpression: z.string().min(1, 'Attribute SQL Expression is required'), + luceneExpression: z.string().optional(), + alias: z.string().optional(), + }), +); + // Log source form schema const LogSourceAugmentation = { kind: z.literal(SourceKind.Log), @@ -570,6 +578,8 @@ const LogSourceAugmentation = { implicitColumnExpression: z.string().optional(), uniqueRowIdExpression: z.string().optional(), tableFilterExpression: z.string().optional(), + highlightedTraceAttributeExpressions: + HighlightedAttributeExpressionsSchema.optional(), }; // Trace source form schema @@ -600,6 +610,8 @@ const TraceSourceAugmentation = { eventAttributesExpression: z.string().optional(), spanEventsValueExpression: z.string().optional(), implicitColumnExpression: z.string().optional(), + highlightedTraceAttributeExpressions: + HighlightedAttributeExpressionsSchema.optional(), }; // Session source form schema