From 744024c900545fd3fedf5a18d53c475d911ba3aa Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 14 Nov 2025 13:04:06 -0500 Subject: [PATCH 1/8] feat: Add custom trace-level attributes above trace waterfall --- .changeset/honest-mice-applaud.md | 7 + packages/api/src/models/source.ts | 3 + packages/app/src/DBSearchPage.tsx | 43 ++- .../app/src/components/DBRowOverviewPanel.tsx | 6 +- .../app/src/components/DBRowSidePanel.tsx | 3 + .../src/components/DBRowSidePanelHeader.tsx | 4 +- .../src/components/DBTraceWaterfallChart.tsx | 272 +++++++++++++++--- packages/app/src/components/EventTag.tsx | 21 +- .../src/components/NetworkPropertyPanel.tsx | 4 - packages/app/src/components/SourceForm.tsx | 159 ++++++++-- .../__tests__/DBTraceWaterfallChart.test.tsx | 170 ++++++++++- packages/app/src/hooks/useRowWhere.tsx | 9 +- packages/common-utils/src/types.ts | 12 + 13 files changed, 603 insertions(+), 110 deletions(-) create mode 100644 .changeset/honest-mice-applaud.md diff --git a/.changeset/honest-mice-applaud.md b/.changeset/honest-mice-applaud.md new file mode 100644 index 000000000..6d0dbb5e5 --- /dev/null +++ b/.changeset/honest-mice-applaud.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/common-utils": patch +"@hyperdx/api": patch +"@hyperdx/app": patch +--- + +feat: Add custom trace-level attributes above trace waterfall diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 078e63b37..6f2485355 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -66,6 +66,9 @@ export const Source = mongoose.model( statusCodeExpression: String, statusMessageExpression: String, spanEventsValueExpression: String, + highlightedAttributeExpressions: { + type: mongoose.Schema.Types.Array, + }, metricTables: { type: { diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 07a76ebb6..0cfcc8e2b 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -34,6 +34,7 @@ import { DisplayType, Filter, SourceKind, + TSource, } from '@hyperdx/common-utils/dist/types'; import { ActionIcon, @@ -1091,22 +1092,39 @@ function DBSearchPage() { ({ where, whereLanguage, + source, }: { where: SearchConfig['where']; whereLanguage: SearchConfig['whereLanguage']; + source?: TSource; }) => { - const qParams = new URLSearchParams({ - where: where || searchedConfig.where || '', - whereLanguage: whereLanguage || 'sql', - from: searchedTimeRange[0].getTime().toString(), - to: searchedTimeRange[1].getTime().toString(), - select: searchedConfig.select || '', - source: searchedSource?.id || '', - filters: JSON.stringify(searchedConfig.filters ?? []), - isLive: 'false', - liveInterval: interval.toString(), - }); - return `/search?${qParams.toString()}`; + // When generating a search based on a different source, + // filters and select for the current source are not preserved. + if (source && source.id !== searchedSource?.id) { + const qParams = new URLSearchParams({ + where: where || '', + whereLanguage: whereLanguage || 'sql', + from: searchedTimeRange[0].getTime().toString(), + to: searchedTimeRange[1].getTime().toString(), + source: source.id, + isLive: 'false', + liveInterval: interval.toString(), + }); + return `/search?${qParams.toString()}`; + } else { + const qParams = new URLSearchParams({ + where: where || searchedConfig.where || '', + whereLanguage: whereLanguage || 'sql', + from: searchedTimeRange[0].getTime().toString(), + to: searchedTimeRange[1].getTime().toString(), + select: searchedConfig.select || '', + source: searchedSource?.id || '', + filters: JSON.stringify(searchedConfig.filters ?? []), + isLive: 'false', + liveInterval: interval.toString(), + }); + return `/search?${qParams.toString()}`; + } }, [ interval, @@ -1829,6 +1847,7 @@ function DBSearchPage() { dbSqlRowTableConfig, isChildModalOpen: isDrawerChildModalOpen, setChildModalOpen: setDrawerChildModalOpen, + source: searchedSource, }} config={dbSqlRowTableConfig} sourceId={searchedConfig.source} diff --git a/packages/app/src/components/DBRowOverviewPanel.tsx b/packages/app/src/components/DBRowOverviewPanel.tsx index 4b57956ec..e9199b223 100644 --- a/packages/app/src/components/DBRowOverviewPanel.tsx +++ b/packages/app/src/components/DBRowOverviewPanel.tsx @@ -81,11 +81,11 @@ export function RowOverviewPanel({ : {}; const _generateSearchUrl = useCallback( - (query?: string, timeRange?: [Date, Date]) => { + (query?: string, queryLanguage?: 'sql' | 'lucene') => { return ( generateSearchUrl?.({ where: query, - whereLanguage: 'lucene', + whereLanguage: queryLanguage, }) ?? '/' ); }, @@ -195,8 +195,6 @@ export function RowOverviewPanel({ diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx index 6270f7fd1..6f902c417 100644 --- a/packages/app/src/components/DBRowSidePanel.tsx +++ b/packages/app/src/components/DBRowSidePanel.tsx @@ -47,9 +47,11 @@ export type RowSidePanelContextProps = { generateSearchUrl?: ({ where, whereLanguage, + source, }: { where: SearchConfig['where']; whereLanguage: SearchConfig['whereLanguage']; + source?: TSource; }) => string; generateChartUrl?: (config: { aggFn: string; @@ -62,6 +64,7 @@ export type RowSidePanelContextProps = { dbSqlRowTableConfig?: ChartConfigWithDateRange; isChildModalOpen?: boolean; setChildModalOpen?: (open: boolean) => void; + source?: TSource; }; export const RowSidePanelContext = createContext({}); diff --git a/packages/app/src/components/DBRowSidePanelHeader.tsx b/packages/app/src/components/DBRowSidePanelHeader.tsx index 6b9c13102..10c488513 100644 --- a/packages/app/src/components/DBRowSidePanelHeader.tsx +++ b/packages/app/src/components/DBRowSidePanelHeader.tsx @@ -175,11 +175,11 @@ export default function DBRowSidePanelHeader({ const maxBoxHeight = 120; const _generateSearchUrl = useCallback( - (query?: string, timeRange?: [Date, Date]) => { + (query?: string, queryLanguage?: 'sql' | 'lucene') => { return ( generateSearchUrl?.({ where: query, - whereLanguage: 'lucene', + whereLanguage: queryLanguage, }) ?? '/' ); }, diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx index ebae6f6e7..baf9822e6 100644 --- a/packages/app/src/components/DBTraceWaterfallChart.tsx +++ b/packages/app/src/components/DBTraceWaterfallChart.tsx @@ -1,13 +1,14 @@ -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useCallback, useContext, useEffect, useMemo, useState } from 'react'; import _ from 'lodash'; import TimestampNano from 'timestamp-nano'; +import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; import { ChartConfig, ChartConfigWithDateRange, SourceKind, TSource, } from '@hyperdx/common-utils/dist/types'; -import { Divider, Text } from '@mantine/core'; +import { Divider, Flex, Text } from '@mantine/core'; import { ContactSupportText } from '@/components/ContactSupportText'; import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; @@ -20,6 +21,10 @@ import { } from '@/source'; import TimelineChart from '@/TimelineChart'; +import { getJSONColumnNames } from './DBRowDataPanel'; +import { RowSidePanelContext } from './DBRowSidePanel'; +import EventTag from './EventTag'; + import styles from '@/../styles/LogSidePanel.module.scss'; import resizeStyles from '@/../styles/ResizablePanel.module.scss'; @@ -68,7 +73,7 @@ function getTableBody(tableModel: TSource) { } function getConfig(source: TSource, traceId: string) { - const alias = { + const alias: Record = { Body: getTableBody(source), Timestamp: getDisplayedTimestampValueExpression(source), Duration: source.durationExpression @@ -82,6 +87,17 @@ function getConfig(source: TSource, traceId: string) { SeverityText: source.severityTextExpression ?? '', SpanAttributes: source.eventAttributesExpression ?? '', }; + + // Aliases for trace attributes must be added here to ensure + // the returned `alias` object includes them and useRowWhere works. + if (source.highlightedAttributeExpressions) { + for (const expr of source.highlightedAttributeExpressions) { + if (expr.alias) { + alias[expr.alias] = expr.sqlExpression; + } + } + } + const select = [ { valueExpression: alias.Body, @@ -105,6 +121,17 @@ function getConfig(source: TSource, traceId: string) { : []), ]; + if (source.kind === SourceKind.Trace || source.kind === SourceKind.Log) { + select.push( + ...(source.highlightedAttributeExpressions ?? []).map( + ({ sqlExpression, alias }) => ({ + valueExpression: sqlExpression, + alias: alias || sqlExpression, + }), + ), + ); + } + if (source.kind === SourceKind.Trace) { select.push( ...[ @@ -177,7 +204,7 @@ export function useEventsData({ dateRange, dateRangeStartInclusive, }; - }, [config, dateRange]); + }, [config, dateRange, dateRangeStartInclusive]); return useOffsetPaginatedQuery(query, { enabled }); } @@ -237,10 +264,151 @@ export function useEventsAroundFocus({ return { rows, + meta, isFetching, }; } +export const getAttributesFromData = ( + source: TSource, + data: Record[], + meta: ResponseJSON['meta'], +) => { + const attributeValuesByDisplayKey = new Map>(); + const sqlExpressionsByDisplayKey = new Map(); + const luceneExpressionsByDisplayKey = new Map(); + const jsonColumns = getJSONColumnNames(meta); + + for (const row of data) { + for (const { + sqlExpression, + luceneExpression, + alias, + } of source.highlightedAttributeExpressions ?? []) { + const displayName = alias || sqlExpression; + + const isJsonExpression = jsonColumns.includes( + sqlExpression.split('.')[0], + ); + const sqlExpressionWithJSONSupport = isJsonExpression + ? `toString(${sqlExpression})` + : sqlExpression; + + sqlExpressionsByDisplayKey.set(displayName, sqlExpressionWithJSONSupport); + if (luceneExpression) { + luceneExpressionsByDisplayKey.set(displayName, luceneExpression); + } + + const value = row[displayName]; + if (value && typeof value === 'string') { + if (!attributeValuesByDisplayKey.has(displayName)) { + attributeValuesByDisplayKey.set(displayName, new Set()); + } + attributeValuesByDisplayKey.get(displayName)!.add(value); + } + } + } + + return Array.from(attributeValuesByDisplayKey.entries()).flatMap( + ([key, values]) => + [...values].map(value => ({ + displayedKey: key, + value, + sql: sqlExpressionsByDisplayKey.get(key)!, + lucene: luceneExpressionsByDisplayKey.get(key), + source, + })), + ); +}; + +interface DBTraceAttributesProps { + traceSource: TSource; + traceEventData: Record[]; + traceEventMeta: ResponseJSON['meta']; + logSource?: TSource | null; + logEventData?: Record[] | null; + logEventMeta?: ResponseJSON['meta']; +} + +export function DBTraceAttributes({ + traceSource, + traceEventData, + traceEventMeta, + logSource, + logEventData, + logEventMeta, +}: DBTraceAttributesProps) { + const { + onPropertyAddClick, + generateSearchUrl, + source: contextSource, + } = useContext(RowSidePanelContext); + + const attributeValues = useMemo(() => { + const attributes = getAttributesFromData( + traceSource, + traceEventData, + traceEventMeta, + ); + if (logSource && logEventData && logEventMeta) { + attributes.push( + ...getAttributesFromData(logSource, logEventData, logEventMeta), + ); + } + return attributes; + }, [ + traceSource, + traceEventData, + traceEventMeta, + logSource, + logEventData, + logEventMeta, + ]); + + const handleGenerateSearchUrl = useCallback( + ( + query: string | undefined, + queryLanguage: 'sql' | 'lucene' | undefined, + source: TSource, + ) => { + return ( + generateSearchUrl?.({ + where: query || '', + whereLanguage: queryLanguage ?? 'lucene', + source, + }) || '' + ); + }, + [generateSearchUrl], + ); + + return ( + + {attributeValues.map(({ displayedKey, value, sql, lucene, source }) => ( + + handleGenerateSearchUrl(query, queryLanguage, source) + } + /> + ))} + + ); +} + // TODO: Optimize with ts lookup tables export function DBTraceWaterfallChartContainer({ traceTableSource, @@ -267,28 +435,36 @@ export function DBTraceWaterfallChartContainer({ }) { const { size, startResize } = useResizable(30, 'bottom'); - const { rows: traceRowsData, isFetching: traceIsFetching } = - useEventsAroundFocus({ - tableSource: traceTableSource, - focusDate, - dateRange, - traceId, - enabled: true, - }); - const { rows: logRowsData, isFetching: logIsFetching } = useEventsAroundFocus( - { - // search data if logTableModel exist - // search invalid date range if no logTableModel(react hook need execute no matter what) - tableSource: logTableSource ? logTableSource : traceTableSource, - focusDate, - dateRange: logTableSource ? dateRange : [dateRange[1], dateRange[0]], // different query to prevent cache - traceId, - enabled: logTableSource ? true : false, // disable fire query if logSource is not exist - }, - ); + const { + rows: traceRowsData, + isFetching: traceIsFetching, + meta: traceRowsMeta, + } = useEventsAroundFocus({ + tableSource: traceTableSource, + focusDate, + dateRange, + traceId, + enabled: true, + }); + const { + rows: logRowsData, + isFetching: logIsFetching, + meta: logRowsMeta, + } = useEventsAroundFocus({ + // search data if logTableModel exist + // search invalid date range if no logTableModel(react hook need execute no matter what) + tableSource: logTableSource ? logTableSource : traceTableSource, + focusDate, + dateRange: logTableSource ? dateRange : [dateRange[1], dateRange[0]], // different query to prevent cache + traceId, + enabled: logTableSource ? true : false, // disable fire query if logSource is not exist + }); const isFetching = traceIsFetching || logIsFetching; - const rows: any[] = [...traceRowsData, ...logRowsData]; + const rows: any[] = useMemo( + () => [...traceRowsData, ...logRowsData], + [traceRowsData, logRowsData], + ); rows.sort((a, b) => { const aDate = TimestampNano.fromString(a.Timestamp); @@ -572,25 +748,35 @@ export function DBTraceWaterfallChartContainer({ An unknown error occurred. ) : ( - {}} - rowHeight={22} - labelWidth={300} - onClick={ts => { - // onTimeClick(ts + startedAt); - }} - onEventClick={event => { - onClick?.({ id: event.id, type: event.type ?? '' }); - }} - cursors={[]} - rows={timelineRows} - initialScrollRowIndex={initialScrollRowIndex} - /> + <> + + {}} + rowHeight={22} + labelWidth={300} + onClick={ts => { + // onTimeClick(ts + startedAt); + }} + onEventClick={event => { + onClick?.({ id: event.id, type: event.type ?? '' }); + }} + cursors={[]} + rows={timelineRows} + initialScrollRowIndex={initialScrollRowIndex} + /> + )} string; + generateSearchUrl?: ( + query?: string, + queryLanguage?: 'sql' | 'lucene', + ) => string; } & ( | { sqlExpression: undefined; @@ -35,6 +43,11 @@ export default function EventTag({ ); } + const searchCondition = + nameLanguage === 'sql' + ? SqlString.format('? = ?', [SqlString.raw(name), value]) + : `${name}:${typeof value === 'string' ? `"${value}"` : value}`; + return ( diff --git a/packages/app/src/components/NetworkPropertyPanel.tsx b/packages/app/src/components/NetworkPropertyPanel.tsx index 4d41a1bb9..47870407a 100644 --- a/packages/app/src/components/NetworkPropertyPanel.tsx +++ b/packages/app/src/components/NetworkPropertyPanel.tsx @@ -25,8 +25,6 @@ import { CurlGenerator } from '@/utils/curlGenerator'; interface NetworkPropertyPanelProps { eventAttributes: Record; - onPropertyAddClick?: (key: string, value: string) => void; - generateSearchUrl: (query?: string, timeRange?: [Date, Date]) => string; } // https://github.com/reduxjs/redux-devtools/blob/f11383d294c1139081f119ef08aa1169bd2ad5ff/packages/react-json-tree/src/createStylingFromTheme.ts @@ -189,8 +187,6 @@ export const NetworkBody = ({ export function NetworkPropertySubpanel({ eventAttributes, - onPropertyAddClick, - generateSearchUrl, }: NetworkPropertyPanelProps) { const requestHeaders = useMemo( () => parseHeaders('http.request.header.', eventAttributes), diff --git a/packages/app/src/components/SourceForm.tsx b/packages/app/src/components/SourceForm.tsx index c5cad69c7..1cbe45847 100644 --- a/packages/app/src/components/SourceForm.tsx +++ b/packages/app/src/components/SourceForm.tsx @@ -2,6 +2,7 @@ import React, { useCallback, useEffect, useState } from 'react'; import { Control, Controller, + useFieldArray, useForm, UseFormSetValue, UseFormWatch, @@ -15,11 +16,13 @@ import { TSourceUnion, } from '@hyperdx/common-utils/dist/types'; import { + ActionIcon, Anchor, Box, Button, Divider, Flex, + Grid, Group, Radio, Slider, @@ -28,10 +31,12 @@ import { Tooltip, } from '@mantine/core'; import { notifications } from '@mantine/notifications'; +import { IconTrash } from '@tabler/icons-react'; import { SourceSelectControlled } from '@/components/SourceSelect'; import { IS_METRICS_ENABLED, IS_SESSIONS_ENABLED } from '@/config'; import { useConnections } from '@/connection'; +import SearchInputV2 from '@/SearchInputV2'; import { inferTableSourceConfig, isValidMetricTable, @@ -102,33 +107,35 @@ function FormRow({ }) { return ( // - - - {typeof label === 'string' ? ( - - {label} - - ) : ( - label - )} - - - - - - + + + + {typeof label === 'string' ? ( + + {label} + + ) : ( + label + )} + + + + + + + + + {highlightedAttributes.map((field, index) => ( + + + + + + + AS + + removeHighlightedAttribute(index)} + > + + + + + + + + + ))} + + + + ); +} + // traceModel= ... // logModel=.... // traceModel.logModel = 'custom' @@ -149,7 +244,8 @@ function FormRow({ // OR traceModel.logModel = 'log_id_blah' // custom always points towards the url param -export function LogTableModelForm({ control, watch }: TableModelProps) { +export function LogTableModelForm(props: TableModelProps) { + const { control, watch } = props; const databaseName = watch(`from.databaseName`, DEFAULT_DATABASE); const tableName = watch(`from.tableName`); const connectionId = watch(`connection`); @@ -377,12 +473,15 @@ export function LogTableModelForm({ control, watch }: TableModelProps) { placeholder="Body" /> + + ); } -export function TraceTableModelForm({ control, watch }: TableModelProps) { +export function TraceTableModelForm(props: TableModelProps) { + const { control, watch } = props; const databaseName = watch(`from.databaseName`, DEFAULT_DATABASE); const tableName = watch(`from.tableName`); const connectionId = watch(`connection`); @@ -643,6 +742,8 @@ export function TraceTableModelForm({ control, watch }: TableModelProps) { disableKeywordAutocomplete /> + + ); } diff --git a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx index 25b58235d..5ac8ee14e 100644 --- a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx +++ b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx @@ -7,8 +7,10 @@ import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; import useRowWhere from '@/hooks/useRowWhere'; import TimelineChart from '@/TimelineChart'; +import { RowSidePanelContext } from '../DBRowSidePanel'; import { DBTraceWaterfallChartContainer, + getAttributesFromData, SpanRow, useEventsAroundFocus, } from '../DBTraceWaterfallChart'; @@ -25,6 +27,9 @@ jest.mock('@/TimelineChart', () => { jest.mock('@/hooks/useOffsetPaginatedQuery'); jest.mock('@/hooks/useRowWhere'); +jest.mock('../DBRowDataPanel', () => ({ + getJSONColumnNames: jest.fn().mockReturnValue([]), +})); const mockUseOffsetPaginatedQuery = useOffsetPaginatedQuery as jest.Mock; const mockUseRowWhere = useRowWhere as jest.Mock; @@ -112,13 +117,15 @@ describe('DBTraceWaterfallChartContainer', () => { logTableSource: typeof mockLogTableSource | null = mockLogTableSource, ) => { return renderWithMantine( - , + + + , ); }; @@ -347,3 +354,152 @@ describe('useEventsAroundFocus', () => { expect(result.rows.length).toBe(0); }); }); + +describe('getAttributesFromData', () => { + it('extracts attributes from data correctly', () => { + const data: Record[] = [ + { + Body: 'POST', + Timestamp: '2025-11-12T21:27:00.053000000Z', + SpanId: 'a51d12055f2058b9', + ServiceName: 'hdx-oss-dev-api', + method: 'POST', + "SpanAttributes['http.host']": 'localhost:8123', + Duration: 0.020954166, + ParentSpanId: '013cca18a6e626a6', + StatusCode: 'Unset', + SpanAttributes: { + 'http.flavor': '1.1', + 'http.host': 'localhost:8123', + 'http.method': 'POST', + }, + type: 'trace', + }, + { + Body: 'POST', + Timestamp: '2025-11-12T21:27:00.053000000Z', + SpanId: 'a51d12055f2058b9', + ServiceName: 'hdx-oss-dev-api', + method: 'GET', + "SpanAttributes['http.host']": 'localhost:8123', + Duration: 0.020954166, + ParentSpanId: '013cca18a6e626a6', + StatusCode: 'Unset', + SpanAttributes: { + 'http.flavor': '1.1', + 'http.host': 'localhost:8123', + 'http.method': 'POST', + }, + type: 'trace', + }, + ]; + + const meta = [ + { + name: 'Body', + type: 'LowCardinality(String)', + }, + { + name: 'Timestamp', + type: 'DateTime64(9)', + }, + { + name: 'SpanId', + type: 'String', + }, + { + name: 'ServiceName', + type: 'LowCardinality(String)', + }, + { + name: 'method', + type: 'String', + }, + { + name: "SpanAttributes['http.host']", + type: 'String', + }, + { + name: 'Duration', + type: 'Float64', + }, + { + name: 'ParentSpanId', + type: 'String', + }, + { + name: 'StatusCode', + type: 'LowCardinality(String)', + }, + { + name: 'SpanAttributes', + type: 'Map(LowCardinality(String), String)', + }, + ]; + + const source: TSource = { + kind: SourceKind.Trace, + from: { + databaseName: 'default', + tableName: 'otel_traces', + }, + timestampValueExpression: 'Timestamp', + connection: '68dd82484f54641b08667893', + name: 'Traces', + displayedTimestampValueExpression: 'Timestamp', + implicitColumnExpression: 'SpanName', + serviceNameExpression: 'ServiceName', + bodyExpression: 'SpanName', + eventAttributesExpression: 'SpanAttributes', + resourceAttributesExpression: 'ResourceAttributes', + defaultTableSelectExpression: + 'Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName', + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + durationExpression: 'Duration', + durationPrecision: 9, + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + statusCodeExpression: 'StatusCode', + statusMessageExpression: 'StatusMessage', + sessionSourceId: '68dd82484f54641b0866789e', + logSourceId: '6900eed982d3b3dfeff12a29', + highlightedAttributeExpressions: [ + { + sqlExpression: "SpanAttributes['http.method']", + alias: 'method', + }, + { + sqlExpression: "SpanAttributes['http.host']", + luceneExpression: 'SpanAttributes.http.host', + alias: '', + }, + ], + id: '68dd82484f54641b08667899', + }; + + const attributes = getAttributesFromData(source, data, meta); + + expect(attributes).toHaveLength(3); + expect(attributes).toContainEqual({ + sql: "SpanAttributes['http.method']", + displayedKey: 'method', + value: 'POST', + source, + }); + expect(attributes).toContainEqual({ + sql: "SpanAttributes['http.method']", + displayedKey: 'method', + value: 'GET', + source, + }); + expect(attributes).toContainEqual({ + sql: "SpanAttributes['http.host']", + displayedKey: "SpanAttributes['http.host']", + value: 'localhost:8123', + lucene: 'SpanAttributes.http.host', + source, + }); + }); +}); diff --git a/packages/app/src/hooks/useRowWhere.tsx b/packages/app/src/hooks/useRowWhere.tsx index 20338c72e..3bfe503cc 100644 --- a/packages/app/src/hooks/useRowWhere.tsx +++ b/packages/app/src/hooks/useRowWhere.tsx @@ -65,7 +65,7 @@ export function processRowToWhereClause( // Handle case for json element, ex: json.c // Currently we can't distinguish null or 'null' - if (value === 'null') { + if (value == null || value === 'null') { return SqlString.format(`isNull(??)`, [valueExpr]); } if (value.length > 1000 || column.length > 1000) { @@ -75,10 +75,11 @@ export function processRowToWhereClause( // escaped strings needs raw, because sqlString will add another layer of escaping // data other than array/object will always return with double quote(because of CH) - // remove double quote to search correctly + // remove double quote to search correctly. + // The coalesce is to handle the case when JSONExtract returns null due to the value being a string. return SqlString.format( - "toJSONString(?) = toJSONString(JSONExtract(?, 'Dynamic'))", - [SqlString.raw(valueExpr), value], + "toJSONString(?) = coalesce(toJSONString(JSONExtract(?, 'Dynamic')), toJSONString(?))", + [SqlString.raw(valueExpr), value, value], ); default: diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index ef0d96c7f..ad0d64890 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -548,6 +548,14 @@ const RequiredTimestampColumnSchema = z .string() .min(1, 'Timestamp Column is required'); +const HighlightedAttributeExpressionsSchema = z.array( + z.object({ + sqlExpression: z.string().min(1, 'Attribute SQL Expression is required'), + luceneExpression: z.string().optional(), + alias: z.string().optional(), + }), +); + // Log source form schema const LogSourceAugmentation = { kind: z.literal(SourceKind.Log), @@ -570,6 +578,8 @@ const LogSourceAugmentation = { implicitColumnExpression: z.string().optional(), uniqueRowIdExpression: z.string().optional(), tableFilterExpression: z.string().optional(), + highlightedAttributeExpressions: + HighlightedAttributeExpressionsSchema.optional(), }; // Trace source form schema @@ -600,6 +610,8 @@ const TraceSourceAugmentation = { eventAttributesExpression: z.string().optional(), spanEventsValueExpression: z.string().optional(), implicitColumnExpression: z.string().optional(), + highlightedAttributeExpressions: + HighlightedAttributeExpressionsSchema.optional(), }; // Session source form schema From 617bd1664e03c27b73ef96089939ff220e025676 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 14 Nov 2025 14:33:53 -0500 Subject: [PATCH 2/8] fix: Clear filter state when URL filters param is removed --- packages/app/src/searchFilters.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx index e62af7bd7..74f6f8965 100644 --- a/packages/app/src/searchFilters.tsx +++ b/packages/app/src/searchFilters.tsx @@ -127,10 +127,7 @@ export const useSearchPageFilterState = ({ const [filters, setFilters] = React.useState({}); React.useEffect(() => { - if ( - !areFiltersEqual(filters, parsedQuery.filters) && - Object.values(parsedQuery.filters).length > 0 - ) { + if (!areFiltersEqual(filters, parsedQuery.filters)) { setFilters(parsedQuery.filters); } // only react to changes in parsed query From d895a3aff9f6dd2a587d864f2647247c40aa681a Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 14 Nov 2025 14:41:06 -0500 Subject: [PATCH 3/8] test: Fix tests --- packages/app/src/hooks/__tests__/useRowWhere.test.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx index 5ce1e649d..59021ead2 100644 --- a/packages/app/src/hooks/__tests__/useRowWhere.test.tsx +++ b/packages/app/src/hooks/__tests__/useRowWhere.test.tsx @@ -174,7 +174,7 @@ describe('processRowToWhereClause', () => { const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - "toJSONString(dynamic_field) = toJSONString(JSONExtract('\\\"quoted_value\\\"', 'Dynamic'))", + "toJSONString(dynamic_field) = coalesce(toJSONString(JSONExtract('\\\"quoted_value\\\"', 'Dynamic')), toJSONString('\\\"quoted_value\\\"'))", ); }); @@ -194,7 +194,7 @@ describe('processRowToWhereClause', () => { const row = { dynamic_field: '{\\"took\\":7, not a valid json' }; const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - "toJSONString(dynamic_field) = toJSONString(JSONExtract('{\\\\\\\"took\\\\\\\":7, not a valid json', 'Dynamic'))", + "toJSONString(dynamic_field) = coalesce(toJSONString(JSONExtract('{\\\\\\\"took\\\\\\\":7, not a valid json', 'Dynamic')), toJSONString('{\\\\\\\"took\\\\\\\":7, not a valid json'))", ); }); @@ -214,7 +214,7 @@ describe('processRowToWhereClause', () => { const row = { dynamic_field: "{'foo': {'bar': 'baz'}}" }; const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - "toJSONString(dynamic_field) = toJSONString(JSONExtract('{\\'foo\\': {\\'bar\\': \\'baz\\'}}', 'Dynamic'))", + "toJSONString(dynamic_field) = coalesce(toJSONString(JSONExtract('{\\'foo\\': {\\'bar\\': \\'baz\\'}}', 'Dynamic')), toJSONString('{\\'foo\\': {\\'bar\\': \\'baz\\'}}'))", ); }); @@ -234,7 +234,7 @@ describe('processRowToWhereClause', () => { const row = { dynamic_field: "['foo', 'bar']" }; const result = processRowToWhereClause(row, columnMap); expect(result).toBe( - "toJSONString(dynamic_field) = toJSONString(JSONExtract('[\\'foo\\', \\'bar\\']', 'Dynamic'))", + "toJSONString(dynamic_field) = coalesce(toJSONString(JSONExtract('[\\'foo\\', \\'bar\\']', 'Dynamic')), toJSONString('[\\'foo\\', \\'bar\\']'))", ); }); From ad3fdf331dc9142da904ca3873157b74349140b2 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 14 Nov 2025 15:13:14 -0500 Subject: [PATCH 4/8] feat: Improvements to the custom trace attributes --- .../src/components/DBTraceWaterfallChart.tsx | 89 ++++++++++--------- packages/app/src/components/SourceForm.tsx | 17 +++- 2 files changed, 62 insertions(+), 44 deletions(-) diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx index baf9822e6..72b53b48e 100644 --- a/packages/app/src/components/DBTraceWaterfallChart.tsx +++ b/packages/app/src/components/DBTraceWaterfallChart.tsx @@ -279,34 +279,41 @@ export const getAttributesFromData = ( const luceneExpressionsByDisplayKey = new Map(); const jsonColumns = getJSONColumnNames(meta); - for (const row of data) { - for (const { - sqlExpression, - luceneExpression, - alias, - } of source.highlightedAttributeExpressions ?? []) { - const displayName = alias || sqlExpression; - - const isJsonExpression = jsonColumns.includes( - sqlExpression.split('.')[0], - ); - const sqlExpressionWithJSONSupport = isJsonExpression - ? `toString(${sqlExpression})` - : sqlExpression; + try { + for (const row of data) { + for (const { + sqlExpression, + luceneExpression, + alias, + } of source.highlightedAttributeExpressions ?? []) { + const displayName = alias || sqlExpression; + + const isJsonExpression = jsonColumns.includes( + sqlExpression.split('.')[0], + ); + const sqlExpressionWithJSONSupport = isJsonExpression + ? `toString(${sqlExpression})` + : sqlExpression; - sqlExpressionsByDisplayKey.set(displayName, sqlExpressionWithJSONSupport); - if (luceneExpression) { - luceneExpressionsByDisplayKey.set(displayName, luceneExpression); - } + sqlExpressionsByDisplayKey.set( + displayName, + sqlExpressionWithJSONSupport, + ); + if (luceneExpression) { + luceneExpressionsByDisplayKey.set(displayName, luceneExpression); + } - const value = row[displayName]; - if (value && typeof value === 'string') { - if (!attributeValuesByDisplayKey.has(displayName)) { - attributeValuesByDisplayKey.set(displayName, new Set()); + const value = row[displayName]; + if (value && typeof value === 'string') { + if (!attributeValuesByDisplayKey.has(displayName)) { + attributeValuesByDisplayKey.set(displayName, new Set()); + } + attributeValuesByDisplayKey.get(displayName)!.add(value); } - attributeValuesByDisplayKey.get(displayName)!.add(value); } } + } catch (e) { + console.error('Error extracting attributes from data', e); } return Array.from(attributeValuesByDisplayKey.entries()).flatMap( @@ -350,12 +357,18 @@ export function DBTraceAttributes({ traceEventData, traceEventMeta, ); + if (logSource && logEventData && logEventMeta) { attributes.push( ...getAttributesFromData(logSource, logEventData, logEventMeta), ); } - return attributes; + + return attributes.sort( + (a, b) => + a.displayedKey.localeCompare(b.displayedKey) || + a.value.localeCompare(b.value), + ); }, [ traceSource, traceEventData, @@ -365,23 +378,6 @@ export function DBTraceAttributes({ logEventMeta, ]); - const handleGenerateSearchUrl = useCallback( - ( - query: string | undefined, - queryLanguage: 'sql' | 'lucene' | undefined, - source: TSource, - ) => { - return ( - generateSearchUrl?.({ - where: query || '', - whereLanguage: queryLanguage ?? 'lucene', - source, - }) || '' - ); - }, - [generateSearchUrl], - ); - return ( {attributeValues.map(({ displayedKey, value, sql, lucene, source }) => ( @@ -400,8 +396,15 @@ export function DBTraceAttributes({ onPropertyAddClick: undefined, sqlExpression: undefined, })} - generateSearchUrl={(query, queryLanguage) => - handleGenerateSearchUrl(query, queryLanguage, source) + generateSearchUrl={ + generateSearchUrl + ? (query, queryLanguage) => + generateSearchUrl({ + where: query || '', + whereLanguage: queryLanguage ?? 'lucene', + source, + }) + : undefined } /> ))} diff --git a/packages/app/src/components/SourceForm.tsx b/packages/app/src/components/SourceForm.tsx index 1cbe45847..5bdb7bcf9 100644 --- a/packages/app/src/components/SourceForm.tsx +++ b/packages/app/src/components/SourceForm.tsx @@ -209,9 +209,24 @@ function HighlightedAttributeExpressionsFormRow({ + + + + + + + ))} From f6e12c86fdd747856ba624a9694dee6f2cd6b142 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Fri, 14 Nov 2025 15:34:09 -0500 Subject: [PATCH 5/8] test: Add additional tests for getAttributesFromData --- .../__tests__/DBTraceWaterfallChart.test.tsx | 248 ++++++++++++++++++ 1 file changed, 248 insertions(+) diff --git a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx index 5ac8ee14e..8b6d21688 100644 --- a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx +++ b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx @@ -356,6 +356,27 @@ describe('useEventsAroundFocus', () => { }); describe('getAttributesFromData', () => { + const createBasicSource = ( + highlightedAttributeExpressions: any[] = [], + ): TSource => ({ + kind: SourceKind.Trace, + from: { + databaseName: 'default', + tableName: 'otel_traces', + }, + timestampValueExpression: 'Timestamp', + connection: 'test-connection', + name: 'Traces', + highlightedAttributeExpressions, + id: 'test-source-id', + }); + + const basicMeta = [ + { name: 'Body', type: 'String' }, + { name: 'Timestamp', type: 'DateTime64(9)' }, + { name: 'method', type: 'String' }, + ]; + it('extracts attributes from data correctly', () => { const data: Record[] = [ { @@ -502,4 +523,231 @@ describe('getAttributesFromData', () => { source, }); }); + + it('returns empty array when data is empty', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const attributes = getAttributesFromData(source, [], basicMeta); + expect(attributes).toEqual([]); + }); + + it('returns empty array when highlightedAttributeExpressions is undefined', () => { + const source = createBasicSource(); + const data = [{ method: 'POST' }]; + const attributes = getAttributesFromData(source, data, basicMeta); + expect(attributes).toEqual([]); + }); + + it('returns empty array when highlightedAttributeExpressions is empty', () => { + const source = createBasicSource([]); + const data = [{ method: 'POST' }]; + const attributes = getAttributesFromData(source, data, basicMeta); + expect(attributes).toEqual([]); + }); + + it('filters out non-string values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + { sqlExpression: 'count', alias: 'count' }, + { sqlExpression: 'isActive', alias: 'isActive' }, + ]); + const data = [ + { + method: 'POST', // string - should be included + count: 123, // number - should be filtered out + isActive: true, // boolean - should be filtered out + }, + ]; + const meta = [ + { name: 'method', type: 'String' }, + { name: 'count', type: 'Int32' }, + { name: 'isActive', type: 'Bool' }, + ]; + const attributes = getAttributesFromData(source, data, meta); + + expect(attributes).toHaveLength(1); + expect(attributes[0]).toEqual({ + displayedKey: 'method', + value: 'POST', + sql: 'method', + lucene: undefined, + source, + }); + }); + + it('deduplicates values from multiple rows', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [ + { method: 'POST' }, + { method: 'POST' }, // duplicate + { method: 'GET' }, + { method: 'POST' }, // duplicate + ]; + const attributes = getAttributesFromData(source, data, basicMeta); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('uses sqlExpression as displayedKey when alias is empty string', () => { + const source = createBasicSource([ + { sqlExpression: "SpanAttributes['http.host']", alias: '' }, + ]); + const data = [{ "SpanAttributes['http.host']": 'localhost:8080' }]; + const meta = [ + { name: "SpanAttributes['http.host']", type: 'String' }, + { name: 'SpanAttributes', type: 'JSON' }, + ]; + const attributes = getAttributesFromData(source, data, meta); + + expect(attributes).toHaveLength(1); + expect(attributes[0].displayedKey).toBe("SpanAttributes['http.host']"); + }); + + it('uses sqlExpression as displayedKey when alias is not provided', () => { + const source = createBasicSource([ + { sqlExpression: 'ServiceName' } as any, // No alias + ]); + const data = [{ ServiceName: 'api-service' }]; + const meta = [{ name: 'ServiceName', type: 'String' }]; + const attributes = getAttributesFromData(source, data, meta); + + expect(attributes).toHaveLength(1); + expect(attributes[0].displayedKey).toBe('ServiceName'); + }); + + it('includes lucene expression when provided', () => { + const source = createBasicSource([ + { + sqlExpression: "SpanAttributes['http.method']", + alias: 'method', + luceneExpression: 'http.method', + }, + ]); + const data = [{ method: 'POST' }]; + const attributes = getAttributesFromData(source, data, basicMeta); + + expect(attributes).toHaveLength(1); + expect(attributes[0].lucene).toBe('http.method'); + }); + + it('omits lucene when not provided', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }]; + const attributes = getAttributesFromData(source, data, basicMeta); + + expect(attributes).toHaveLength(1); + expect(attributes[0].lucene).toBeUndefined(); + }); + + it('handles multiple attributes with different values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + { sqlExpression: 'status', alias: 'status' }, + ]); + const data = [ + { method: 'POST', status: '200' }, + { method: 'GET', status: '404' }, + ]; + const meta = [ + { name: 'method', type: 'String' }, + { name: 'status', type: 'String' }, + ]; + const attributes = getAttributesFromData(source, data, meta); + + expect(attributes).toHaveLength(4); + expect( + attributes.filter(a => a.displayedKey === 'method').map(a => a.value), + ).toEqual(['POST', 'GET']); + expect( + attributes.filter(a => a.displayedKey === 'status').map(a => a.value), + ).toEqual(['200', '404']); + }); + + it('ignores rows with null attribute values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: null }, { method: 'GET' }]; + const attributes = getAttributesFromData(source, data, basicMeta); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('ignores rows with undefined attribute values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: undefined }, { method: 'GET' }]; + const attributes = getAttributesFromData(source, data, basicMeta); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('ignores rows with empty string values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: '' }, { method: 'GET' }]; + const attributes = getAttributesFromData(source, data, basicMeta); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('handles errors gracefully and returns empty array', () => { + // Create a source that will cause an error during iteration + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + + // Mock console.error to suppress error output during test + const consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => {}); + + // Create data that throws when accessed + const data = [ + Object.create( + {}, + { + method: { + get() { + throw new Error('Test error'); + }, + }, + }, + ), + ]; + + const attributes = getAttributesFromData(source, data, basicMeta); + + expect(attributes).toEqual([]); + expect(consoleErrorSpy).toHaveBeenCalledWith( + 'Error extracting attributes from data', + expect.any(Error), + ); + + consoleErrorSpy.mockRestore(); + }); + + it('includes source reference in each attribute', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: 'GET' }]; + const attributes = getAttributesFromData(source, data, basicMeta); + + expect(attributes).toHaveLength(2); + attributes.forEach(attr => { + expect(attr.source).toBe(source); + }); + }); }); From 2f5d71f30246c1ddb9a9310c1ebb9fd783d22a58 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Mon, 17 Nov 2025 10:13:53 -0500 Subject: [PATCH 6/8] refactor: Rename highlightedTraceAttributeExpressions --- packages/api/src/models/source.ts | 2 +- packages/app/src/components/DBTraceWaterfallChart.tsx | 8 ++++---- packages/app/src/components/SourceForm.tsx | 8 ++++---- .../__tests__/DBTraceWaterfallChart.test.tsx | 10 +++++----- packages/common-utils/src/types.ts | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 6f2485355..a1f5c3ded 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -66,7 +66,7 @@ export const Source = mongoose.model( statusCodeExpression: String, statusMessageExpression: String, spanEventsValueExpression: String, - highlightedAttributeExpressions: { + highlightedTraceAttributeExpressions: { type: mongoose.Schema.Types.Array, }, diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx index 72b53b48e..ea21311f6 100644 --- a/packages/app/src/components/DBTraceWaterfallChart.tsx +++ b/packages/app/src/components/DBTraceWaterfallChart.tsx @@ -90,8 +90,8 @@ function getConfig(source: TSource, traceId: string) { // Aliases for trace attributes must be added here to ensure // the returned `alias` object includes them and useRowWhere works. - if (source.highlightedAttributeExpressions) { - for (const expr of source.highlightedAttributeExpressions) { + if (source.highlightedTraceAttributeExpressions) { + for (const expr of source.highlightedTraceAttributeExpressions) { if (expr.alias) { alias[expr.alias] = expr.sqlExpression; } @@ -123,7 +123,7 @@ function getConfig(source: TSource, traceId: string) { if (source.kind === SourceKind.Trace || source.kind === SourceKind.Log) { select.push( - ...(source.highlightedAttributeExpressions ?? []).map( + ...(source.highlightedTraceAttributeExpressions ?? []).map( ({ sqlExpression, alias }) => ({ valueExpression: sqlExpression, alias: alias || sqlExpression, @@ -285,7 +285,7 @@ export const getAttributesFromData = ( sqlExpression, luceneExpression, alias, - } of source.highlightedAttributeExpressions ?? []) { + } of source.highlightedTraceAttributeExpressions ?? []) { const displayName = alias || sqlExpression; const isJsonExpression = jsonColumns.includes( diff --git a/packages/app/src/components/SourceForm.tsx b/packages/app/src/components/SourceForm.tsx index 5bdb7bcf9..1bd103437 100644 --- a/packages/app/src/components/SourceForm.tsx +++ b/packages/app/src/components/SourceForm.tsx @@ -162,7 +162,7 @@ function HighlightedAttributeExpressionsFormRow({ remove: removeHighlightedAttribute, } = useFieldArray({ control, - name: 'highlightedAttributeExpressions', + name: 'highlightedTraceAttributeExpressions', }); return ( @@ -181,7 +181,7 @@ function HighlightedAttributeExpressionsFormRow({ connectionId, }} control={control} - name={`highlightedAttributeExpressions.${index}.sqlExpression`} + name={`highlightedTraceAttributeExpressions.${index}.sqlExpression`} disableKeywordAutocomplete placeholder="ResourceAttributes['http.host']" /> @@ -191,7 +191,7 @@ function HighlightedAttributeExpressionsFormRow({ AS @@ -208,7 +208,7 @@ function HighlightedAttributeExpressionsFormRow({ diff --git a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx index 8b6d21688..6a47a3160 100644 --- a/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx +++ b/packages/app/src/components/__tests__/DBTraceWaterfallChart.test.tsx @@ -357,7 +357,7 @@ describe('useEventsAroundFocus', () => { describe('getAttributesFromData', () => { const createBasicSource = ( - highlightedAttributeExpressions: any[] = [], + highlightedTraceAttributeExpressions: any[] = [], ): TSource => ({ kind: SourceKind.Trace, from: { @@ -367,7 +367,7 @@ describe('getAttributesFromData', () => { timestampValueExpression: 'Timestamp', connection: 'test-connection', name: 'Traces', - highlightedAttributeExpressions, + highlightedTraceAttributeExpressions, id: 'test-source-id', }); @@ -486,7 +486,7 @@ describe('getAttributesFromData', () => { statusMessageExpression: 'StatusMessage', sessionSourceId: '68dd82484f54641b0866789e', logSourceId: '6900eed982d3b3dfeff12a29', - highlightedAttributeExpressions: [ + highlightedTraceAttributeExpressions: [ { sqlExpression: "SpanAttributes['http.method']", alias: 'method', @@ -532,14 +532,14 @@ describe('getAttributesFromData', () => { expect(attributes).toEqual([]); }); - it('returns empty array when highlightedAttributeExpressions is undefined', () => { + it('returns empty array when highlightedTraceAttributeExpressions is undefined', () => { const source = createBasicSource(); const data = [{ method: 'POST' }]; const attributes = getAttributesFromData(source, data, basicMeta); expect(attributes).toEqual([]); }); - it('returns empty array when highlightedAttributeExpressions is empty', () => { + it('returns empty array when highlightedTraceAttributeExpressions is empty', () => { const source = createBasicSource([]); const data = [{ method: 'POST' }]; const attributes = getAttributesFromData(source, data, basicMeta); diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index ad0d64890..fbbc78f33 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -578,7 +578,7 @@ const LogSourceAugmentation = { implicitColumnExpression: z.string().optional(), uniqueRowIdExpression: z.string().optional(), tableFilterExpression: z.string().optional(), - highlightedAttributeExpressions: + highlightedTraceAttributeExpressions: HighlightedAttributeExpressionsSchema.optional(), }; @@ -610,7 +610,7 @@ const TraceSourceAugmentation = { eventAttributesExpression: z.string().optional(), spanEventsValueExpression: z.string().optional(), implicitColumnExpression: z.string().optional(), - highlightedAttributeExpressions: + highlightedTraceAttributeExpressions: HighlightedAttributeExpressionsSchema.optional(), }; From c1f6abdaf99d4271e2438925367832b9beb6af91 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 18 Nov 2025 08:48:27 -0500 Subject: [PATCH 7/8] refactor: Create DBHighlightedAttributesList and related utils --- .../DBHighlightedAttributesList.tsx | 67 +++ .../src/components/DBTraceWaterfallChart.tsx | 205 ++------ .../__tests__/DBTraceWaterfallChart.test.tsx | 398 --------------- .../__tests__/highlightedAttributes.test.ts | 480 ++++++++++++++++++ .../app/src/utils/highlightedAttributes.ts | 69 +++ 5 files changed, 660 insertions(+), 559 deletions(-) create mode 100644 packages/app/src/components/DBHighlightedAttributesList.tsx create mode 100644 packages/app/src/utils/__tests__/highlightedAttributes.test.ts create mode 100644 packages/app/src/utils/highlightedAttributes.ts diff --git a/packages/app/src/components/DBHighlightedAttributesList.tsx b/packages/app/src/components/DBHighlightedAttributesList.tsx new file mode 100644 index 000000000..92f36fe9e --- /dev/null +++ b/packages/app/src/components/DBHighlightedAttributesList.tsx @@ -0,0 +1,67 @@ +import { useContext, useMemo } from 'react'; +import { TSource } from '@hyperdx/common-utils/dist/types'; +import { Flex } from '@mantine/core'; + +import { RowSidePanelContext } from './DBRowSidePanel'; +import EventTag from './EventTag'; + +export type HighlightedAttribute = { + source: TSource; + displayedKey: string; + value: string; + sql: string; + lucene?: string; +}; + +export function DBHighlightedAttributesList({ + attributes = [], +}: { + attributes: HighlightedAttribute[]; +}) { + const { + onPropertyAddClick, + generateSearchUrl, + source: contextSource, + } = useContext(RowSidePanelContext); + + const sortedAttributes = useMemo(() => { + return attributes.sort( + (a, b) => + a.displayedKey.localeCompare(b.displayedKey) || + a.value.localeCompare(b.value), + ); + }, [attributes]); + + return ( + + {sortedAttributes.map(({ displayedKey, value, sql, lucene, source }) => ( + + generateSearchUrl({ + where: query || '', + whereLanguage: queryLanguage ?? 'lucene', + source, + }) + : undefined + } + /> + ))} + + ); +} diff --git a/packages/app/src/components/DBTraceWaterfallChart.tsx b/packages/app/src/components/DBTraceWaterfallChart.tsx index ea21311f6..b101caa90 100644 --- a/packages/app/src/components/DBTraceWaterfallChart.tsx +++ b/packages/app/src/components/DBTraceWaterfallChart.tsx @@ -1,14 +1,13 @@ -import { useCallback, useContext, useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import _ from 'lodash'; import TimestampNano from 'timestamp-nano'; -import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; import { ChartConfig, ChartConfigWithDateRange, SourceKind, TSource, } from '@hyperdx/common-utils/dist/types'; -import { Divider, Flex, Text } from '@mantine/core'; +import { Divider, Text } from '@mantine/core'; import { ContactSupportText } from '@/components/ContactSupportText'; import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery'; @@ -20,10 +19,12 @@ import { getSpanEventBody, } from '@/source'; import TimelineChart from '@/TimelineChart'; +import { + getHighlightedAttributesFromData, + getSelectExpressionsForHighlightedAttributes, +} from '@/utils/highlightedAttributes'; -import { getJSONColumnNames } from './DBRowDataPanel'; -import { RowSidePanelContext } from './DBRowSidePanel'; -import EventTag from './EventTag'; +import { DBHighlightedAttributesList } from './DBHighlightedAttributesList'; import styles from '@/../styles/LogSidePanel.module.scss'; import resizeStyles from '@/../styles/ResizablePanel.module.scss'; @@ -123,11 +124,8 @@ function getConfig(source: TSource, traceId: string) { if (source.kind === SourceKind.Trace || source.kind === SourceKind.Log) { select.push( - ...(source.highlightedTraceAttributeExpressions ?? []).map( - ({ sqlExpression, alias }) => ({ - valueExpression: sqlExpression, - alias: alias || sqlExpression, - }), + ...getSelectExpressionsForHighlightedAttributes( + source.highlightedTraceAttributeExpressions, ), ); } @@ -269,149 +267,6 @@ export function useEventsAroundFocus({ }; } -export const getAttributesFromData = ( - source: TSource, - data: Record[], - meta: ResponseJSON['meta'], -) => { - const attributeValuesByDisplayKey = new Map>(); - const sqlExpressionsByDisplayKey = new Map(); - const luceneExpressionsByDisplayKey = new Map(); - const jsonColumns = getJSONColumnNames(meta); - - try { - for (const row of data) { - for (const { - sqlExpression, - luceneExpression, - alias, - } of source.highlightedTraceAttributeExpressions ?? []) { - const displayName = alias || sqlExpression; - - const isJsonExpression = jsonColumns.includes( - sqlExpression.split('.')[0], - ); - const sqlExpressionWithJSONSupport = isJsonExpression - ? `toString(${sqlExpression})` - : sqlExpression; - - sqlExpressionsByDisplayKey.set( - displayName, - sqlExpressionWithJSONSupport, - ); - if (luceneExpression) { - luceneExpressionsByDisplayKey.set(displayName, luceneExpression); - } - - const value = row[displayName]; - if (value && typeof value === 'string') { - if (!attributeValuesByDisplayKey.has(displayName)) { - attributeValuesByDisplayKey.set(displayName, new Set()); - } - attributeValuesByDisplayKey.get(displayName)!.add(value); - } - } - } - } catch (e) { - console.error('Error extracting attributes from data', e); - } - - return Array.from(attributeValuesByDisplayKey.entries()).flatMap( - ([key, values]) => - [...values].map(value => ({ - displayedKey: key, - value, - sql: sqlExpressionsByDisplayKey.get(key)!, - lucene: luceneExpressionsByDisplayKey.get(key), - source, - })), - ); -}; - -interface DBTraceAttributesProps { - traceSource: TSource; - traceEventData: Record[]; - traceEventMeta: ResponseJSON['meta']; - logSource?: TSource | null; - logEventData?: Record[] | null; - logEventMeta?: ResponseJSON['meta']; -} - -export function DBTraceAttributes({ - traceSource, - traceEventData, - traceEventMeta, - logSource, - logEventData, - logEventMeta, -}: DBTraceAttributesProps) { - const { - onPropertyAddClick, - generateSearchUrl, - source: contextSource, - } = useContext(RowSidePanelContext); - - const attributeValues = useMemo(() => { - const attributes = getAttributesFromData( - traceSource, - traceEventData, - traceEventMeta, - ); - - if (logSource && logEventData && logEventMeta) { - attributes.push( - ...getAttributesFromData(logSource, logEventData, logEventMeta), - ); - } - - return attributes.sort( - (a, b) => - a.displayedKey.localeCompare(b.displayedKey) || - a.value.localeCompare(b.value), - ); - }, [ - traceSource, - traceEventData, - traceEventMeta, - logSource, - logEventData, - logEventMeta, - ]); - - return ( - - {attributeValues.map(({ displayedKey, value, sql, lucene, source }) => ( - - generateSearchUrl({ - where: query || '', - whereLanguage: queryLanguage ?? 'lucene', - source, - }) - : undefined - } - /> - ))} - - ); -} - // TODO: Optimize with ts lookup tables export function DBTraceWaterfallChartContainer({ traceTableSource, @@ -480,6 +335,39 @@ export function DBTraceWaterfallChartContainer({ } }); + const highlightedAttributeValues = useMemo(() => { + const attributes = getHighlightedAttributesFromData( + traceTableSource, + traceTableSource.highlightedTraceAttributeExpressions, + traceRowsData, + traceRowsMeta, + ); + + if (logTableSource && logRowsData && logRowsMeta) { + attributes.push( + ...getHighlightedAttributesFromData( + logTableSource, + logTableSource.highlightedTraceAttributeExpressions, + logRowsData, + logRowsMeta, + ), + ); + } + + return attributes.sort( + (a, b) => + a.displayedKey.localeCompare(b.displayedKey) || + a.value.localeCompare(b.value), + ); + }, [ + traceTableSource, + traceRowsData, + traceRowsMeta, + logTableSource, + logRowsData, + logRowsMeta, + ]); + useEffect(() => { if (initialRowHighlightHint && onClick && highlightedRowWhere == null) { const initialRowHighlightIndex = rows.findIndex(row => { @@ -752,13 +640,8 @@ export function DBTraceWaterfallChartContainer({ ) : ( <> - { expect(result.rows.length).toBe(0); }); }); - -describe('getAttributesFromData', () => { - const createBasicSource = ( - highlightedTraceAttributeExpressions: any[] = [], - ): TSource => ({ - kind: SourceKind.Trace, - from: { - databaseName: 'default', - tableName: 'otel_traces', - }, - timestampValueExpression: 'Timestamp', - connection: 'test-connection', - name: 'Traces', - highlightedTraceAttributeExpressions, - id: 'test-source-id', - }); - - const basicMeta = [ - { name: 'Body', type: 'String' }, - { name: 'Timestamp', type: 'DateTime64(9)' }, - { name: 'method', type: 'String' }, - ]; - - it('extracts attributes from data correctly', () => { - const data: Record[] = [ - { - Body: 'POST', - Timestamp: '2025-11-12T21:27:00.053000000Z', - SpanId: 'a51d12055f2058b9', - ServiceName: 'hdx-oss-dev-api', - method: 'POST', - "SpanAttributes['http.host']": 'localhost:8123', - Duration: 0.020954166, - ParentSpanId: '013cca18a6e626a6', - StatusCode: 'Unset', - SpanAttributes: { - 'http.flavor': '1.1', - 'http.host': 'localhost:8123', - 'http.method': 'POST', - }, - type: 'trace', - }, - { - Body: 'POST', - Timestamp: '2025-11-12T21:27:00.053000000Z', - SpanId: 'a51d12055f2058b9', - ServiceName: 'hdx-oss-dev-api', - method: 'GET', - "SpanAttributes['http.host']": 'localhost:8123', - Duration: 0.020954166, - ParentSpanId: '013cca18a6e626a6', - StatusCode: 'Unset', - SpanAttributes: { - 'http.flavor': '1.1', - 'http.host': 'localhost:8123', - 'http.method': 'POST', - }, - type: 'trace', - }, - ]; - - const meta = [ - { - name: 'Body', - type: 'LowCardinality(String)', - }, - { - name: 'Timestamp', - type: 'DateTime64(9)', - }, - { - name: 'SpanId', - type: 'String', - }, - { - name: 'ServiceName', - type: 'LowCardinality(String)', - }, - { - name: 'method', - type: 'String', - }, - { - name: "SpanAttributes['http.host']", - type: 'String', - }, - { - name: 'Duration', - type: 'Float64', - }, - { - name: 'ParentSpanId', - type: 'String', - }, - { - name: 'StatusCode', - type: 'LowCardinality(String)', - }, - { - name: 'SpanAttributes', - type: 'Map(LowCardinality(String), String)', - }, - ]; - - const source: TSource = { - kind: SourceKind.Trace, - from: { - databaseName: 'default', - tableName: 'otel_traces', - }, - timestampValueExpression: 'Timestamp', - connection: '68dd82484f54641b08667893', - name: 'Traces', - displayedTimestampValueExpression: 'Timestamp', - implicitColumnExpression: 'SpanName', - serviceNameExpression: 'ServiceName', - bodyExpression: 'SpanName', - eventAttributesExpression: 'SpanAttributes', - resourceAttributesExpression: 'ResourceAttributes', - defaultTableSelectExpression: - 'Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName', - traceIdExpression: 'TraceId', - spanIdExpression: 'SpanId', - durationExpression: 'Duration', - durationPrecision: 9, - parentSpanIdExpression: 'ParentSpanId', - spanNameExpression: 'SpanName', - spanKindExpression: 'SpanKind', - statusCodeExpression: 'StatusCode', - statusMessageExpression: 'StatusMessage', - sessionSourceId: '68dd82484f54641b0866789e', - logSourceId: '6900eed982d3b3dfeff12a29', - highlightedTraceAttributeExpressions: [ - { - sqlExpression: "SpanAttributes['http.method']", - alias: 'method', - }, - { - sqlExpression: "SpanAttributes['http.host']", - luceneExpression: 'SpanAttributes.http.host', - alias: '', - }, - ], - id: '68dd82484f54641b08667899', - }; - - const attributes = getAttributesFromData(source, data, meta); - - expect(attributes).toHaveLength(3); - expect(attributes).toContainEqual({ - sql: "SpanAttributes['http.method']", - displayedKey: 'method', - value: 'POST', - source, - }); - expect(attributes).toContainEqual({ - sql: "SpanAttributes['http.method']", - displayedKey: 'method', - value: 'GET', - source, - }); - expect(attributes).toContainEqual({ - sql: "SpanAttributes['http.host']", - displayedKey: "SpanAttributes['http.host']", - value: 'localhost:8123', - lucene: 'SpanAttributes.http.host', - source, - }); - }); - - it('returns empty array when data is empty', () => { - const source = createBasicSource([ - { sqlExpression: 'method', alias: 'method' }, - ]); - const attributes = getAttributesFromData(source, [], basicMeta); - expect(attributes).toEqual([]); - }); - - it('returns empty array when highlightedTraceAttributeExpressions is undefined', () => { - const source = createBasicSource(); - const data = [{ method: 'POST' }]; - const attributes = getAttributesFromData(source, data, basicMeta); - expect(attributes).toEqual([]); - }); - - it('returns empty array when highlightedTraceAttributeExpressions is empty', () => { - const source = createBasicSource([]); - const data = [{ method: 'POST' }]; - const attributes = getAttributesFromData(source, data, basicMeta); - expect(attributes).toEqual([]); - }); - - it('filters out non-string values', () => { - const source = createBasicSource([ - { sqlExpression: 'method', alias: 'method' }, - { sqlExpression: 'count', alias: 'count' }, - { sqlExpression: 'isActive', alias: 'isActive' }, - ]); - const data = [ - { - method: 'POST', // string - should be included - count: 123, // number - should be filtered out - isActive: true, // boolean - should be filtered out - }, - ]; - const meta = [ - { name: 'method', type: 'String' }, - { name: 'count', type: 'Int32' }, - { name: 'isActive', type: 'Bool' }, - ]; - const attributes = getAttributesFromData(source, data, meta); - - expect(attributes).toHaveLength(1); - expect(attributes[0]).toEqual({ - displayedKey: 'method', - value: 'POST', - sql: 'method', - lucene: undefined, - source, - }); - }); - - it('deduplicates values from multiple rows', () => { - const source = createBasicSource([ - { sqlExpression: 'method', alias: 'method' }, - ]); - const data = [ - { method: 'POST' }, - { method: 'POST' }, // duplicate - { method: 'GET' }, - { method: 'POST' }, // duplicate - ]; - const attributes = getAttributesFromData(source, data, basicMeta); - - expect(attributes).toHaveLength(2); - expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); - }); - - it('uses sqlExpression as displayedKey when alias is empty string', () => { - const source = createBasicSource([ - { sqlExpression: "SpanAttributes['http.host']", alias: '' }, - ]); - const data = [{ "SpanAttributes['http.host']": 'localhost:8080' }]; - const meta = [ - { name: "SpanAttributes['http.host']", type: 'String' }, - { name: 'SpanAttributes', type: 'JSON' }, - ]; - const attributes = getAttributesFromData(source, data, meta); - - expect(attributes).toHaveLength(1); - expect(attributes[0].displayedKey).toBe("SpanAttributes['http.host']"); - }); - - it('uses sqlExpression as displayedKey when alias is not provided', () => { - const source = createBasicSource([ - { sqlExpression: 'ServiceName' } as any, // No alias - ]); - const data = [{ ServiceName: 'api-service' }]; - const meta = [{ name: 'ServiceName', type: 'String' }]; - const attributes = getAttributesFromData(source, data, meta); - - expect(attributes).toHaveLength(1); - expect(attributes[0].displayedKey).toBe('ServiceName'); - }); - - it('includes lucene expression when provided', () => { - const source = createBasicSource([ - { - sqlExpression: "SpanAttributes['http.method']", - alias: 'method', - luceneExpression: 'http.method', - }, - ]); - const data = [{ method: 'POST' }]; - const attributes = getAttributesFromData(source, data, basicMeta); - - expect(attributes).toHaveLength(1); - expect(attributes[0].lucene).toBe('http.method'); - }); - - it('omits lucene when not provided', () => { - const source = createBasicSource([ - { sqlExpression: 'method', alias: 'method' }, - ]); - const data = [{ method: 'POST' }]; - const attributes = getAttributesFromData(source, data, basicMeta); - - expect(attributes).toHaveLength(1); - expect(attributes[0].lucene).toBeUndefined(); - }); - - it('handles multiple attributes with different values', () => { - const source = createBasicSource([ - { sqlExpression: 'method', alias: 'method' }, - { sqlExpression: 'status', alias: 'status' }, - ]); - const data = [ - { method: 'POST', status: '200' }, - { method: 'GET', status: '404' }, - ]; - const meta = [ - { name: 'method', type: 'String' }, - { name: 'status', type: 'String' }, - ]; - const attributes = getAttributesFromData(source, data, meta); - - expect(attributes).toHaveLength(4); - expect( - attributes.filter(a => a.displayedKey === 'method').map(a => a.value), - ).toEqual(['POST', 'GET']); - expect( - attributes.filter(a => a.displayedKey === 'status').map(a => a.value), - ).toEqual(['200', '404']); - }); - - it('ignores rows with null attribute values', () => { - const source = createBasicSource([ - { sqlExpression: 'method', alias: 'method' }, - ]); - const data = [{ method: 'POST' }, { method: null }, { method: 'GET' }]; - const attributes = getAttributesFromData(source, data, basicMeta); - - expect(attributes).toHaveLength(2); - expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); - }); - - it('ignores rows with undefined attribute values', () => { - const source = createBasicSource([ - { sqlExpression: 'method', alias: 'method' }, - ]); - const data = [{ method: 'POST' }, { method: undefined }, { method: 'GET' }]; - const attributes = getAttributesFromData(source, data, basicMeta); - - expect(attributes).toHaveLength(2); - expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); - }); - - it('ignores rows with empty string values', () => { - const source = createBasicSource([ - { sqlExpression: 'method', alias: 'method' }, - ]); - const data = [{ method: 'POST' }, { method: '' }, { method: 'GET' }]; - const attributes = getAttributesFromData(source, data, basicMeta); - - expect(attributes).toHaveLength(2); - expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); - }); - - it('handles errors gracefully and returns empty array', () => { - // Create a source that will cause an error during iteration - const source = createBasicSource([ - { sqlExpression: 'method', alias: 'method' }, - ]); - - // Mock console.error to suppress error output during test - const consoleErrorSpy = jest - .spyOn(console, 'error') - .mockImplementation(() => {}); - - // Create data that throws when accessed - const data = [ - Object.create( - {}, - { - method: { - get() { - throw new Error('Test error'); - }, - }, - }, - ), - ]; - - const attributes = getAttributesFromData(source, data, basicMeta); - - expect(attributes).toEqual([]); - expect(consoleErrorSpy).toHaveBeenCalledWith( - 'Error extracting attributes from data', - expect.any(Error), - ); - - consoleErrorSpy.mockRestore(); - }); - - it('includes source reference in each attribute', () => { - const source = createBasicSource([ - { sqlExpression: 'method', alias: 'method' }, - ]); - const data = [{ method: 'POST' }, { method: 'GET' }]; - const attributes = getAttributesFromData(source, data, basicMeta); - - expect(attributes).toHaveLength(2); - attributes.forEach(attr => { - expect(attr.source).toBe(source); - }); - }); -}); diff --git a/packages/app/src/utils/__tests__/highlightedAttributes.test.ts b/packages/app/src/utils/__tests__/highlightedAttributes.test.ts new file mode 100644 index 000000000..236cde572 --- /dev/null +++ b/packages/app/src/utils/__tests__/highlightedAttributes.test.ts @@ -0,0 +1,480 @@ +import { SourceKind, TSource } from '@hyperdx/common-utils/dist/types'; + +import { getHighlightedAttributesFromData } from '../highlightedAttributes'; + +describe('getHighlightedAttributesFromData', () => { + const createBasicSource = ( + highlightedTraceAttributeExpressions: any[] = [], + ): TSource => ({ + kind: SourceKind.Trace, + from: { + databaseName: 'default', + tableName: 'otel_traces', + }, + timestampValueExpression: 'Timestamp', + connection: 'test-connection', + name: 'Traces', + highlightedTraceAttributeExpressions, + id: 'test-source-id', + }); + + const basicMeta = [ + { name: 'Body', type: 'String' }, + { name: 'Timestamp', type: 'DateTime64(9)' }, + { name: 'method', type: 'String' }, + ]; + + it('extracts attributes from data correctly', () => { + const data: Record[] = [ + { + Body: 'POST', + Timestamp: '2025-11-12T21:27:00.053000000Z', + SpanId: 'a51d12055f2058b9', + ServiceName: 'hdx-oss-dev-api', + method: 'POST', + "SpanAttributes['http.host']": 'localhost:8123', + Duration: 0.020954166, + ParentSpanId: '013cca18a6e626a6', + StatusCode: 'Unset', + SpanAttributes: { + 'http.flavor': '1.1', + 'http.host': 'localhost:8123', + 'http.method': 'POST', + }, + type: 'trace', + }, + { + Body: 'POST', + Timestamp: '2025-11-12T21:27:00.053000000Z', + SpanId: 'a51d12055f2058b9', + ServiceName: 'hdx-oss-dev-api', + method: 'GET', + "SpanAttributes['http.host']": 'localhost:8123', + Duration: 0.020954166, + ParentSpanId: '013cca18a6e626a6', + StatusCode: 'Unset', + SpanAttributes: { + 'http.flavor': '1.1', + 'http.host': 'localhost:8123', + 'http.method': 'POST', + }, + type: 'trace', + }, + ]; + + const meta = [ + { + name: 'Body', + type: 'LowCardinality(String)', + }, + { + name: 'Timestamp', + type: 'DateTime64(9)', + }, + { + name: 'SpanId', + type: 'String', + }, + { + name: 'ServiceName', + type: 'LowCardinality(String)', + }, + { + name: 'method', + type: 'String', + }, + { + name: "SpanAttributes['http.host']", + type: 'String', + }, + { + name: 'Duration', + type: 'Float64', + }, + { + name: 'ParentSpanId', + type: 'String', + }, + { + name: 'StatusCode', + type: 'LowCardinality(String)', + }, + { + name: 'SpanAttributes', + type: 'Map(LowCardinality(String), String)', + }, + ]; + + const source: TSource = { + kind: SourceKind.Trace, + from: { + databaseName: 'default', + tableName: 'otel_traces', + }, + timestampValueExpression: 'Timestamp', + connection: '68dd82484f54641b08667893', + name: 'Traces', + displayedTimestampValueExpression: 'Timestamp', + implicitColumnExpression: 'SpanName', + serviceNameExpression: 'ServiceName', + bodyExpression: 'SpanName', + eventAttributesExpression: 'SpanAttributes', + resourceAttributesExpression: 'ResourceAttributes', + defaultTableSelectExpression: + 'Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName', + traceIdExpression: 'TraceId', + spanIdExpression: 'SpanId', + durationExpression: 'Duration', + durationPrecision: 9, + parentSpanIdExpression: 'ParentSpanId', + spanNameExpression: 'SpanName', + spanKindExpression: 'SpanKind', + statusCodeExpression: 'StatusCode', + statusMessageExpression: 'StatusMessage', + sessionSourceId: '68dd82484f54641b0866789e', + logSourceId: '6900eed982d3b3dfeff12a29', + highlightedTraceAttributeExpressions: [ + { + sqlExpression: "SpanAttributes['http.method']", + alias: 'method', + }, + { + sqlExpression: "SpanAttributes['http.host']", + luceneExpression: 'SpanAttributes.http.host', + alias: '', + }, + ], + id: '68dd82484f54641b08667899', + }; + + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + meta, + ); + + expect(attributes).toHaveLength(3); + expect(attributes).toContainEqual({ + sql: "SpanAttributes['http.method']", + displayedKey: 'method', + value: 'POST', + source, + }); + expect(attributes).toContainEqual({ + sql: "SpanAttributes['http.method']", + displayedKey: 'method', + value: 'GET', + source, + }); + expect(attributes).toContainEqual({ + sql: "SpanAttributes['http.host']", + displayedKey: "SpanAttributes['http.host']", + value: 'localhost:8123', + lucene: 'SpanAttributes.http.host', + source, + }); + }); + + it('returns empty array when data is empty', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + [], + basicMeta, + ); + expect(attributes).toEqual([]); + }); + + it('returns empty array when highlightedTraceAttributeExpressions is undefined', () => { + const source = createBasicSource(); + const data = [{ method: 'POST' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + expect(attributes).toEqual([]); + }); + + it('returns empty array when highlightedTraceAttributeExpressions is empty', () => { + const source = createBasicSource([]); + const data = [{ method: 'POST' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + expect(attributes).toEqual([]); + }); + + it('filters out non-string values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + { sqlExpression: 'count', alias: 'count' }, + { sqlExpression: 'isActive', alias: 'isActive' }, + ]); + const data = [ + { + method: 'POST', // string - should be included + count: 123, // number - should be filtered out + isActive: true, // boolean - should be filtered out + }, + ]; + const meta = [ + { name: 'method', type: 'String' }, + { name: 'count', type: 'Int32' }, + { name: 'isActive', type: 'Bool' }, + ]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + meta, + ); + + expect(attributes).toHaveLength(1); + expect(attributes[0]).toEqual({ + displayedKey: 'method', + value: 'POST', + sql: 'method', + lucene: undefined, + source, + }); + }); + + it('deduplicates values from multiple rows', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [ + { method: 'POST' }, + { method: 'POST' }, // duplicate + { method: 'GET' }, + { method: 'POST' }, // duplicate + ]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('uses sqlExpression as displayedKey when alias is empty string', () => { + const source = createBasicSource([ + { sqlExpression: "SpanAttributes['http.host']", alias: '' }, + ]); + const data = [{ "SpanAttributes['http.host']": 'localhost:8080' }]; + const meta = [ + { name: "SpanAttributes['http.host']", type: 'String' }, + { name: 'SpanAttributes', type: 'JSON' }, + ]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + meta, + ); + + expect(attributes).toHaveLength(1); + expect(attributes[0].displayedKey).toBe("SpanAttributes['http.host']"); + }); + + it('uses sqlExpression as displayedKey when alias is not provided', () => { + const source = createBasicSource([ + { sqlExpression: 'ServiceName' } as any, // No alias + ]); + const data = [{ ServiceName: 'api-service' }]; + const meta = [{ name: 'ServiceName', type: 'String' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + meta, + ); + + expect(attributes).toHaveLength(1); + expect(attributes[0].displayedKey).toBe('ServiceName'); + }); + + it('includes lucene expression when provided', () => { + const source = createBasicSource([ + { + sqlExpression: "SpanAttributes['http.method']", + alias: 'method', + luceneExpression: 'http.method', + }, + ]); + const data = [{ method: 'POST' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(1); + expect(attributes[0].lucene).toBe('http.method'); + }); + + it('omits lucene when not provided', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(1); + expect(attributes[0].lucene).toBeUndefined(); + }); + + it('handles multiple attributes with different values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + { sqlExpression: 'status', alias: 'status' }, + ]); + const data = [ + { method: 'POST', status: '200' }, + { method: 'GET', status: '404' }, + ]; + const meta = [ + { name: 'method', type: 'String' }, + { name: 'status', type: 'String' }, + ]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + meta, + ); + + expect(attributes).toHaveLength(4); + expect( + attributes.filter(a => a.displayedKey === 'method').map(a => a.value), + ).toEqual(['POST', 'GET']); + expect( + attributes.filter(a => a.displayedKey === 'status').map(a => a.value), + ).toEqual(['200', '404']); + }); + + it('ignores rows with null attribute values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: null }, { method: 'GET' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('ignores rows with undefined attribute values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: undefined }, { method: 'GET' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('ignores rows with empty string values', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: '' }, { method: 'GET' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(2); + expect(attributes.map(a => a.value).sort()).toEqual(['GET', 'POST']); + }); + + it('handles errors gracefully and returns empty array', () => { + // Create a source that will cause an error during iteration + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + + // Mock console.error to suppress error output during test + const consoleErrorSpy = jest + .spyOn(console, 'error') + .mockImplementation(() => {}); + + // Create data that throws when accessed + const data = [ + Object.create( + {}, + { + method: { + get() { + throw new Error('Test error'); + }, + }, + }, + ), + ]; + + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toEqual([]); + expect(consoleErrorSpy).toHaveBeenCalledWith( + 'Error extracting attributes from data', + expect.any(Error), + ); + + consoleErrorSpy.mockRestore(); + }); + + it('includes source reference in each attribute', () => { + const source = createBasicSource([ + { sqlExpression: 'method', alias: 'method' }, + ]); + const data = [{ method: 'POST' }, { method: 'GET' }]; + const attributes = getHighlightedAttributesFromData( + source, + source.highlightedTraceAttributeExpressions, + data, + basicMeta, + ); + + expect(attributes).toHaveLength(2); + attributes.forEach(attr => { + expect(attr.source).toBe(source); + }); + }); +}); diff --git a/packages/app/src/utils/highlightedAttributes.ts b/packages/app/src/utils/highlightedAttributes.ts new file mode 100644 index 000000000..a0deff516 --- /dev/null +++ b/packages/app/src/utils/highlightedAttributes.ts @@ -0,0 +1,69 @@ +import { ResponseJSON } from '@hyperdx/common-utils/dist/clickhouse'; +import { TSource } from '@hyperdx/common-utils/dist/types'; + +import { getJSONColumnNames } from '@/components/DBRowDataPanel'; + +export function getSelectExpressionsForHighlightedAttributes( + expressions: TSource['highlightedTraceAttributeExpressions'] = [], +) { + return expressions.map(({ sqlExpression, alias }) => ({ + valueExpression: sqlExpression, + alias: alias || sqlExpression, + })); +} + +export function getHighlightedAttributesFromData( + source: TSource, + attributes: TSource['highlightedTraceAttributeExpressions'] = [], + data: Record[], + meta: ResponseJSON['meta'], +) { + const attributeValuesByDisplayKey = new Map>(); + const sqlExpressionsByDisplayKey = new Map(); + const luceneExpressionsByDisplayKey = new Map(); + const jsonColumns = getJSONColumnNames(meta); + + try { + for (const row of data) { + for (const { sqlExpression, luceneExpression, alias } of attributes) { + const displayName = alias || sqlExpression; + + const isJsonExpression = jsonColumns.includes( + sqlExpression.split('.')[0], + ); + const sqlExpressionWithJSONSupport = isJsonExpression + ? `toString(${sqlExpression})` + : sqlExpression; + + sqlExpressionsByDisplayKey.set( + displayName, + sqlExpressionWithJSONSupport, + ); + if (luceneExpression) { + luceneExpressionsByDisplayKey.set(displayName, luceneExpression); + } + + const value = row[displayName]; + if (value && typeof value === 'string') { + if (!attributeValuesByDisplayKey.has(displayName)) { + attributeValuesByDisplayKey.set(displayName, new Set()); + } + attributeValuesByDisplayKey.get(displayName)!.add(value); + } + } + } + } catch (e) { + console.error('Error extracting attributes from data', e); + } + + return Array.from(attributeValuesByDisplayKey.entries()).flatMap( + ([key, values]) => + [...values].map(value => ({ + displayedKey: key, + value, + sql: sqlExpressionsByDisplayKey.get(key)!, + lucene: luceneExpressionsByDisplayKey.get(key), + source, + })), + ); +} From 9dd50a72657b01802f4f6ce0c4605c7b72f81f44 Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 18 Nov 2025 13:48:13 -0500 Subject: [PATCH 8/8] review: Fix style --- packages/app/src/DBSearchPage.tsx | 38 ++++++++++-------------- packages/app/src/components/EventTag.tsx | 5 ++-- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/packages/app/src/DBSearchPage.tsx b/packages/app/src/DBSearchPage.tsx index 0cfcc8e2b..80ce1ce08 100644 --- a/packages/app/src/DBSearchPage.tsx +++ b/packages/app/src/DBSearchPage.tsx @@ -1098,33 +1098,27 @@ function DBSearchPage() { whereLanguage: SearchConfig['whereLanguage']; source?: TSource; }) => { + const qParams = new URLSearchParams({ + whereLanguage: whereLanguage || 'sql', + from: searchedTimeRange[0].getTime().toString(), + to: searchedTimeRange[1].getTime().toString(), + isLive: 'false', + liveInterval: interval.toString(), + }); + // When generating a search based on a different source, // filters and select for the current source are not preserved. if (source && source.id !== searchedSource?.id) { - const qParams = new URLSearchParams({ - where: where || '', - whereLanguage: whereLanguage || 'sql', - from: searchedTimeRange[0].getTime().toString(), - to: searchedTimeRange[1].getTime().toString(), - source: source.id, - isLive: 'false', - liveInterval: interval.toString(), - }); - return `/search?${qParams.toString()}`; + qParams.append('where', where || ''); + qParams.append('source', source.id); } else { - const qParams = new URLSearchParams({ - where: where || searchedConfig.where || '', - whereLanguage: whereLanguage || 'sql', - from: searchedTimeRange[0].getTime().toString(), - to: searchedTimeRange[1].getTime().toString(), - select: searchedConfig.select || '', - source: searchedSource?.id || '', - filters: JSON.stringify(searchedConfig.filters ?? []), - isLive: 'false', - liveInterval: interval.toString(), - }); - return `/search?${qParams.toString()}`; + qParams.append('select', searchedConfig.select || ''); + qParams.append('where', where || searchedConfig.where || ''); + qParams.append('filters', JSON.stringify(searchedConfig.filters ?? [])); + qParams.append('source', searchedSource?.id || ''); } + + return `/search?${qParams.toString()}`; }, [ interval, diff --git a/packages/app/src/components/EventTag.tsx b/packages/app/src/components/EventTag.tsx index 848b7a038..9b8832fdd 100644 --- a/packages/app/src/components/EventTag.tsx +++ b/packages/app/src/components/EventTag.tsx @@ -1,6 +1,7 @@ import { useState } from 'react'; import Link from 'next/link'; import SqlString from 'sqlstring'; +import { SearchConditionLanguage } from '@hyperdx/common-utils/dist/types'; import { Button, Popover, Stack } from '@mantine/core'; export default function EventTag({ @@ -16,11 +17,11 @@ export default function EventTag({ /** Property name, in lucene or sql syntax (ex. col.prop or col['prop']) */ name: string; /** The language of the property name, defaults to 'lucene' */ - nameLanguage?: 'sql' | 'lucene'; + nameLanguage?: SearchConditionLanguage; value: string; generateSearchUrl?: ( query?: string, - queryLanguage?: 'sql' | 'lucene', + queryLanguage?: SearchConditionLanguage, ) => string; } & ( | {