From b56d712eb489f4a5f04c26b34db5c02ea94f5bdb Mon Sep 17 00:00:00 2001 From: Idriss Neumann Date: Thu, 16 Nov 2023 18:17:44 +0100 Subject: [PATCH] Issue #12: implement DataSourceWithLogsContextSupport interface --- CONTRIBUTING.md | 13 ++- build_and_start.sh | 6 ++ docker-compose.yaml | 6 +- package-lock.json | 124 +++++--------------------- pkg/quickwit/client/search_request.go | 6 +- pkg/quickwit/data_query.go | 7 ++ src/IntervalMap.ts | 18 ++++ src/datasource.ts | 124 ++++++++++++++++++++++++-- 8 files changed, 187 insertions(+), 117 deletions(-) create mode 100755 build_and_start.sh create mode 100644 src/IntervalMap.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0267fe2..27e7716 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,14 +14,20 @@ You need: ### Building +### All the stack + +```shell +./build_and_start.sh +``` + #### Frontend ```bash -$ yarn install -$ yarn build +$ npm install +$ npm run build ``` -When developing the front, use `yarn dev`. +When developing the front, use `npm run dev`. #### Backend @@ -49,7 +55,6 @@ $ npm run test $ go test -v ./pkg/... ``` - ## Release TODO diff --git a/build_and_start.sh b/build_and_start.sh new file mode 100755 index 0000000..efbdaf6 --- /dev/null +++ b/build_and_start.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +npm install +npm run build +mage -v +docker-compose up --build --force-recreate diff --git a/docker-compose.yaml b/docker-compose.yaml index 5061f7d..75a87ab 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -12,7 +12,9 @@ services: volumes: - ./:/var/lib/grafana/plugins/grafana-quickwit-datasource - ./provisioning:/etc/grafana/provisioning - # - ./grafana/storage:/var/lib/grafana - # - ./grafana/grafana.ini:/etc/grafana/grafana.ini + - gquickwit:/var/lib/grafana extra_hosts: - "host.docker.internal:host-gateway" + +volumes: + gquickwit: diff --git a/package-lock.json b/package-lock.json index 23e5df8..b6f2f12 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1961,17 +1961,21 @@ "license": "MIT" }, "node_modules/@babel/runtime": { - "version": "7.19.4", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.19.4.tgz", - "integrity": "sha512-EXpLCrk55f+cYqmHsSR+yD/0gAIMxxA9QK9lnQWzhMCvt+YmoBN7Zx94s++Kv0+unHk39vxNO8t+CMA2WSS3wA==", - "license": "MIT", + "version": "7.23.4", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.23.4.tgz", + "integrity": "sha512-2Yv65nlWnWlSpe3fXEyX5i7fx5kIKo4Qbcj+hMO0odwaneFjfXw5fdum+4yL20O0QiaHpia0cYQ9xpNMqrBwHg==", "dependencies": { - "regenerator-runtime": "^0.13.4" + "regenerator-runtime": "^0.14.0" }, "engines": { "node": ">=6.9.0" } }, + "node_modules/@babel/runtime/node_modules/regenerator-runtime": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz", + "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==" + }, "node_modules/@babel/template": { "version": "7.22.5", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.5.tgz", @@ -2590,7 +2594,6 @@ "version": "9.5.1", "resolved": "https://registry.npmjs.org/@grafana/data/-/data-9.5.1.tgz", "integrity": "sha512-XXOV6cSGtBYQkSlQzVG4LaCjCdM49qacajTMU7a1wOKmDY9QJsFiKTbom7vQ1hHd+X83RqjndSJJiCFt7JX9LA==", - "license": "Apache-2.0", "dependencies": { "@braintree/sanitize-url": "6.0.2", "@grafana/schema": "9.5.1", @@ -4360,18 +4363,6 @@ "react-dom": ">=16.9.0" } }, - "node_modules/@rc-component/portal/node_modules/@babel/runtime": { - "version": "7.21.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.21.5.tgz", - "integrity": "sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==", - "license": "MIT", - "dependencies": { - "regenerator-runtime": "^0.13.11" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@rc-component/portal/node_modules/rc-util": { "version": "5.30.0", "resolved": "https://registry.npmjs.org/rc-util/-/rc-util-5.30.0.tgz", @@ -9689,8 +9680,7 @@ "node_modules/eventemitter3": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.0.tgz", - "integrity": "sha512-riuVbElZZNXLeLEoprfNYoDSwTBRR44X3mnhdI1YcnENpWTCsTTVZ2zFuqQcpoyqPQIUXdiPEU0ECAq0KQRaHg==", - "license": "MIT" + "integrity": "sha512-riuVbElZZNXLeLEoprfNYoDSwTBRR44X3mnhdI1YcnENpWTCsTTVZ2zFuqQcpoyqPQIUXdiPEU0ECAq0KQRaHg==" }, "node_modules/events": { "version": "3.3.0", @@ -10830,18 +10820,6 @@ "@babel/runtime": "^7.20.6" } }, - "node_modules/i18next/node_modules/@babel/runtime": { - "version": "7.21.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.21.5.tgz", - "integrity": "sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==", - "license": "MIT", - "dependencies": { - "regenerator-runtime": "^0.13.11" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -13280,7 +13258,6 @@ "version": "4.2.12", "resolved": "https://registry.npmjs.org/marked/-/marked-4.2.12.tgz", "integrity": "sha512-yr8hSKa3Fv4D3jdZmtMMPghgVt6TWbk86WQaWhDloQjRSQhMMYCAro7jP7VDJrjjdV8pxVxMssXS8B8Y5DZ5aw==", - "license": "MIT", "bin": { "marked": "bin/marked.js" }, @@ -13740,7 +13717,6 @@ "version": "0.5.41", "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.41.tgz", "integrity": "sha512-e0jGNZDOHfBXJGz8vR/sIMXvBIGJJcqFjmlg9lmE+5KX1U7/RZNMswfD8nKnNCnQdKTIj50IaRKwl1fvMLyyRg==", - "license": "MIT", "dependencies": { "moment": "^2.29.4" }, @@ -14217,8 +14193,7 @@ "node_modules/papaparse": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.3.2.tgz", - "integrity": "sha512-6dNZu0Ki+gyV0eBsFKJhYr+MdQYAzFUGlBMNj3GNrmHxmz1lfRa24CjFObPXtjcetlOv5Ad299MhIK0znp3afw==", - "license": "MIT" + "integrity": "sha512-6dNZu0Ki+gyV0eBsFKJhYr+MdQYAzFUGlBMNj3GNrmHxmz1lfRa24CjFObPXtjcetlOv5Ad299MhIK0znp3afw==" }, "node_modules/parent-module": { "version": "1.0.1", @@ -15613,18 +15588,6 @@ "react-dom": "*" } }, - "node_modules/rc-virtual-list/node_modules/@babel/runtime": { - "version": "7.21.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.21.5.tgz", - "integrity": "sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==", - "license": "MIT", - "dependencies": { - "regenerator-runtime": "^0.13.11" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/react": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react/-/react-17.0.2.tgz", @@ -15819,18 +15782,6 @@ } } }, - "node_modules/react-i18next/node_modules/@babel/runtime": { - "version": "7.21.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.21.5.tgz", - "integrity": "sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==", - "license": "MIT", - "dependencies": { - "regenerator-runtime": "^0.13.11" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/react-immutable-proptypes": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/react-immutable-proptypes/-/react-immutable-proptypes-2.2.0.tgz", @@ -20460,11 +20411,18 @@ "dev": true }, "@babel/runtime": { - "version": "7.19.4", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.19.4.tgz", - "integrity": "sha512-EXpLCrk55f+cYqmHsSR+yD/0gAIMxxA9QK9lnQWzhMCvt+YmoBN7Zx94s++Kv0+unHk39vxNO8t+CMA2WSS3wA==", + "version": "7.23.4", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.23.4.tgz", + "integrity": "sha512-2Yv65nlWnWlSpe3fXEyX5i7fx5kIKo4Qbcj+hMO0odwaneFjfXw5fdum+4yL20O0QiaHpia0cYQ9xpNMqrBwHg==", "requires": { - "regenerator-runtime": "^0.13.4" + "regenerator-runtime": "^0.14.0" + }, + "dependencies": { + "regenerator-runtime": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz", + "integrity": "sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==" + } } }, "@babel/template": { @@ -22347,14 +22305,6 @@ "rc-util": "^5.24.4" }, "dependencies": { - "@babel/runtime": { - "version": "7.21.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.21.5.tgz", - "integrity": "sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==", - "requires": { - "regenerator-runtime": "^0.13.11" - } - }, "rc-util": { "version": "5.30.0", "resolved": "https://registry.npmjs.org/rc-util/-/rc-util-5.30.0.tgz", @@ -26961,16 +26911,6 @@ "integrity": "sha512-yYudtbFrrmWKLEhl6jvKUYyYunj4bTBCe2qIUYAxbXoPusY7YmdwPvOE6fx6UIfWvmlbCWDItr7wIs8KEBZ5Zg==", "requires": { "@babel/runtime": "^7.20.6" - }, - "dependencies": { - "@babel/runtime": { - "version": "7.21.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.21.5.tgz", - "integrity": "sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==", - "requires": { - "regenerator-runtime": "^0.13.11" - } - } } }, "iconv-lite": { @@ -30301,16 +30241,6 @@ "classnames": "^2.2.6", "rc-resize-observer": "^1.0.0", "rc-util": "^5.15.0" - }, - "dependencies": { - "@babel/runtime": { - "version": "7.21.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.21.5.tgz", - "integrity": "sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==", - "requires": { - "regenerator-runtime": "^0.13.11" - } - } } }, "react": { @@ -30433,16 +30363,6 @@ "requires": { "@babel/runtime": "^7.20.6", "html-parse-stringify": "^3.0.1" - }, - "dependencies": { - "@babel/runtime": { - "version": "7.21.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.21.5.tgz", - "integrity": "sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==", - "requires": { - "regenerator-runtime": "^0.13.11" - } - } } }, "react-immutable-proptypes": { diff --git a/pkg/quickwit/client/search_request.go b/pkg/quickwit/client/search_request.go index 1c327c7..efe268a 100644 --- a/pkg/quickwit/client/search_request.go +++ b/pkg/quickwit/client/search_request.go @@ -101,11 +101,11 @@ func (b *SearchRequestBuilder) Sort(order SortOrder, field string, unmappedType return b } -func (b *SearchRequestBuilder) AddSearchAfter(value interface{}) *SearchRequestBuilder { +func (b *SearchRequestBuilder) AddSearchAfter(value any) *SearchRequestBuilder { if b.customProps["search_after"] == nil { - b.customProps["search_after"] = []interface{}{value} + b.customProps["search_after"] = []any{value} } else { - b.customProps["search_after"] = append(b.customProps["search_after"].([]interface{}), value) + b.customProps["search_after"] = append(b.customProps["search_after"].([]any), value) } return b diff --git a/pkg/quickwit/data_query.go b/pkg/quickwit/data_query.go index 32ee1f8..96efbf8 100644 --- a/pkg/quickwit/data_query.go +++ b/pkg/quickwit/data_query.go @@ -340,6 +340,13 @@ func processLogsQuery(q *Query, b *es.SearchRequestBuilder, from, to int64, defa b.Size(stringToIntWithDefaultValue(metric.Settings.Get("limit").MustString(), defaultSize)) // TODO when hightlight is supported in quickwit // b.AddHighlight() + + // This is currently used only for log context query to get + // log lines before and after the selected log line + searchAfter := metric.Settings.Get("searchAfter").MustArray() + for _, value := range searchAfter { + b.AddSearchAfter(value) + } } func processDocumentQuery(q *Query, b *es.SearchRequestBuilder, from, to int64, defaultTimeField string) { diff --git a/src/IntervalMap.ts b/src/IntervalMap.ts new file mode 100644 index 0000000..08e1164 --- /dev/null +++ b/src/IntervalMap.ts @@ -0,0 +1,18 @@ +import { DurationUnit } from '@grafana/data'; +import { Interval } from './types'; + +type IntervalMap = Record< + Interval, + { + startOf: DurationUnit; + amount: DurationUnit; + } +>; + +export const intervalMap: IntervalMap = { + Hourly: { startOf: 'hour', amount: 'hours' }, + Daily: { startOf: 'day', amount: 'days' }, + Weekly: { startOf: 'isoWeek', amount: 'weeks' }, + Monthly: { startOf: 'month', amount: 'months' }, + Yearly: { startOf: 'year', amount: 'years' }, +}; diff --git a/src/datasource.ts b/src/datasource.ts index 59bf6fd..e17a863 100644 --- a/src/datasource.ts +++ b/src/datasource.ts @@ -1,35 +1,40 @@ import { cloneDeep, first as _first, map as _map, groupBy } from 'lodash'; import { Observable, lastValueFrom, from, isObservable, of } from 'rxjs'; import { catchError, mergeMap, map } from 'rxjs/operators'; +import { intervalMap } from './IntervalMap'; import { AbstractQuery, + CoreApp, DataFrame, + DataQueryError, DataQueryRequest, DataQueryResponse, DataSourceApi, DataSourceInstanceSettings, DataSourceJsonData, + DataSourceWithLogsContextSupport, DataSourceWithQueryImportSupport, DataSourceWithSupplementaryQueriesSupport, + dateTime, FieldColorModeId, FieldType, getDefaultTimeRange, LoadingState, LogLevel, + LogRowModel, LogsVolumeCustomMetaData, LogsVolumeType, MetricFindValue, QueryFixAction, + rangeUtil, ScopedVars, SupplementaryQueryType, TimeRange, } from '@grafana/data'; -import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, Field, FieldMapping, IndexMetadata, TermsQuery } from './types'; -import { - DataSourceWithBackend, getTemplateSrv, TemplateSrv, -} from '@grafana/runtime'; -import { QuickwitOptions } from 'quickwit'; +import { BucketAggregation, DataLinkConfig, ElasticsearchQuery, Field, FieldMapping, IndexMetadata, Logs, TermsQuery, Interval } from './types'; +import { DataSourceWithBackend, getTemplateSrv, TemplateSrv } from '@grafana/runtime'; +import { LogRowContextOptions, LogRowContextQueryDirection, QuickwitOptions } from 'quickwit'; import { ElasticQueryBuilder } from 'QueryBuilder'; import { colors } from '@grafana/ui'; @@ -39,7 +44,7 @@ import { isMetricAggregationWithField } from 'components/QueryEditor/MetricAggre import { bucketAggregationConfig } from 'components/QueryEditor/BucketAggregationsEditor/utils'; import { isBucketAggregationWithField } from 'components/QueryEditor/BucketAggregationsEditor/aggregations'; import ElasticsearchLanguageProvider from 'LanguageProvider'; - +import { ReactNode } from 'react'; export const REF_ID_STARTER_LOG_VOLUME = 'log-volume-'; @@ -48,6 +53,7 @@ export type ElasticDatasource = QuickwitDataSource; export class QuickwitDataSource extends DataSourceWithBackend implements + DataSourceWithLogsContextSupport, DataSourceWithSupplementaryQueriesSupport, DataSourceWithQueryImportSupport { @@ -59,6 +65,7 @@ export class QuickwitDataSource queryBuilder: ElasticQueryBuilder; dataLinks: DataLinkConfig[]; languageProvider: ElasticsearchLanguageProvider; + intervalPattern?: Interval; constructor( instanceSettings: DataSourceInstanceSettings, @@ -427,6 +434,78 @@ export class QuickwitDataSource return text; } + private makeLogContextDataRequest = (row: LogRowModel, options?: LogRowContextOptions) => { + const direction = options?.direction || LogRowContextQueryDirection.Backward; + const searchAfterNs = row.dataFrame.fields.find((f) => f.name === 'sort')?.values.get(row.rowIndex) ?? [row.timeEpochNs] + const searchAfterMs = [Math.round(searchAfterNs[0]/1000000)] + + const logQuery: Logs = { + type: 'logs', + id: '1', + settings: { + limit: options?.limit ? options?.limit.toString() : '10', + // Sorting of results in the context query + sortDirection: direction === LogRowContextQueryDirection.Backward ? 'desc' : 'asc', + // Used to get the next log lines before/after the current log line using sort field of selected log line + searchAfter: searchAfterMs, + }, + }; + + const query: ElasticsearchQuery = { + refId: `log-context-${row.dataFrame.refId}-${direction}`, + metrics: [logQuery], + query: '', + }; + + const timeRange = createContextTimeRange(row.timeEpochMs, direction, this.intervalPattern); + const range = { + from: timeRange.from, + to: timeRange.to, + raw: timeRange, + }; + + const interval = rangeUtil.calculateInterval(range, 1); + + const contextRequest: DataQueryRequest = { + requestId: `log-context-request-${row.dataFrame.refId}-${options?.direction}`, + targets: [query], + interval: interval.interval, + intervalMs: interval.intervalMs, + range, + scopedVars: {}, + timezone: 'UTC', + app: CoreApp.Explore, + startTime: Date.now(), + hideFromInspector: true, + }; + return contextRequest; + }; + + getLogRowContext = async (row: LogRowModel, options?: LogRowContextOptions): Promise<{ data: DataFrame[] }> => { + const contextRequest = this.makeLogContextDataRequest(row, options); + + return lastValueFrom( + this.query(contextRequest).pipe( + catchError((err) => { + const error: DataQueryError = { + message: 'Error during context query. Please check JS console logs.', + status: err.status, + statusText: err.statusText, + }; + throw error; + }) + ) + ); + }; + + showContextToggle(row?: LogRowModel | undefined): boolean { + return true; + } + + getLogRowContextUi?(row: LogRowModel, runContextQuery?: (() => void) | undefined): ReactNode { + return true; + } + /** * Returns false if the query should be skipped */ @@ -740,3 +819,36 @@ function luceneEscape(value: string) { return value.replace(/([\!\*\+\-\=<>\s\&\|\(\)\[\]\{\}\^\~\?\:\\/"])/g, '\\$1'); } + +function createContextTimeRange(rowTimeEpochMs: number, direction: string, intervalPattern: Interval | undefined) { + const offset = 7; + // For log context, we want to request data from 7 subsequent/previous indices + if (intervalPattern) { + const intervalInfo = intervalMap[intervalPattern]; + if (direction === LogRowContextQueryDirection.Forward) { + return { + from: dateTime(rowTimeEpochMs).utc(), + to: dateTime(rowTimeEpochMs).add(offset, intervalInfo.amount).utc().startOf(intervalInfo.startOf), + }; + } else { + return { + from: dateTime(rowTimeEpochMs).subtract(offset, intervalInfo.amount).utc().startOf(intervalInfo.startOf), + to: dateTime(rowTimeEpochMs).utc(), + }; + } + // If we don't have an interval pattern, we can't do this, so we just request data from 7h before/after + } else { + if (direction === LogRowContextQueryDirection.Forward) { + return { + from: dateTime(rowTimeEpochMs).utc(), + to: dateTime(rowTimeEpochMs).add(offset, 'hours').utc(), + }; + } else { + return { + from: dateTime(rowTimeEpochMs).subtract(offset, 'hours').utc(), + to: dateTime(rowTimeEpochMs).utc(), + }; + } + } +} +