From 5039dff001456902225577c5df207f5cab916774 Mon Sep 17 00:00:00 2001 From: Galen Date: Wed, 14 Feb 2024 14:44:22 -0600 Subject: [PATCH 1/7] add CSV download to logs panel and extract labels --- .../explore/Logs/LogsMetaRow.test.tsx | 74 ++++++++++++++++++- .../app/features/explore/Logs/LogsMetaRow.tsx | 38 +++++++++- .../app/features/explore/Logs/LogsTable.tsx | 4 +- 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/public/app/features/explore/Logs/LogsMetaRow.test.tsx b/public/app/features/explore/Logs/LogsMetaRow.test.tsx index 9e506d6b61f7..2840f8cefafe 100644 --- a/public/app/features/explore/Logs/LogsMetaRow.test.tsx +++ b/public/app/features/explore/Logs/LogsMetaRow.test.tsx @@ -3,10 +3,12 @@ import userEvent from '@testing-library/user-event'; import saveAs from 'file-saver'; import React, { ComponentProps } from 'react'; -import { FieldType, LogLevel, LogsDedupStrategy, toDataFrame } from '@grafana/data'; +import { FieldType, LogLevel, LogsDedupStrategy, standardTransformersRegistry, toDataFrame } from '@grafana/data'; +import { organizeFieldsTransformer } from '@grafana/data/src/transformations/transformers/organize'; import { MAX_CHARACTERS } from '../../logs/components/LogRowMessage'; import { logRowsToReadableJson } from '../../logs/utils'; +import { extractFieldsTransformer } from '../../transformers/extractFields/extractFields'; import { LogsMetaRow } from './LogsMetaRow'; @@ -200,4 +202,74 @@ describe('LogsMetaRow', () => { const text = await blob.text(); expect(text).toBe(JSON.stringify(logRowsToReadableJson(rows))); }); + + it('renders a button to download CSV', async () => { + const transformers = [extractFieldsTransformer, organizeFieldsTransformer]; + standardTransformersRegistry.setInit(() => { + return transformers.map((t) => { + return { + id: t.id, + aliasIds: t.aliasIds, + name: t.name, + transformation: t, + description: t.description, + editor: () => null, + }; + }); + }); + + const rows = [ + { + rowIndex: 1, + entryFieldIndex: 0, + dataFrame: toDataFrame({ + name: 'logs', + fields: [ + { + name: 'time', + type: FieldType.time, + values: ['1970-01-01T00:00:00Z'], + }, + { + name: 'message', + type: FieldType.string, + values: ['INFO 1'], + labels: { + foo: 'bar', + }, + }, + ], + }), + entry: 'test entry', + hasAnsi: false, + hasUnescapedContent: false, + labels: { + foo: 'bar', + }, + logLevel: LogLevel.info, + raw: '', + timeEpochMs: 10, + timeEpochNs: '123456789', + timeFromNow: '', + timeLocal: '', + timeUtc: '', + uid: '2', + }, + ]; + setup({ logRows: rows }); + + await userEvent.click(screen.getByText('Download').closest('button')!); + + await userEvent.click( + screen.getByRole('menuitem', { + name: 'csv', + }) + ); + + expect(saveAs).toBeCalled(); + const blob = (saveAs as unknown as jest.Mock).mock.lastCall[0]; + expect(blob.type).toBe('text/csv;charset=utf-8'); + const text = await blob.text(); + expect(text).toBe(`"time","message bar"\r\n1970-01-01T00:00:00Z,INFO 1`); + }); }); diff --git a/public/app/features/explore/Logs/LogsMetaRow.tsx b/public/app/features/explore/Logs/LogsMetaRow.tsx index 5bf8cf7a0341..458fc4a03da3 100644 --- a/public/app/features/explore/Logs/LogsMetaRow.tsx +++ b/public/app/features/explore/Logs/LogsMetaRow.tsx @@ -1,17 +1,30 @@ import { css } from '@emotion/css'; import saveAs from 'file-saver'; import React from 'react'; +import { lastValueFrom } from 'rxjs'; -import { LogsDedupStrategy, LogsMetaItem, LogsMetaKind, LogRowModel, CoreApp, dateTimeFormat } from '@grafana/data'; +import { + LogsDedupStrategy, + LogsMetaItem, + LogsMetaKind, + LogRowModel, + CoreApp, + dateTimeFormat, + transformDataFrame, + DataTransformerConfig, + CustomTransformOperator, +} from '@grafana/data'; import { reportInteraction } from '@grafana/runtime'; import { Button, Dropdown, Menu, ToolbarButton, Tooltip, useStyles2 } from '@grafana/ui'; -import { downloadLogsModelAsTxt } from '../../inspector/utils/download'; +import { downloadDataFrameAsCsv, downloadLogsModelAsTxt } from '../../inspector/utils/download'; import { LogLabels } from '../../logs/components/LogLabels'; import { MAX_CHARACTERS } from '../../logs/components/LogRowMessage'; import { logRowsToReadableJson } from '../../logs/utils'; import { MetaInfoText, MetaItemProps } from '../MetaInfoText'; +import { getLogsExtractFields } from './LogsTable'; + const getStyles = () => ({ metaContainer: css` flex: 1; @@ -35,6 +48,7 @@ export type Props = { enum DownloadFormat { Text = 'text', Json = 'json', + CSV = 'csv', } export const LogsMetaRow = React.memo( @@ -51,13 +65,15 @@ export const LogsMetaRow = React.memo( }: Props) => { const style = useStyles2(getStyles); - const downloadLogs = (format: DownloadFormat) => { + const downloadLogs = async (format: DownloadFormat) => { reportInteraction('grafana_logs_download_logs_clicked', { app: CoreApp.Explore, format, area: 'logs-meta-row', }); + const fileName = `Explore-logs-${dateTimeFormat(new Date())}.json`; + switch (format) { case DownloadFormat.Text: downloadLogsModelAsTxt({ meta, rows: logRows }, 'Explore'); @@ -68,9 +84,22 @@ export const LogsMetaRow = React.memo( type: 'application/json;charset=utf-8', }); - const fileName = `Explore-logs-${dateTimeFormat(new Date())}.json`; saveAs(blob, fileName); break; + case DownloadFormat.CSV: + const dataFrame = logRows[0].dataFrame; + const transforms: Array = getLogsExtractFields(dataFrame); + transforms.push({ + id: 'organize', + options: { + excludeByName: { + ['labels']: true, + ['labelTypes']: true, + }, + }, + }); + const transformedDataFrame = await lastValueFrom(transformDataFrame(transforms, [dataFrame])); + downloadDataFrameAsCsv(transformedDataFrame[0], fileName); } }; @@ -131,6 +160,7 @@ export const LogsMetaRow = React.memo( downloadLogs(DownloadFormat.Text)} /> downloadLogs(DownloadFormat.Json)} /> + downloadLogs(DownloadFormat.CSV)} /> ); return ( diff --git a/public/app/features/explore/Logs/LogsTable.tsx b/public/app/features/explore/Logs/LogsTable.tsx index 23c78fed2bf7..8f50a8d8ba8d 100644 --- a/public/app/features/explore/Logs/LogsTable.tsx +++ b/public/app/features/explore/Logs/LogsTable.tsx @@ -105,7 +105,7 @@ export function LogsTable(props: Props) { } // create extract JSON transformation for every field that is `json.RawMessage` - const transformations: Array = extractFields(dataFrame); + const transformations: Array = getLogsExtractFields(dataFrame); let labelFilters = buildLabelFilters(columnsWithMeta); @@ -197,7 +197,7 @@ const isFieldFilterable = (field: Field, bodyName: string, timeName: string) => // TODO: explore if `logsFrame.ts` can help us with getting the right fields // TODO Why is typeInfo not defined on the Field interface? -function extractFields(dataFrame: DataFrame) { +export function getLogsExtractFields(dataFrame: DataFrame) { return dataFrame.fields .filter((field: Field & { typeInfo?: { frame: string } }) => { const isFieldLokiLabels = From 09b11e3dc4fd1b8b1b09804d9571b2ba0a999369 Mon Sep 17 00:00:00 2001 From: Galen Date: Thu, 15 Feb 2024 07:52:09 -0600 Subject: [PATCH 2/7] update filename, download csv for each dataframe --- .../app/features/explore/Logs/LogsMetaRow.tsx | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/public/app/features/explore/Logs/LogsMetaRow.tsx b/public/app/features/explore/Logs/LogsMetaRow.tsx index 458fc4a03da3..0291a79da59d 100644 --- a/public/app/features/explore/Logs/LogsMetaRow.tsx +++ b/public/app/features/explore/Logs/LogsMetaRow.tsx @@ -14,6 +14,7 @@ import { DataTransformerConfig, CustomTransformOperator, } from '@grafana/data'; +import { DataFrame } from '@grafana/data/'; import { reportInteraction } from '@grafana/runtime'; import { Button, Dropdown, Menu, ToolbarButton, Tooltip, useStyles2 } from '@grafana/ui'; @@ -87,19 +88,26 @@ export const LogsMetaRow = React.memo( saveAs(blob, fileName); break; case DownloadFormat.CSV: - const dataFrame = logRows[0].dataFrame; - const transforms: Array = getLogsExtractFields(dataFrame); - transforms.push({ - id: 'organize', - options: { - excludeByName: { - ['labels']: true, - ['labelTypes']: true, + const dataFrameMap = new Map(); + logRows.forEach((row) => { + if (row.dataFrame?.refId && !dataFrameMap.has(row.dataFrame?.refId)) { + dataFrameMap.set(row.dataFrame?.refId, row.dataFrame); + } + }); + dataFrameMap.forEach(async (dataFrame) => { + const transforms: Array = getLogsExtractFields(dataFrame); + transforms.push({ + id: 'organize', + options: { + excludeByName: { + ['labels']: true, + ['labelTypes']: true, + }, }, - }, + }); + const transformedDataFrame = await lastValueFrom(transformDataFrame(transforms, [dataFrame])); + downloadDataFrameAsCsv(transformedDataFrame[0], 'Explore-logs'); }); - const transformedDataFrame = await lastValueFrom(transformDataFrame(transforms, [dataFrame])); - downloadDataFrameAsCsv(transformedDataFrame[0], fileName); } }; From 162c992bc75e99b998dafb56f6e349e823232eb4 Mon Sep 17 00:00:00 2001 From: Galen Date: Thu, 15 Feb 2024 07:53:37 -0600 Subject: [PATCH 3/7] clean up --- public/app/features/explore/Logs/LogsMetaRow.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/public/app/features/explore/Logs/LogsMetaRow.tsx b/public/app/features/explore/Logs/LogsMetaRow.tsx index 0291a79da59d..d0279ff72070 100644 --- a/public/app/features/explore/Logs/LogsMetaRow.tsx +++ b/public/app/features/explore/Logs/LogsMetaRow.tsx @@ -73,8 +73,6 @@ export const LogsMetaRow = React.memo( area: 'logs-meta-row', }); - const fileName = `Explore-logs-${dateTimeFormat(new Date())}.json`; - switch (format) { case DownloadFormat.Text: downloadLogsModelAsTxt({ meta, rows: logRows }, 'Explore'); @@ -84,7 +82,7 @@ export const LogsMetaRow = React.memo( const blob = new Blob([JSON.stringify(jsonLogs)], { type: 'application/json;charset=utf-8', }); - + const fileName = `Explore-logs-${dateTimeFormat(new Date())}.json`; saveAs(blob, fileName); break; case DownloadFormat.CSV: From 651617676e1076cf5258ffac6e4551346036fd3b Mon Sep 17 00:00:00 2001 From: Galen Date: Thu, 15 Feb 2024 07:57:06 -0600 Subject: [PATCH 4/7] add refId to filename --- public/app/features/explore/Logs/LogsMetaRow.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/features/explore/Logs/LogsMetaRow.tsx b/public/app/features/explore/Logs/LogsMetaRow.tsx index d0279ff72070..55f5bdacc028 100644 --- a/public/app/features/explore/Logs/LogsMetaRow.tsx +++ b/public/app/features/explore/Logs/LogsMetaRow.tsx @@ -104,7 +104,7 @@ export const LogsMetaRow = React.memo( }, }); const transformedDataFrame = await lastValueFrom(transformDataFrame(transforms, [dataFrame])); - downloadDataFrameAsCsv(transformedDataFrame[0], 'Explore-logs'); + downloadDataFrameAsCsv(transformedDataFrame[0], `Explore-logs-${dataFrame.refId}`); }); } }; From 8fcfc530e5c5f3b5f05d79541b294a50a24bd087 Mon Sep 17 00:00:00 2001 From: Galen Date: Thu, 15 Feb 2024 09:02:11 -0600 Subject: [PATCH 5/7] fix test --- .../explore/Logs/LogsMetaRow.test.tsx | 45 +++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/public/app/features/explore/Logs/LogsMetaRow.test.tsx b/public/app/features/explore/Logs/LogsMetaRow.test.tsx index 2840f8cefafe..e7417e520bc4 100644 --- a/public/app/features/explore/Logs/LogsMetaRow.test.tsx +++ b/public/app/features/explore/Logs/LogsMetaRow.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, render, screen } from '@testing-library/react'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import saveAs from 'file-saver'; import React, { ComponentProps } from 'react'; @@ -224,6 +224,7 @@ describe('LogsMetaRow', () => { entryFieldIndex: 0, dataFrame: toDataFrame({ name: 'logs', + refId: 'A', fields: [ { name: 'time', @@ -255,6 +256,43 @@ describe('LogsMetaRow', () => { timeUtc: '', uid: '2', }, + { + rowIndex: 2, + entryFieldIndex: 1, + dataFrame: toDataFrame({ + name: 'logs', + refId: 'B', + fields: [ + { + name: 'time', + type: FieldType.time, + values: ['1970-01-02T00:00:00Z'], + }, + { + name: 'message', + type: FieldType.string, + values: ['INFO 1'], + labels: { + foo: 'bar', + }, + }, + ], + }), + entry: 'test entry', + hasAnsi: false, + hasUnescapedContent: false, + labels: { + foo: 'bar', + }, + logLevel: LogLevel.info, + raw: '', + timeEpochMs: 10, + timeEpochNs: '123456789', + timeFromNow: '', + timeLocal: '', + timeUtc: '', + uid: '2', + }, ]; setup({ logRows: rows }); @@ -266,10 +304,11 @@ describe('LogsMetaRow', () => { }) ); - expect(saveAs).toBeCalled(); + expect(saveAs).toBeCalledTimes(2); + const blob = (saveAs as unknown as jest.Mock).mock.lastCall[0]; expect(blob.type).toBe('text/csv;charset=utf-8'); const text = await blob.text(); - expect(text).toBe(`"time","message bar"\r\n1970-01-01T00:00:00Z,INFO 1`); + expect(text).toBe(`"time","message bar"\r\n1970-01-02T00:00:00Z,INFO 1`); }); }); From 1645ae7f68aa8a1462993ed67ac0f2de18fc335b Mon Sep 17 00:00:00 2001 From: Galen Date: Thu, 15 Feb 2024 09:03:23 -0600 Subject: [PATCH 6/7] optimize imports --- public/app/features/explore/Logs/LogsMetaRow.test.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/app/features/explore/Logs/LogsMetaRow.test.tsx b/public/app/features/explore/Logs/LogsMetaRow.test.tsx index e7417e520bc4..e957f08ee906 100644 --- a/public/app/features/explore/Logs/LogsMetaRow.test.tsx +++ b/public/app/features/explore/Logs/LogsMetaRow.test.tsx @@ -1,4 +1,4 @@ -import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { fireEvent, render, screen } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import saveAs from 'file-saver'; import React, { ComponentProps } from 'react'; From 472844f1a91087649892bd47462c65525f6f0962 Mon Sep 17 00:00:00 2001 From: Galen Date: Thu, 15 Feb 2024 09:28:06 -0600 Subject: [PATCH 7/7] remove assertion of how many times saveAs was called since it changes depending on test execution context --- public/app/features/explore/Logs/LogsMetaRow.test.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/public/app/features/explore/Logs/LogsMetaRow.test.tsx b/public/app/features/explore/Logs/LogsMetaRow.test.tsx index e957f08ee906..83755343cdbb 100644 --- a/public/app/features/explore/Logs/LogsMetaRow.test.tsx +++ b/public/app/features/explore/Logs/LogsMetaRow.test.tsx @@ -303,8 +303,7 @@ describe('LogsMetaRow', () => { name: 'csv', }) ); - - expect(saveAs).toBeCalledTimes(2); + expect(saveAs).toBeCalled(); const blob = (saveAs as unknown as jest.Mock).mock.lastCall[0]; expect(blob.type).toBe('text/csv;charset=utf-8');