From 9c91f99b7dc8ecaef1240b2d43249c97a80eaf4d Mon Sep 17 00:00:00 2001 From: balaji-jr Date: Wed, 14 Aug 2024 14:57:15 +0530 Subject: [PATCH] use fields from query response as table headers --- src/@types/parseable/api/query.ts | 5 ++ src/api/cluster.ts | 2 +- src/api/constants.ts | 19 +++++++- src/api/query.ts | 47 +++++++++++-------- src/hooks/useGetLogStreamSchema.ts | 4 -- src/hooks/useQueryLogs.ts | 45 ++++++++++-------- src/hooks/useQueryResult.tsx | 4 +- .../Stream/components/EventTimeLineGraph.tsx | 37 +++++++++------ src/pages/Stream/index.tsx | 2 +- src/pages/Stream/providers/LogsProvider.tsx | 41 ++++------------ 10 files changed, 111 insertions(+), 95 deletions(-) diff --git a/src/@types/parseable/api/query.ts b/src/@types/parseable/api/query.ts index 0867f592..a63f45e7 100644 --- a/src/@types/parseable/api/query.ts +++ b/src/@types/parseable/api/query.ts @@ -34,6 +34,11 @@ export type Log = { [key: string]: string | number | null | Date; }; +export type LogsResponseWithHeaders = { + fields: string[]; + records: Log[]; +} | null; + export type LogSelectedTimeRange = { state: 'fixed' | 'custom'; value: string; diff --git a/src/api/cluster.ts b/src/api/cluster.ts index dfac2d6d..efebff68 100644 --- a/src/api/cluster.ts +++ b/src/api/cluster.ts @@ -10,7 +10,7 @@ export const getIngestorInfo = (domain_name: string | null, startTime: Date, end const query = `SELECT * FROM pmeta where address = '${domain_name}' ORDER BY event_time DESC LIMIT 10 OFFSET 0`; return Axios().post( - LOG_QUERY_URL, + LOG_QUERY_URL(), { query, startTime, diff --git a/src/api/constants.ts b/src/api/constants.ts index de47f721..111a48c9 100644 --- a/src/api/constants.ts +++ b/src/api/constants.ts @@ -1,9 +1,26 @@ +import _ from 'lodash'; + const API_V1 = 'api/v1'; +export type Params = Record | null | {} | undefined; + +const parseParamsToQueryString = (params: Params) => { + if (_.isEmpty(params) || _.isNil(params) || !params) return ''; + + return _.reduce( + params, + (acc, value, key) => { + const slugPartPrefix = acc === '?' ? '' : '&'; + return acc + slugPartPrefix + key + '=' + value; + }, + '?', + ); +}; + // Streams Management export const LOG_STREAM_LIST_URL = `${API_V1}/logstream`; export const LOG_STREAMS_SCHEMA_URL = (streamName: string) => `${LOG_STREAM_LIST_URL}/${streamName}/schema`; -export const LOG_QUERY_URL = `${API_V1}/query`; +export const LOG_QUERY_URL = (params?: Params) => `${API_V1}/query` + parseParamsToQueryString(params); export const LOG_STREAMS_ALERTS_URL = (streamName: string) => `${LOG_STREAM_LIST_URL}/${streamName}/alert`; export const LIST_SAVED_FILTERS_URL = (userId: string) => `${API_V1}/filters/${userId}`; export const UPDATE_SAVED_FILTERS_URL = (filterId: string) => `${API_V1}/filters/filter/${filterId}`; diff --git a/src/api/query.ts b/src/api/query.ts index 177f665e..c7c819c0 100644 --- a/src/api/query.ts +++ b/src/api/query.ts @@ -1,7 +1,7 @@ import dayjs from 'dayjs'; import { Axios } from './axios'; import { LOG_QUERY_URL } from './constants'; -import { LogsQuery } from '@/@types/parseable/api/query'; +import { Log, LogsQuery, LogsResponseWithHeaders } from '@/@types/parseable/api/query'; type QueryLogs = { streamName: string; @@ -11,38 +11,47 @@ type QueryLogs = { pageOffset: number; }; -// to optimize performace, it has been decided to round off the time at the given level +// to optimize query performace, it has been decided to round off the time at the given level // so making the end-time inclusive const optimizeEndTime = (endTime: Date) => { return dayjs(endTime).add(1, 'minute').toDate(); }; -export const getQueryLogs = (logsQuery: QueryLogs) => { - const { startTime, endTime, streamName, limit, pageOffset } = logsQuery; +// ------ Default sql query +const makeDefaultQueryRequestData = (logsQuery: QueryLogs) => { + const { startTime, endTime, streamName, limit, pageOffset } = logsQuery; const query = `SELECT * FROM ${streamName} LIMIT ${limit} OFFSET ${pageOffset}`; + return { query, startTime, endTime: optimizeEndTime(endTime) }; +}; + +export const getQueryLogs = (logsQuery: QueryLogs) => { + return Axios().post(LOG_QUERY_URL(), makeDefaultQueryRequestData(logsQuery), {}); +}; - return Axios().post( - LOG_QUERY_URL, - { - query, - startTime, - endTime: optimizeEndTime(endTime), - }, +export const getQueryLogsWithHeaders = (logsQuery: QueryLogs) => { + return Axios().post( + LOG_QUERY_URL({ fields: true }), + makeDefaultQueryRequestData(logsQuery), {}, ); }; -export const getQueryResult = (logsQuery: LogsQuery, query = '') => { +// ------ Custom sql query + +const makeCustomQueryRequestData = (logsQuery: LogsQuery, query: string) => { const { startTime, endTime } = logsQuery; + return { query, startTime, endTime: optimizeEndTime(endTime) }; +}; + +export const getQueryResult = (logsQuery: LogsQuery, query = '') => { + return Axios().post(LOG_QUERY_URL(), makeCustomQueryRequestData(logsQuery, query), {}); +}; - return Axios().post( - LOG_QUERY_URL, - { - query, - startTime, - endTime: optimizeEndTime(endTime), - }, +export const getQueryResultWithHeaders = (logsQuery: LogsQuery, query = '') => { + return Axios().post( + LOG_QUERY_URL({ fields: true }), + makeCustomQueryRequestData(logsQuery, query), {}, ); }; diff --git a/src/hooks/useGetLogStreamSchema.ts b/src/hooks/useGetLogStreamSchema.ts index 4ea0ef90..1e784f8c 100644 --- a/src/hooks/useGetLogStreamSchema.ts +++ b/src/hooks/useGetLogStreamSchema.ts @@ -4,20 +4,17 @@ import { StatusCodes } from 'http-status-codes'; import useMountedState from './useMountedState'; import { Field } from '@/@types/parseable/dataType'; import { useAppStore } from '@/layouts/MainLayout/providers/AppProvider'; -import { useLogsStore, logsStoreReducers } from '@/pages/Stream/providers/LogsProvider'; import { useStreamStore, streamStoreReducers } from '@/pages/Stream/providers/StreamProvider'; import { AxiosError } from 'axios'; import _ from 'lodash'; const { setStreamSchema } = streamStoreReducers; -const { setTableHeaders } = logsStoreReducers; export const useGetLogStreamSchema = () => { const [data, setData] = useMountedState(null); const [error, setError] = useMountedState(null); const [loading, setLoading] = useMountedState(false); const [currentStream] = useAppStore((store) => store.currentStream); - const [, setLogsStore] = useLogsStore((_store) => null); const [, setStreamStore] = useStreamStore((_store) => null); const getDataSchema = async (stream: string | null = currentStream) => { @@ -34,7 +31,6 @@ export const useGetLogStreamSchema = () => { setData(schema); setStreamStore((store) => setStreamSchema(store, schema)); - setLogsStore((store) => setTableHeaders(store, schema)); break; } default: { diff --git a/src/hooks/useQueryLogs.ts b/src/hooks/useQueryLogs.ts index e8bdc379..e6cb9201 100644 --- a/src/hooks/useQueryLogs.ts +++ b/src/hooks/useQueryLogs.ts @@ -1,5 +1,5 @@ import { SortOrder, type Log, type LogsData, type LogsSearch } from '@/@types/parseable/api/query'; -import { getQueryLogs, getQueryResult } from '@/api/query'; +import { getQueryLogsWithHeaders, getQueryResultWithHeaders } from '@/api/query'; import { StatusCodes } from 'http-status-codes'; import useMountedState from './useMountedState'; import { useCallback, useEffect, useRef } from 'react'; @@ -7,11 +7,11 @@ import { useLogsStore, logsStoreReducers, LOAD_LIMIT, isJqSearch } from '@/pages import { useAppStore } from '@/layouts/MainLayout/providers/AppProvider'; import { useQueryResult } from './useQueryResult'; import _ from 'lodash'; -import { useStreamStore } from '@/pages/Stream/providers/StreamProvider'; import { AxiosError } from 'axios'; import jqSearch from '@/utils/jqSearch'; +import { useGetLogStreamSchema } from './useGetLogStreamSchema'; -const { setData, setTotalCount } = logsStoreReducers; +const { setLogData, setTotalCount } = logsStoreReducers; type QueryLogs = { streamName: string; @@ -34,6 +34,7 @@ export const useQueryLogs = () => { const [loading, setLoading] = useMountedState(false); const [isFetchingCount, setIsFetchingCount] = useMountedState(false); const [pageLogData, setPageLogData] = useMountedState(null); + const { getDataSchema } = useGetLogStreamSchema(); const [querySearch, setQuerySearch] = useMountedState({ search: '', filters: {}, @@ -43,7 +44,6 @@ export const useQueryLogs = () => { }, }); const [currentStream] = useAppStore((store) => store.currentStream); - const [schema] = useStreamStore((store) => store.schema); const [ { timeRange, @@ -86,36 +86,41 @@ export const useQueryLogs = () => { try { setLoading(true); setError(null); - + getDataSchema(); // fetch schema parallelly every time we fetch logs const logsQueryRes = isQuerySearchActive - ? await getQueryResult({ ...logsQuery, access: [] }, appendOffsetToQuery(custSearchQuery, logsQuery.pageOffset)) - : await getQueryLogs(logsQuery); + ? await getQueryResultWithHeaders( + { ...logsQuery, access: [] }, + appendOffsetToQuery(custSearchQuery, logsQuery.pageOffset), + ) + : await getQueryLogsWithHeaders(logsQuery); - const data = logsQueryRes.data; + const logs = logsQueryRes.data; + const isInvalidResponse = _.isEmpty(logs) || _.isNil(logs) || logsQueryRes.status !== StatusCodes.OK; + if (isInvalidResponse) return setError('Failed to query log'); - if (logsQueryRes.status === StatusCodes.OK) { - const jqFilteredData = isJqSearch(instantSearchValue) ? await jqSearch(data, instantSearchValue) : []; - return setLogsStore((store) => setData(store, data, schema, jqFilteredData)); - } - if (typeof data === 'string' && data.includes('Stream is not initialized yet')) { - return setLogsStore((store) => setData(store, [], schema)); - } - setError('Failed to query log'); + const { records, fields } = logs; + const jqFilteredData = isJqSearch(instantSearchValue) ? await jqSearch(records, instantSearchValue) : []; + return setLogsStore((store) => setLogData(store, records, fields, jqFilteredData)); } catch (e) { const axiosError = e as AxiosError; const errorMessage = axiosError?.response?.data; setError(_.isString(errorMessage) && !_.isEmpty(errorMessage) ? errorMessage : 'Failed to query log'); - return setLogsStore((store) => setData(store, [], schema)); + return setLogsStore((store) => setLogData(store, [], [])); } finally { setLoading(false); } }; + // fetchQueryMutation is used only on fetching count + // refactor this hook if you want to use mutation anywhere else const { fetchQueryMutation } = useQueryResult(); + useEffect(() => { - const response = _.first(fetchQueryMutation?.data) as { count: number }; - if (response) { - setLogsStore((store) => setTotalCount(store, response?.count)); + const { fields = [], records = [] } = fetchQueryMutation.data || {}; + const firstRecord = _.first(records); + if (_.includes(fields, 'count') && _.includes(_.keys(firstRecord), 'count')) { + const count = _.get(firstRecord, 'count', 0); + setLogsStore((store) => setTotalCount(store, _.toInteger(count))); } }, [fetchQueryMutation.data]); diff --git a/src/hooks/useQueryResult.tsx b/src/hooks/useQueryResult.tsx index f1ecd426..511b9f94 100644 --- a/src/hooks/useQueryResult.tsx +++ b/src/hooks/useQueryResult.tsx @@ -1,4 +1,4 @@ -import { getQueryResult } from '@/api/query'; +import { getQueryResultWithHeaders } from '@/api/query'; import { LogsQuery } from '@/@types/parseable/api/query'; import { notifications } from '@mantine/notifications'; import { isAxiosError, AxiosError } from 'axios'; @@ -13,7 +13,7 @@ type QueryData = { export const useQueryResult = () => { const fetchQueryHandler = async (data: QueryData) => { - const response = await getQueryResult(data.logsQuery, data.query); + const response = await getQueryResultWithHeaders(data.logsQuery, data.query); if (response.status !== 200) { throw new Error(response.statusText); } diff --git a/src/pages/Stream/components/EventTimeLineGraph.tsx b/src/pages/Stream/components/EventTimeLineGraph.tsx index abf31305..c35c57a9 100644 --- a/src/pages/Stream/components/EventTimeLineGraph.tsx +++ b/src/pages/Stream/components/EventTimeLineGraph.tsx @@ -8,6 +8,8 @@ import { HumanizeNumber } from '@/utils/formatBytes'; import { logsStoreReducers, useLogsStore } from '../providers/LogsProvider'; import { useAppStore } from '@/layouts/MainLayout/providers/AppProvider'; import { useFilterStore, filterStoreReducers } from '../providers/FilterProvider'; +import { LogsResponseWithHeaders } from '@/@types/parseable/api/query'; +import _ from 'lodash'; const { setTimeRange } = logsStoreReducers; const { parseQuery } = filterStoreReducers; @@ -156,29 +158,36 @@ const NoDataView = () => { ); }; -type GraphRecord = { - date_bin_timestamp: string; - log_count: number; -}; +const calcAverage = (data: LogsResponseWithHeaders | undefined) => { + if (!data || !Array.isArray(data?.records)) return 0; -const calcAverage = (data: GraphRecord[]) => { - if (!Array.isArray(data) || data.length === 0) return 0; + const { fields, records } = data; + if (_.isEmpty(records) || !_.includes(fields, 'log_count')) return 0; - const total = data.reduce((acc, d) => { - return acc + d.log_count; + const total = records.reduce((acc, d) => { + return acc + _.toNumber(d.log_count) || 0; }, 0); - return parseInt(Math.abs(total / data.length).toFixed(0)); + return parseInt(Math.abs(total / records.length).toFixed(0)); }; // date_bin removes tz info // filling data with empty values where there is no rec -const parseGraphData = (data: GraphRecord[] = [], avg: number, startTime: Date, endTime: Date, interval: number) => { - if (!Array.isArray(data)) return []; - const { modifiedEndTime, modifiedStartTime, compactType } = getModifiedTimeRange(startTime, endTime, interval); +const parseGraphData = ( + data: LogsResponseWithHeaders | undefined, + avg: number, + startTime: Date, + endTime: Date, + interval: number, +) => { + if (!data || !Array.isArray(data?.records)) return []; + const { fields, records } = data; + if (_.isEmpty(records) || !_.includes(fields, 'log_count') || !_.includes(fields, 'date_bin_timestamp')) return []; + + const { modifiedEndTime, modifiedStartTime, compactType } = getModifiedTimeRange(startTime, endTime, interval); const allTimestamps = getAllIntervals(modifiedStartTime, modifiedEndTime, compactType); const parsedData = allTimestamps.map((ts) => { - const countData = data.find((d) => { + const countData = records.find((d) => { return new Date(`${d.date_bin_timestamp}Z`).toISOString() === ts.toISOString(); }); @@ -190,7 +199,7 @@ const parseGraphData = (data: GraphRecord[] = [], avg: number, startTime: Date, compactType, }; } else { - const aboveAvgCount = countData.log_count - avg; + const aboveAvgCount = _.toNumber(countData.log_count) - avg; const aboveAvgPercent = parseInt(((aboveAvgCount / avg) * 100).toFixed(2)); return { events: countData.log_count, diff --git a/src/pages/Stream/index.tsx b/src/pages/Stream/index.tsx index cf0af849..13075514 100644 --- a/src/pages/Stream/index.tsx +++ b/src/pages/Stream/index.tsx @@ -50,7 +50,7 @@ const Logs: FC = () => { }, [currentStream]); useEffect(() => { - if (!_.isEmpty(currentStream)) { + if (!_.isEmpty(currentStream) && view !== 'explore') { fetchSchema(); } }, [currentStream]); diff --git a/src/pages/Stream/providers/LogsProvider.tsx b/src/pages/Stream/providers/LogsProvider.tsx index 7b9649ae..b573ec89 100644 --- a/src/pages/Stream/providers/LogsProvider.tsx +++ b/src/pages/Stream/providers/LogsProvider.tsx @@ -4,7 +4,7 @@ import { FIXED_DURATIONS, FixedDuration } from '@/constants/timeConstants'; import initContext from '@/utils/initContext'; import dayjs, { Dayjs } from 'dayjs'; import { addOrRemoveElement } from '@/utils'; -import { getPageSlice, makeHeadersFromSchema, makeHeadersfromData } from '../utils'; +import { getPageSlice } from '../utils'; import _ from 'lodash'; import { sanitizeCSVData } from '@/utils/exportHelpers'; @@ -260,7 +260,7 @@ type LogsStoreReducers = { getCleanStoreForRefetch: (store: LogsStore) => ReducerOutput; // data reducers - setData: (store: LogsStore, data: Log[], schema: LogStreamSchemaData | null, jqFilteredData?: Log[]) => ReducerOutput; + setLogData: (store: LogsStore, data: Log[], headers: string[], jqFilteredData?: Log[]) => ReducerOutput; setStreamSchema: (store: LogsStore, schema: LogStreamSchemaData) => ReducerOutput; applyCustomQuery: ( store: LogsStore, @@ -272,7 +272,6 @@ type LogsStoreReducers = { getUniqueValues: (data: Log[], key: string) => string[]; makeExportData: (data: Log[], headers: string[], type: string) => Log[]; setRetention: (store: LogsStore, retention: { description: string; duration: string }) => ReducerOutput; - setTableHeaders: (store: LogsStore, schema: LogStreamSchemaData) => ReducerOutput; setCleanStoreForStreamChange: (store: LogsStore) => ReducerOutput; updateSavedFilterId: (store: LogsStore, savedFilterId: string | null) => ReducerOutput; @@ -501,7 +500,7 @@ const filterAndSortData = ( return doesMatch ? [...acc, d] : acc; }, [], - ) as Log[]); + ) as Log[]); const sortedData = _.orderBy(filteredData, [sortKey], [sortOrder]); return sortedData; }; @@ -522,37 +521,17 @@ const searchAndSortData = (opts: { searchValue: string }, data: Log[]) => { return doesMatch ? [...acc, d] : acc; }, [], - ) as Log[]); + ) as Log[]); const sortedData = _.orderBy(filteredData, [defaultSortKey], [defaultSortOrder]); return sortedData; }; -const setTableHeaders = (store: LogsStore, schema: LogStreamSchemaData) => { - const { data: existingData, custQuerySearchState, tableOpts } = store; - const { filteredData } = existingData; - const newHeaders = - filteredData && custQuerySearchState.isQuerySearchActive - ? makeHeadersfromData(filteredData) - : makeHeadersFromSchema(schema); - return { - tableOpts: { - ...tableOpts, - headers: newHeaders, - }, - }; -}; - export const isJqSearch = (value: string) => { return _.startsWith(value, 'jq .'); }; -const setData = (store: LogsStore, data: Log[], schema: LogStreamSchemaData | null, jqFilteredData?: Log[]) => { - const { - data: existingData, - tableOpts, - custQuerySearchState: { isQuerySearchActive, activeMode }, - viewMode, - } = store; +const setLogData = (store: LogsStore, data: Log[], headers: string[], jqFilteredData?: Log[]) => { + const { data: existingData, tableOpts, viewMode } = store; const isJsonView = viewMode === 'json'; const currentPage = tableOpts.currentPage === 0 ? 1 : tableOpts.currentPage; const filteredData = @@ -562,14 +541,11 @@ const setData = (store: LogsStore, data: Log[], schema: LogStreamSchemaData | nu : searchAndSortData({ searchValue: tableOpts.instantSearchValue }, data) : filterAndSortData(tableOpts, data); const newPageSlice = filteredData && getPageSlice(currentPage, tableOpts.perPage, filteredData); - const newHeaders = - isQuerySearchActive && activeMode === 'sql' ? makeHeadersfromData(data) : makeHeadersFromSchema(schema); - return { tableOpts: { ...store.tableOpts, ...(newPageSlice ? { pageData: newPageSlice } : {}), - ...(!_.isEmpty(newHeaders) ? { headers: newHeaders } : {}), + headers, currentPage, totalPages: getTotalPages(filteredData, tableOpts.perPage), }, @@ -915,7 +891,7 @@ const logsStoreReducers: LogsStoreReducers = { toggleDeleteModal, toggleDisabledColumns, togglePinnedColumns, - setData, + setLogData, setStreamSchema, setPerPage, setCurrentPage, @@ -933,7 +909,6 @@ const logsStoreReducers: LogsStoreReducers = { setRetention, setCleanStoreForStreamChange, toggleSideBar, - setTableHeaders, updateSavedFilterId, setInstantSearchValue, applyInstantSearch,