diff --git a/src/api/constants.ts b/src/api/constants.ts index edace540..ee23796b 100644 --- a/src/api/constants.ts +++ b/src/api/constants.ts @@ -21,6 +21,7 @@ const parseParamsToQueryString = (params: Params) => { export const LOG_STREAM_LIST_URL = `${API_V1}/logstream`; export const LOG_STREAMS_SCHEMA_URL = (streamName: string) => `${LOG_STREAM_LIST_URL}/${streamName}/schema`; export const LOG_QUERY_URL = (params?: Params) => `${API_V1}/query` + parseParamsToQueryString(params); +export const LOG_TRINO_QUERY_URL = (params?: Params) => `${API_V1}/trinoquery` + parseParamsToQueryString(params); export const LOG_STREAMS_ALERTS_URL = (streamName: string) => `${LOG_STREAM_LIST_URL}/${streamName}/alert`; export const LIST_SAVED_FILTERS_URL = (userId: string) => `${API_V1}/filters/${userId}`; export const LIST_DASHBOARDS = (userId: string) => `${API_V1}/dashboards/${userId}`; diff --git a/src/api/query.ts b/src/api/query.ts index 76f300d7..f5fe8d4d 100644 --- a/src/api/query.ts +++ b/src/api/query.ts @@ -1,7 +1,10 @@ +import _ from 'lodash'; import { Axios } from './axios'; -import { LOG_QUERY_URL } from './constants'; +import { LOG_QUERY_URL, LOG_TRINO_QUERY_URL } from './constants'; import { Log, LogsQuery, LogsResponseWithHeaders } from '@/@types/parseable/api/query'; +import timeRangeUtils from '@/utils/timeRangeUtils'; +const { formatDateAsCastType } = timeRangeUtils; type QueryLogs = { streamName: string; startTime: Date; @@ -20,22 +23,34 @@ const optimizeTime = (date: Date) => { // ------ Default sql query -const makeDefaultQueryRequestData = (logsQuery: QueryLogs) => { - const { startTime, endTime, streamName, limit, pageOffset } = logsQuery; - const query = `SELECT * FROM ${streamName} LIMIT ${limit} OFFSET ${pageOffset}`; - return { query, startTime: optimizeTime(startTime), endTime: optimizeTime(endTime) }; +type FormQueryOptsType = Omit & { + pageOffset?: number; + timePartitionColumn?: string; +}; + +export const timeRangeSQLCondition = (timePartitionColumn: string, startTime: Date, endTime: Date) => { + return `${timePartitionColumn} >= CAST('${formatDateAsCastType( + optimizeTime(startTime), + )}' AS TIMESTAMP) and ${timePartitionColumn} < CAST('${formatDateAsCastType(optimizeTime(endTime))}' AS TIMESTAMP)`; +}; + +export const formQueryOpts = (logsQuery: FormQueryOptsType) => { + const { startTime, endTime, streamName, limit, pageOffset, timePartitionColumn = 'p_timestamp' } = logsQuery; + const optimizedStartTime = optimizeTime(startTime); + const optimizedEndTime = optimizeTime(endTime); + const orderBy = `ORDER BY ${timePartitionColumn} desc`; + const timestampClause = timeRangeSQLCondition(timePartitionColumn, optimizedStartTime, optimizedEndTime); + const offsetPart = _.isNumber(pageOffset) ? `OFFSET ${pageOffset}` : ''; + const query = `SELECT * FROM ${streamName} where ${timestampClause} ${orderBy} ${offsetPart} LIMIT ${limit} `; + return { query, startTime: optimizedStartTime, endTime: optimizedEndTime }; }; export const getQueryLogs = (logsQuery: QueryLogs) => { - return Axios().post(LOG_QUERY_URL(), makeDefaultQueryRequestData(logsQuery), {}); + return Axios().post(LOG_TRINO_QUERY_URL(), formQueryOpts(logsQuery), {}); }; export const getQueryLogsWithHeaders = (logsQuery: QueryLogs) => { - return Axios().post( - LOG_QUERY_URL({ fields: true }), - makeDefaultQueryRequestData(logsQuery), - {}, - ); + return Axios().post(LOG_TRINO_QUERY_URL({ fields: true }), formQueryOpts(logsQuery), {}); }; // ------ Custom sql query @@ -45,14 +60,12 @@ const makeCustomQueryRequestData = (logsQuery: LogsQuery, query: string) => { return { query, startTime: optimizeTime(startTime), endTime: optimizeTime(endTime) }; }; -export const getQueryResult = (logsQuery: LogsQuery, query = '') => { - return Axios().post(LOG_QUERY_URL(), makeCustomQueryRequestData(logsQuery, query), {}); +export const getQueryResult = (logsQuery: LogsQuery, query = '', useTrino = true) => { + const endPoint = useTrino ? LOG_TRINO_QUERY_URL() : LOG_QUERY_URL(); + return Axios().post(endPoint, makeCustomQueryRequestData(logsQuery, query), {}); }; -export const getQueryResultWithHeaders = (logsQuery: LogsQuery, query = '') => { - return Axios().post( - LOG_QUERY_URL({ fields: true }), - makeCustomQueryRequestData(logsQuery, query), - {}, - ); +export const getQueryResultWithHeaders = (logsQuery: LogsQuery, query = '', useTrino = true) => { + const endPoint = useTrino ? LOG_TRINO_QUERY_URL({ fields: true }) : LOG_QUERY_URL({ fields: true }); + return Axios().post(endPoint, makeCustomQueryRequestData(logsQuery, query), {}); }; diff --git a/src/hooks/useGetStreamInfo.tsx b/src/hooks/useGetStreamInfo.tsx index 76530e3b..e2aab68c 100644 --- a/src/hooks/useGetStreamInfo.tsx +++ b/src/hooks/useGetStreamInfo.tsx @@ -14,6 +14,7 @@ export const useGetStreamInfo = (currentStream: string) => { isSuccess: getStreamInfoSuccess, isLoading: getStreamInfoLoading, refetch: getStreamInfoRefetch, + isRefetching: getStreamInfoRetching } = useQuery(['stream-info', currentStream], () => getLogStreamInfo(currentStream), { retry: false, refetchOnWindowFocus: false, @@ -38,5 +39,6 @@ export const useGetStreamInfo = (currentStream: string) => { getStreamInfoSuccess, getStreamInfoLoading, getStreamInfoRefetch, + getStreamInfoRetching }; }; diff --git a/src/hooks/useQueryLogs.ts b/src/hooks/useQueryLogs.ts index 19efdece..35f35738 100644 --- a/src/hooks/useQueryLogs.ts +++ b/src/hooks/useQueryLogs.ts @@ -9,16 +9,11 @@ import _ from 'lodash'; import { AxiosError } from 'axios'; import jqSearch from '@/utils/jqSearch'; import { useGetStreamSchema } from '@/hooks/useGetLogStreamSchema'; +import { useStreamStore } from '@/pages/Stream/providers/StreamProvider'; +import { useFilterStore, filterStoreReducers } from '@/pages/Stream/providers/FilterProvider'; const { setLogData } = logsStoreReducers; - -type QueryLogs = { - streamName: string; - startTime: Date; - endTime: Date; - limit: number; - pageOffset: number; -}; +const { parseQuery } = filterStoreReducers; const appendOffsetToQuery = (query: string, offset: number) => { const hasOffset = query.toLowerCase().includes('offset'); @@ -40,8 +35,9 @@ export const useQueryLogs = () => { order: SortOrder.DESCENDING, }, }); - + const [streamInfo] = useStreamStore((store) => store.info); const [currentStream] = useAppStore((store) => store.currentStream); + const timePartitionColumn = _.get(streamInfo, 'time_partition', 'p_timestamp'); const { refetch: refetchSchema } = useGetStreamSchema({ streamName: currentStream || '' }); const [ { @@ -51,7 +47,8 @@ export const useQueryLogs = () => { }, setLogsStore, ] = useLogsStore((store) => store); - const { isQuerySearchActive, custSearchQuery } = custQuerySearchState; + const [appliedQuery] = useFilterStore((store) => store.appliedQuery); + const { isQuerySearchActive, custSearchQuery, activeMode } = custQuerySearchState; const getColumnFilters = useCallback( (columnName: string) => { @@ -80,19 +77,31 @@ export const useQueryLogs = () => { endTime: timeRange.endTime, limit: LOAD_LIMIT, pageOffset: currentOffset, + timePartitionColumn, }; - const getQueryData = async (logsQuery: QueryLogs = defaultQueryOpts) => { + const getQueryData = async () => { try { setLoading(true); setError(null); refetchSchema(); // fetch schema parallelly every time we fetch logs - const logsQueryRes = isQuerySearchActive - ? await getQueryResultWithHeaders( - { ...logsQuery, access: [] }, - appendOffsetToQuery(custSearchQuery, logsQuery.pageOffset), - ) - : await getQueryLogsWithHeaders(logsQuery); - + const logsQueryRes = await (async () => { + if (isQuerySearchActive) { + if (activeMode === 'filters') { + const { parsedQuery } = parseQuery(appliedQuery, currentStream || '', { + startTime: timeRange.startTime, + endTime: timeRange.endTime, + timePartitionColumn, + }); + const queryStrWithOffset = appendOffsetToQuery(parsedQuery, defaultQueryOpts.pageOffset); + return await getQueryResultWithHeaders({ ...defaultQueryOpts, access: [] }, queryStrWithOffset); + } else { + const queryStrWithOffset = appendOffsetToQuery(custSearchQuery, defaultQueryOpts.pageOffset); + return await getQueryResultWithHeaders({ ...defaultQueryOpts, access: [] }, queryStrWithOffset); + } + } else { + return await getQueryLogsWithHeaders(defaultQueryOpts); + } + })(); const logs = logsQueryRes.data; const isInvalidResponse = _.isEmpty(logs) || _.isNil(logs) || logsQueryRes.status !== StatusCodes.OK; if (isInvalidResponse) return setError('Failed to query log'); diff --git a/src/hooks/useQueryResult.tsx b/src/hooks/useQueryResult.tsx index 81b05604..f2155b8f 100644 --- a/src/hooks/useQueryResult.tsx +++ b/src/hooks/useQueryResult.tsx @@ -13,11 +13,12 @@ type QueryData = { logsQuery: LogsQuery; query: string; onSuccess?: () => void; + useTrino?: boolean; }; export const useQueryResult = () => { const fetchQueryHandler = async (data: QueryData) => { - const response = await getQueryResultWithHeaders(data.logsQuery, data.query); + const response = await getQueryResultWithHeaders(data.logsQuery, data.query, data.useTrino); if (response.status !== 200) { throw new Error(response.statusText); } @@ -54,13 +55,21 @@ export const useFetchCount = () => { const { setTotalCount } = logsStoreReducers; const [custQuerySearchState] = useLogsStore((store) => store.custQuerySearchState); const [timeRange, setLogsStore] = useLogsStore((store) => store.timeRange); - const { isQuerySearchActive, custSearchQuery } = custQuerySearchState; + const { isQuerySearchActive, custSearchQuery, activeMode } = custQuerySearchState; const defaultQuery = `select count(*) as count from ${currentStream}`; - const query = isQuerySearchActive - ? custSearchQuery.replace(/SELECT[\s\S]*?FROM/i, 'SELECT COUNT(*) as count FROM') - : defaultQuery; - + const query = (() => { + if (isQuerySearchActive) { + const finalQuery = custSearchQuery.replace(/SELECT[\s\S]*?FROM/i, 'SELECT COUNT(*) as count FROM'); + if (activeMode === 'filters') { + return finalQuery; + } else { + return finalQuery.replace(/ORDER\s+BY\s+[\w\s,.]+(?:ASC|DESC)?\s*(LIMIT\s+\d+)?\s*;?/i, ''); + } + } else { + return defaultQuery; + } + })(); const logsQuery = { streamName: currentStream || '', startTime: timeRange.startTime, @@ -71,7 +80,7 @@ export const useFetchCount = () => { isLoading: isCountLoading, isRefetching: isCountRefetching, refetch: refetchCount, - } = useQuery(['fetchCount', logsQuery], () => getQueryResult(logsQuery, query), { + } = useQuery(['fetchCount', logsQuery], () => getQueryResult(logsQuery, query, false), { onSuccess: (resp) => { const count = _.first(resp.data)?.count; typeof count === 'number' && setLogsStore((store) => setTotalCount(store, count)); diff --git a/src/pages/Stream/Views/Explore/LogsView.tsx b/src/pages/Stream/Views/Explore/LogsView.tsx index 549debba..bde343ad 100644 --- a/src/pages/Stream/Views/Explore/LogsView.tsx +++ b/src/pages/Stream/Views/Explore/LogsView.tsx @@ -5,10 +5,11 @@ import LogTable from './StaticLogTable'; import useLogsFetcher from './useLogsFetcher'; import LogsViewConfig from './LogsViewConfig'; -const LogsView = (props: { schemaLoading: boolean }) => { - const { schemaLoading } = props; +const LogsView = (props: { schemaLoading: boolean, infoLoading: boolean }) => { + const { schemaLoading, infoLoading } = props; const { errorMessage, hasNoData, showTable, isFetchingCount, logsLoading } = useLogsFetcher({ schemaLoading, + infoLoading }); const [viewMode] = useLogsStore((store) => store.viewMode); const viewOpts = { @@ -21,7 +22,7 @@ const LogsView = (props: { schemaLoading: boolean }) => { return ( - {viewMode === 'table' && } + {viewMode === 'table' && } {viewMode === 'table' ? : } ); diff --git a/src/pages/Stream/Views/Explore/LogsViewConfig.tsx b/src/pages/Stream/Views/Explore/LogsViewConfig.tsx index 3007a200..ed74beda 100644 --- a/src/pages/Stream/Views/Explore/LogsViewConfig.tsx +++ b/src/pages/Stream/Views/Explore/LogsViewConfig.tsx @@ -264,15 +264,15 @@ const ColumnsList = (props: { isLoading: boolean }) => { ); }; -const LogsViewConfig = (props: { schemaLoading: boolean; logsLoading: boolean }) => { +const LogsViewConfig = (props: { schemaLoading: boolean; logsLoading: boolean, infoLoading: boolean }) => { const [configViewType] = useLogsStore((store) => store.tableOpts.configViewType); return (
{configViewType === 'schema' ? ( - + ) : ( - + )} ); diff --git a/src/pages/Stream/Views/Explore/useLogsFetcher.ts b/src/pages/Stream/Views/Explore/useLogsFetcher.ts index 93cec0c3..b22c9a1a 100644 --- a/src/pages/Stream/Views/Explore/useLogsFetcher.ts +++ b/src/pages/Stream/Views/Explore/useLogsFetcher.ts @@ -6,8 +6,8 @@ import { useFetchCount } from '@/hooks/useQueryResult'; const { setCleanStoreForStreamChange } = logsStoreReducers; -const useLogsFetcher = (props: { schemaLoading: boolean }) => { - const { schemaLoading } = props; +const useLogsFetcher = (props: { schemaLoading: boolean, infoLoading: boolean }) => { + const { schemaLoading, infoLoading } = props; const [currentStream] = useAppStore((store) => store.currentStream); const [{ tableOpts, timeRange }, setLogsStore] = useLogsStore((store) => store); const { currentOffset, currentPage, pageData } = tableOpts; @@ -22,17 +22,21 @@ const useLogsFetcher = (props: { schemaLoading: boolean }) => { }, [currentStream]); useEffect(() => { + if (infoLoading) return; + if (currentPage === 0 && currentOffset === 0) { getQueryData(); refetchCount(); } - }, [currentPage, currentStream, timeRange]); + }, [currentPage, currentStream, timeRange, infoLoading]); useEffect(() => { + if (infoLoading) return; + if (currentOffset !== 0 && currentPage !== 0) { getQueryData(); } - }, [currentOffset]); + }, [currentOffset, infoLoading]); return { logsLoading, diff --git a/src/pages/Stream/components/EventTimeLineGraph.tsx b/src/pages/Stream/components/EventTimeLineGraph.tsx index 7a61600f..1810283c 100644 --- a/src/pages/Stream/components/EventTimeLineGraph.tsx +++ b/src/pages/Stream/components/EventTimeLineGraph.tsx @@ -1,7 +1,7 @@ import { Paper, Skeleton, Stack, Text } from '@mantine/core'; import classes from '../styles/EventTimeLineGraph.module.css'; import { useQueryResult } from '@/hooks/useQueryResult'; -import { useCallback, useEffect, useMemo } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import dayjs from 'dayjs'; import { ChartTooltipProps, AreaChart } from '@mantine/charts'; import { HumanizeNumber } from '@/utils/formatBytes'; @@ -257,26 +257,31 @@ const EventTimeLineGraph = () => { const [appliedQuery] = useFilterStore((store) => store.appliedQuery); const [{ activeMode, custSearchQuery }] = useLogsStore((store) => store.custQuerySearchState); const [{ interval, startTime, endTime }] = useLogsStore((store) => store.timeRange); + const [localStream, setLocalStream] = useState(''); useEffect(() => { - if (!currentStream || currentStream.length === 0) return; + setLocalStream(currentStream); + }, [currentStream]); + + useEffect(() => { + if (!localStream || localStream.length === 0) return; const { modifiedEndTime, modifiedStartTime, compactType } = getModifiedTimeRange(startTime, endTime, interval); const logsQuery = { - streamName: currentStream, + streamName: localStream, startTime: modifiedStartTime, endTime: modifiedEndTime, access: [], }; - const whereClause = - activeMode === 'sql' ? extractWhereClause(custSearchQuery) : parseQuery(appliedQuery, currentStream).where; - const query = generateCountQuery(currentStream, modifiedStartTime, modifiedEndTime, compactType, whereClause); + activeMode === 'sql' ? extractWhereClause(custSearchQuery) : parseQuery(appliedQuery, localStream).where; + const query = generateCountQuery(localStream, modifiedStartTime, modifiedEndTime, compactType, whereClause); fetchQueryMutation.mutate({ logsQuery, query, + useTrino: false, }); - }, [currentStream, startTime.toISOString(), endTime.toISOString(), custSearchQuery]); + }, [localStream, startTime.toISOString(), endTime.toISOString(), custSearchQuery]); const isLoading = fetchQueryMutation.isLoading; const avgEventCount = useMemo(() => calcAverage(fetchQueryMutation?.data), [fetchQueryMutation?.data]); diff --git a/src/pages/Stream/components/Querier/QueryCodeEditor.tsx b/src/pages/Stream/components/Querier/QueryCodeEditor.tsx index 6ed18879..1b7ac1bd 100644 --- a/src/pages/Stream/components/Querier/QueryCodeEditor.tsx +++ b/src/pages/Stream/components/Querier/QueryCodeEditor.tsx @@ -11,6 +11,8 @@ import queryCodeStyles from '../../styles/QueryCode.module.css'; import { useAppStore } from '@/layouts/MainLayout/providers/AppProvider'; import { LOAD_LIMIT, useLogsStore } from '../../providers/LogsProvider'; import { useStreamStore } from '../../providers/StreamProvider'; +import { formQueryOpts } from '@/api/query'; +import _ from 'lodash'; const genColumnConfig = (fields: Field[]) => { const columnConfig = { leftColumns: [], rightColumns: [] }; @@ -29,9 +31,10 @@ const genColumnConfig = (fields: Field[]) => { }, columnConfig); }; -export const defaultCustSQLQuery = (streamName: string | null) => { +export const defaultCustSQLQuery = (streamName: string | null, startTime: Date, endTime: Date, timePartitionColumn: string) => { if (streamName && streamName.length > 0) { - return `SELECT * FROM ${streamName} LIMIT ${LOAD_LIMIT};`; + const { query } = formQueryOpts({ streamName: streamName || '', limit: LOAD_LIMIT, startTime, endTime, timePartitionColumn, pageOffset: 0 }); + return query; } else { return ''; } @@ -43,6 +46,8 @@ const QueryCodeEditor: FC<{ onClear: () => void; }> = (props) => { const [llmActive] = useAppStore((store) => store.instanceConfig?.llmActive); + const [streamInfo] = useStreamStore((store) => store.info); + const timePartitionColumn = _.get(streamInfo, 'time_partition', 'p_timestamp'); const [{ isQuerySearchActive, activeMode, savedFilterId, custSearchQuery }] = useLogsStore( (store) => store.custQuerySearchState, ); @@ -58,12 +63,13 @@ const QueryCodeEditor: FC<{ const { data: resAIQuery, postLLMQuery } = usePostLLM(); const isLlmActive = !!llmActive; const isSqlSearchActive = isQuerySearchActive && activeMode === 'sql'; + const [timeRange] = useLogsStore((store) => store.timeRange); useEffect(() => { if (props.queryCodeEditorRef.current === '' || currentStream !== localStreamName) { - props.queryCodeEditorRef.current = defaultCustSQLQuery(currentStream); + props.queryCodeEditorRef.current = defaultCustSQLQuery(currentStream, timeRange.startTime, timeRange.endTime, timePartitionColumn); } - }, [currentStream]); + }, [currentStream, timeRange.startTime, timeRange.endTime, timePartitionColumn]); const updateQuery = useCallback((query: string) => { props.queryCodeEditorRef.current = query; diff --git a/src/pages/Stream/components/Querier/index.tsx b/src/pages/Stream/components/Querier/index.tsx index 40c8ff12..3039f3aa 100644 --- a/src/pages/Stream/components/Querier/index.tsx +++ b/src/pages/Stream/components/Querier/index.tsx @@ -74,14 +74,22 @@ const QuerierModal = (props: { }) => { const [currentStream] = useAppStore((store) => store.currentStream); const [{ showQueryBuilder, viewMode }, setLogsStore] = useLogsStore((store) => store.custQuerySearchState); + const [streamInfo] = useStreamStore((store) => store.info); + const [timeRange] = useLogsStore((store) => store.timeRange); + const timePartitionColumn = _.get(streamInfo, 'time_partition', 'p_timestamp'); const onClose = useCallback(() => { setLogsStore((store) => toggleQueryBuilder(store, false)); }, []); const queryCodeEditorRef = useRef(''); // to store input value even after the editor unmounts useEffect(() => { - queryCodeEditorRef.current = defaultCustSQLQuery(currentStream); - }, [currentStream]); + queryCodeEditorRef.current = defaultCustSQLQuery( + currentStream, + timeRange.startTime, + timeRange.endTime, + timePartitionColumn, + ); + }, [currentStream, timeRange.endTime, timeRange.startTime, timePartitionColumn]); return ( { const [custQuerySearchState, setLogsStore] = useLogsStore((store) => store.custQuerySearchState); + const [{ startTime, endTime }] = useLogsStore((store) => store.timeRange); const { isQuerySearchActive, viewMode, showQueryBuilder, activeMode, savedFilterId } = custQuerySearchState; const [currentStream] = useAppStore((store) => store.currentStream); const [activeSavedFilters] = useAppStore((store) => store.activeSavedFilters); @@ -118,7 +127,9 @@ const Querier = () => { setLogsStore((store) => toggleQueryBuilder(store)); }, []); const [schema] = useStreamStore((store) => store.schema); + const [streamInfo] = useStreamStore((store) => store.info); const [{ query, isSumbitDisabled }, setFilterStore] = useFilterStore((store) => store); + const timePartitionColumn = _.get(streamInfo, 'time_partition', 'p_timestamp'); useEffect(() => { if (schema) { @@ -143,11 +154,11 @@ const Querier = () => { if (!currentStream) return; const { isUncontrolled } = opts || {}; - const { parsedQuery } = parseQuery(query, currentStream); + const { parsedQuery } = parseQuery(query, currentStream, { startTime, endTime, timePartitionColumn }); setFilterStore((store) => storeAppliedQuery(store)); triggerRefetch(parsedQuery, 'filters', isUncontrolled && savedFilterId ? savedFilterId : undefined); }, - [query, currentStream, savedFilterId], + [query, currentStream, savedFilterId, endTime, startTime, timePartitionColumn], ); const onSqlSearchApply = useCallback( diff --git a/src/pages/Stream/index.tsx b/src/pages/Stream/index.tsx index 4e09bac4..2de43030 100644 --- a/src/pages/Stream/index.tsx +++ b/src/pages/Stream/index.tsx @@ -17,10 +17,11 @@ import { Text } from '@mantine/core'; import { RetryBtn } from '@/components/Button/Retry'; import LogsView from './Views/Explore/LogsView'; import { useGetStreamSchema } from '@/hooks/useGetLogStreamSchema'; +import { useGetStreamInfo } from '@/hooks/useGetStreamInfo'; const { streamChangeCleanup } = streamStoreReducers; -const SchemaErrorView = (props: { error: string | null; fetchSchema: () => void }) => { +const ErrorView = (props: { error: string | null; onRetry: () => void }) => { return ( @@ -28,19 +29,20 @@ const SchemaErrorView = (props: { error: string | null; fetchSchema: () => void {props.error || 'Error'} - + ); }; -const Logs: FC = () => { +const Stream: FC = () => { useDocumentTitle('Parseable | Stream'); const { view } = useParams(); const [currentStream] = useAppStore((store) => store.currentStream); const [maximized] = useAppStore((store) => store.maximized); const [sideBarOpen, setStreamStore] = useStreamStore((store) => store.sideBarOpen); + const {getStreamInfoRefetch, getStreamInfoLoading, getStreamInfoRetching} = useGetStreamInfo(currentStream || ''); const { refetch: refetchSchema, @@ -56,8 +58,13 @@ const Logs: FC = () => { }, [currentStream]); useEffect(() => { - if (!_.isEmpty(currentStream) && view !== 'explore') { - fetchSchema(); + if (!_.isEmpty(currentStream)) { + if (view === 'explore') { + setStreamStore(streamChangeCleanup); + getStreamInfoRefetch(); + } else { + fetchSchema(); + } } }, [currentStream]); @@ -65,8 +72,9 @@ const Logs: FC = () => { if (!currentStream) return null; if (!_.includes(STREAM_VIEWS, view)) return null; - const isSchemaFetching = isSchemaRefetching || isSchemaLoading + const isSchemaFetching = isSchemaRefetching || isSchemaLoading; + const isInfoLoading = (getStreamInfoLoading || getStreamInfoRetching) && view === 'explore'; return ( { {view === 'explore' && } {view === 'explore' ? ( isSchemaError ? ( - + ) : ( - + ) ) : view === 'live-tail' ? ( @@ -105,4 +113,4 @@ const Logs: FC = () => { ); }; -export default Logs; +export default Stream; diff --git a/src/pages/Stream/providers/FilterProvider.tsx b/src/pages/Stream/providers/FilterProvider.tsx index 5dd468a7..e1fe4a2c 100644 --- a/src/pages/Stream/providers/FilterProvider.tsx +++ b/src/pages/Stream/providers/FilterProvider.tsx @@ -3,6 +3,7 @@ import { generateRandomId } from '@/utils'; import initContext from '@/utils/initContext'; import { Field, RuleGroupType, RuleType, formatQuery } from 'react-querybuilder'; import { LOAD_LIMIT } from './LogsProvider'; +import { timeRangeSQLCondition } from '@/api/query'; // write transformer (for saved filters) if you are updating the operators below export const textFieldOperators = [ @@ -126,7 +127,11 @@ type FilterStoreReducers = { updateGroupCombinator: (store: FilterStore, id: string, op: Combinator) => ReducerOutput; updateParentCombinator: (store: FilterStore, combinator: Combinator) => ReducerOutput; updateRule: (store: FilterStore, groupId: string, ruleId: string, updateOpts: RuleUpdateOpts) => ReducerOutput; - parseQuery: (query: QueryType, currentStream: string) => { where: string; parsedQuery: string }; + parseQuery: ( + query: QueryType, + currentStream: string, + timeRangeOpts?: { startTime: Date; endTime: Date, timePartitionColumn: string }, + ) => { where: string; parsedQuery: string }; toggleSubmitBtn: (store: FilterStore, val: boolean) => ReducerOutput; toggleSaveFiltersModal: (_store: FilterStore, val: boolean) => ReducerOutput; toggleSavedFiltersModal: (_store: FilterStore, val: boolean) => ReducerOutput; @@ -235,10 +240,13 @@ const toggleSubmitBtn = (_store: FilterStore, val: boolean) => { }; // todo - custom rule processor to prevent converting number strings into numbers for text fields -const parseQuery = (query: QueryType, currentStream: string) => { +const parseQuery = (query: QueryType, currentStream: string, timeRangeOpts?: { startTime: Date; endTime: Date, timePartitionColumn: string }) => { // todo - custom rule processor to prevent converting number strings into numbers for text fields const where = formatQuery(query, { format: 'sql', parseNumbers: true, quoteFieldNamesWith: ['"', '"'] }); - const parsedQuery = `select * from ${currentStream} where ${where} limit ${LOAD_LIMIT}`; + const timeRangeCondition = timeRangeOpts + ? timeRangeSQLCondition(timeRangeOpts.timePartitionColumn, timeRangeOpts.startTime, timeRangeOpts.endTime) + : '(1=1)'; + const parsedQuery = `select * from ${currentStream} where ${where} AND ${timeRangeCondition} offset 0 limit ${LOAD_LIMIT}`; return { where, parsedQuery }; }; diff --git a/src/pages/Stream/providers/LogsProvider.tsx b/src/pages/Stream/providers/LogsProvider.tsx index e6aad7bc..66e9cfba 100644 --- a/src/pages/Stream/providers/LogsProvider.tsx +++ b/src/pages/Stream/providers/LogsProvider.tsx @@ -446,10 +446,11 @@ const toggleQueryBuilder = (store: LogsStore, val?: boolean) => { }; const resetCustQuerySearchState = (store: LogsStore) => { - const { custQuerySearchState } = store; + const { custQuerySearchState, timeRange } = store; return { custQuerySearchState: { ...defaultCustQuerySearchState, viewMode: custQuerySearchState.viewMode }, ...getCleanStoreForRefetch(store), + timeRange }; }; @@ -699,11 +700,11 @@ const applyCustomQuery = ( savedFilterId?: string, timeRangePayload?: { from: string; to: string } | null, ) => { - const { custQuerySearchState } = store; + const { custQuerySearchState, timeRange } = store; - const timeRange = (() => { + const updatedTimeRange = (() => { if (!timeRangePayload) { - return {}; + return { timeRange }; } else { const startTime = dayjs(timeRangePayload.from); const endTime = dayjs(timeRangePayload.to); @@ -733,7 +734,7 @@ const applyCustomQuery = ( viewMode: mode, }, ...getCleanStoreForRefetch(store), - ...timeRange, + ...updatedTimeRange, }; }; diff --git a/src/utils/timeRangeUtils.ts b/src/utils/timeRangeUtils.ts index bdbdb907..546b821c 100644 --- a/src/utils/timeRangeUtils.ts +++ b/src/utils/timeRangeUtils.ts @@ -73,6 +73,10 @@ const formatDateWithTimezone = (dateTime: string, format: string = 'DD/MM/YYYY h return convertedDate.format(format); }; +const formatDateAsCastType = (date: Date): string => { + return dayjs(date).utc().format('YYYY-MM-DD HH:mm:ss[Z]'); +}; + const timeRangeUtils = { defaultTimeRangeOption, formatDateWithTimezone, @@ -82,6 +86,7 @@ const timeRangeUtils = { getDefaultTimeRangeOption, formatTime, formatDay, + formatDateAsCastType, }; export default timeRangeUtils;