Query engine

src/@types/parseable/api/about.ts

@@ -1,3 +1,5 @@
+export type QueryEngineType = 'Trino' | 'Parseable' | undefined;
+
 export type AboutData = {
 	commit: string;
 	deploymentId: string;
@@ -17,4 +19,5 @@ export type AboutData = {
 	analytics: {
 		clarityTag: string;
 	};
+	queryEngine: QueryEngineType;
 };

src/@types/parseable/api/query.ts

@@ -1,4 +1,6 @@
+import { QueryEngineType } from '@/@types/parseable/api/about';
 export type LogsQuery = {
+	queryEngine?: QueryEngineType;
 	streamName: string;
 	startTime: Date;
 	endTime: Date;

src/api/constants.ts

@@ -20,11 +20,11 @@ const parseParamsToQueryString = (params: Params) => {
 // Streams Management
 export const LOG_STREAM_LIST_URL = `${API_V1}/logstream`;
 export const LOG_STREAMS_SCHEMA_URL = (streamName: string) => `${LOG_STREAM_LIST_URL}/${streamName}/schema`;
-export const LOG_QUERY_URL = (params?: Params) => `${API_V1}/query` + parseParamsToQueryString(params);
-export const LOG_TRINO_QUERY_URL = (params?: Params) => `${API_V1}/trinoquery` + parseParamsToQueryString(params);
+export const LOG_QUERY_URL = (params?: Params, resourcePath: string = 'query') =>
+	`${API_V1}/${resourcePath}` + parseParamsToQueryString(params);
 export const LOG_STREAMS_ALERTS_URL = (streamName: string) => `${LOG_STREAM_LIST_URL}/${streamName}/alert`;
 export const LIST_SAVED_FILTERS_URL = `${API_V1}/filters`;
-export const LIST_DASHBOARDS =  `${API_V1}/dashboards`;
+export const LIST_DASHBOARDS = `${API_V1}/dashboards`;
 export const UPDATE_SAVED_FILTERS_URL = (filterId: string) => `${API_V1}/filters/${filterId}`;
 export const UPDATE_DASHBOARDS_URL = (dashboardId: string) => `${API_V1}/dashboards/${dashboardId}`;
 export const DELETE_DASHBOARDS_URL = (dashboardId: string) => `${API_V1}/dashboards/${dashboardId}`;

src/api/query.ts

@@ -1,11 +1,15 @@
 import _ from 'lodash';
 import { Axios } from './axios';
-import { LOG_QUERY_URL, LOG_TRINO_QUERY_URL } from './constants';
+import { LOG_QUERY_URL } from './constants';
 import { Log, LogsQuery, LogsResponseWithHeaders } from '@/@types/parseable/api/query';
+import { QueryEngineType } from '@/@types/parseable/api/about';
 import timeRangeUtils from '@/utils/timeRangeUtils';
+import { QueryBuilder } from '@/utils/queryBuilder';
 
 const { formatDateAsCastType } = timeRangeUtils;
+type QueryEngine = QueryEngineType;
 type QueryLogs = {
+	queryEngine: QueryEngine;
 	streamName: string;
 	startTime: Date;
 	endTime: Date;
@@ -35,22 +39,22 @@ export const timeRangeSQLCondition = (timePartitionColumn: string, startTime: Da
 };
 
 export const formQueryOpts = (logsQuery: FormQueryOptsType) => {
-	const { startTime, endTime, streamName, limit, pageOffset, timePartitionColumn = 'p_timestamp' } = logsQuery;
-	const optimizedStartTime = optimizeTime(startTime);
-	const optimizedEndTime = optimizeTime(endTime);
-	const orderBy = `ORDER BY ${timePartitionColumn} desc`;
-	const timestampClause = timeRangeSQLCondition(timePartitionColumn, optimizedStartTime, optimizedEndTime);
-	const offsetPart = _.isNumber(pageOffset) ? `OFFSET ${pageOffset}` : '';
-	const query = `SELECT * FROM \"${streamName}\" where ${timestampClause} ${orderBy} ${offsetPart} LIMIT ${limit} `;
-	return { query, startTime: optimizedStartTime, endTime: optimizedEndTime };
+	const queryBuilder = new QueryBuilder(logsQuery);
+	const query = queryBuilder.getQuery();
+	const startTime = queryBuilder.getStartTime();
+	const endTime = queryBuilder.getEndTime();
+
+	return { query, startTime, endTime };
 };
 
 export const getQueryLogs = (logsQuery: QueryLogs) => {
-	return Axios().post<Log[]>(LOG_TRINO_QUERY_URL(), formQueryOpts(logsQuery), {});
+	return Axios().post<Log[]>(LOG_QUERY_URL(), formQueryOpts(logsQuery), {});
 };
 
 export const getQueryLogsWithHeaders = (logsQuery: QueryLogs) => {
-	return Axios().post<LogsResponseWithHeaders>(LOG_TRINO_QUERY_URL({ fields: true }), formQueryOpts(logsQuery), {});
+	const queryBuilder = new QueryBuilder(logsQuery);
+	const endPoint = LOG_QUERY_URL({ fields: true }, queryBuilder.getResourcePath());
+	return Axios().post<LogsResponseWithHeaders>(endPoint, formQueryOpts(logsQuery), {});
 };
 
 // ------ Custom sql query
@@ -60,12 +64,13 @@ const makeCustomQueryRequestData = (logsQuery: LogsQuery, query: string) => {
 	return { query, startTime: optimizeTime(startTime), endTime: optimizeTime(endTime) };
 };
 
-export const getQueryResult = (logsQuery: LogsQuery, query = '', useTrino = true) => {
-	const endPoint = useTrino ? LOG_TRINO_QUERY_URL() : LOG_QUERY_URL();
+export const getQueryResult = (logsQuery: LogsQuery, query = '') => {
+	const endPoint = LOG_QUERY_URL();
 	return Axios().post<Log[]>(endPoint, makeCustomQueryRequestData(logsQuery, query), {});
 };
 
-export const getQueryResultWithHeaders = (logsQuery: LogsQuery, query = '', useTrino = true) => {
-	const endPoint = useTrino ? LOG_TRINO_QUERY_URL({ fields: true }) : LOG_QUERY_URL({ fields: true });
+export const getQueryResultWithHeaders = (logsQuery: LogsQuery, query = '') => {
+	const queryBuilder = new QueryBuilder(logsQuery);
+	const endPoint = LOG_QUERY_URL({ fields: true }, queryBuilder.getResourcePath());
 	return Axios().post<LogsResponseWithHeaders>(endPoint, makeCustomQueryRequestData(logsQuery, query), {});
 };

src/hooks/useGetStreamInfo.tsx

@@ -6,22 +6,23 @@ import { useQuery } from 'react-query';
 
 const { setStreamInfo } = streamStoreReducers;
 
-export const useGetStreamInfo = (currentStream: string) => {
+export const useGetStreamInfo = (currentStream: string, initialFetch: boolean) => {
 	const [, setStreamStore] = useStreamStore((_store) => null);
 	const {
 		data: getStreamInfoData,
 		isError: getStreamInfoError,
 		isSuccess: getStreamInfoSuccess,
 		isLoading: getStreamInfoLoading,
 		refetch: getStreamInfoRefetch,
-		isRefetching: getStreamInfoRetching
+		isRefetching: getStreamInfoRefetching,
 	} = useQuery(['stream-info', currentStream], () => getLogStreamInfo(currentStream), {
 		retry: false,
 		refetchOnWindowFocus: false,
 		refetchOnMount: true,
-		enabled: currentStream !== '',
+		enabled: initialFetch,
+		// currentStream !== '',
 		onSuccess: (data) => {
-			setStreamStore((store) => setStreamInfo(store, data))
+			setStreamStore((store) => setStreamInfo(store, data));
 		},
 		onError: (data: AxiosError) => {
 			if (isAxiosError(data) && data.response) {
@@ -39,6 +40,6 @@ export const useGetStreamInfo = (currentStream: string) => {
 		getStreamInfoSuccess,
 		getStreamInfoLoading,
 		getStreamInfoRefetch,
-		getStreamInfoRetching
+		getStreamInfoRefetching,
 	};
 };

src/hooks/useQueryLogs.ts

@@ -35,6 +35,7 @@ export const useQueryLogs = () => {
 			order: SortOrder.DESCENDING,
 		},
 	});
+	const [queryEngine] = useAppStore((store) => store.instanceConfig?.queryEngine);
 	const [streamInfo] = useStreamStore((store) => store.info);
 	const [currentStream] = useAppStore((store) => store.currentStream);
 	const timePartitionColumn = _.get(streamInfo, 'time_partition', 'p_timestamp');
@@ -72,6 +73,7 @@ export const useQueryLogs = () => {
 
 	// refactor
 	const defaultQueryOpts = {
+		queryEngine,
 		streamName: currentStream || '',
 		startTime: timeRange.startTime,
 		endTime: timeRange.endTime,
@@ -87,7 +89,7 @@ export const useQueryLogs = () => {
 			const logsQueryRes = await (async () => {
 				if (isQuerySearchActive) {
 					if (activeMode === 'filters') {
-						const { parsedQuery } = parseQuery(appliedQuery, currentStream || '', {
+						const { parsedQuery } = parseQuery(queryEngine, appliedQuery, currentStream || '', {
 							startTime: timeRange.startTime,
 							endTime: timeRange.endTime,
 							timePartitionColumn,

src/hooks/useQueryResult.tsx

@@ -8,8 +8,10 @@ import { logsStoreReducers, useLogsStore } from '@/pages/Stream/providers/LogsPr
 import _ from 'lodash';
 import { useAppStore } from '@/layouts/MainLayout/providers/AppProvider';
 import { notifyError } from '@/utils/notification';
+import { QueryEngineType } from '@/@types/parseable/api/about';
 
 type QueryData = {
+	queryEngine: QueryEngineType;
 	logsQuery: LogsQuery;
 	query: string;
 	onSuccess?: () => void;
@@ -18,7 +20,7 @@ type QueryData = {
 
 export const useQueryResult = () => {
 	const fetchQueryHandler = async (data: QueryData) => {
-		const response = await getQueryResultWithHeaders(data.logsQuery, data.query, data.useTrino);
+		const response = await getQueryResultWithHeaders(data.logsQuery, data.query);
 		if (response.status !== 200) {
 			throw new Error(response.statusText);
 		}
@@ -62,9 +64,14 @@ export const useFetchCount = () => {
 		if (isQuerySearchActive) {
 			const finalQuery = custSearchQuery.replace(/SELECT[\s\S]*?FROM/i, 'SELECT COUNT(*) as count FROM');
 			if (activeMode === 'filters') {
-				return finalQuery;
+				return finalQuery
+					.replace(/LIMIT\s*\d+\s*(OFFSET\s*\d+)?\s*;?/i, '') // Removes LIMIT and optional OFFSET
+					.replace(/OFFSET\s*\d+\s*;?/i, '');
 			} else {
-				return finalQuery.replace(/ORDER\s+BY\s+[\w\s,.]+(?:ASC|DESC)?\s*(LIMIT\s+\d+)?\s*;?/i, '');
+				return finalQuery
+					.replace(/ORDER\s+BY\s+[\w\s,.]+(?:ASC|DESC)?\s*(LIMIT\s*\d+)?\s*(OFFSET\s*\d+)?\s*;?/i, '') // Removes ORDER BY, LIMIT, and OFFSET
+					.replace(/LIMIT\s*\d+\s*(OFFSET\s*\d+)?\s*;?/i, '') // Removes LIMIT and optional OFFSET
+					.replace(/OFFSET\s*\d+\s*;?/i, '');
 			}
 		} else {
 			return defaultQuery;
@@ -80,15 +87,21 @@ export const useFetchCount = () => {
 		isLoading: isCountLoading,
 		isRefetching: isCountRefetching,
 		refetch: refetchCount,
-	} = useQuery(['fetchCount', logsQuery], () => getQueryResult(logsQuery, query, false), {
-		onSuccess: (resp) => {
-			const count = _.first(resp.data)?.count;
+	} = useQuery(
+		['fetchCount', logsQuery],
+		async () => {
+			const data = await getQueryResult(logsQuery, query);
+			const count = _.first(data.data)?.count;
 			typeof count === 'number' && setLogsStore((store) => setTotalCount(store, count));
+			return data;
 		},
-		refetchOnWindowFocus: false,
-		retry: false,
-		enabled: false,
-	});
+		{
+			// query for count should always hit the endpoint for parseable query
+			refetchOnWindowFocus: false,
+			retry: false,
+			enabled: false,
+		},
+	);
 
 	return {
 		isCountLoading,

src/pages/Stream/Views/Explore/LogsView.tsx

@@ -5,11 +5,11 @@ import LogTable from './StaticLogTable';
 import useLogsFetcher from './useLogsFetcher';
 import LogsViewConfig from './LogsViewConfig';
 
-const LogsView = (props: { schemaLoading: boolean, infoLoading: boolean }) => {
+const LogsView = (props: { schemaLoading: boolean; infoLoading: boolean }) => {
 	const { schemaLoading, infoLoading } = props;
 	const { errorMessage, hasNoData, showTable, isFetchingCount, logsLoading } = useLogsFetcher({
 		schemaLoading,
-		infoLoading
+		infoLoading,
 	});
 	const [viewMode] = useLogsStore((store) => store.viewMode);
 	const viewOpts = {
@@ -22,7 +22,9 @@ const LogsView = (props: { schemaLoading: boolean, infoLoading: boolean }) => {
 
 	return (
 		<Box style={{ display: 'flex', flex: 1, overflow: 'hidden' }}>
-			{viewMode === 'table' && <LogsViewConfig schemaLoading={schemaLoading} logsLoading={logsLoading} infoLoading={infoLoading}/>}
+			{viewMode === 'table' && (
+				<LogsViewConfig schemaLoading={schemaLoading} logsLoading={logsLoading} infoLoading={infoLoading} />
+			)}
 			{viewMode === 'table' ? <LogTable {...viewOpts} /> : <JsonView {...viewOpts} />}
 		</Box>
 	);

src/pages/Stream/Views/Explore/useLogsFetcher.ts

@@ -39,7 +39,7 @@ const useLogsFetcher = (props: { schemaLoading: boolean; infoLoading: boolean })
 	}, [currentOffset, infoLoading]);
 
 	return {
-		logsLoading,
+		logsLoading: infoLoading || logsLoading,
 		errorMessage,
 		hasContentLoaded,
 		hasNoData,

src/pages/Stream/Views/Manage/Management.tsx

@@ -17,20 +17,22 @@ const Management = (props: { schemaLoading: boolean }) => {
 	const getStreamAlertsConfig = useAlertsQuery(currentStream || '');
 	const getStreamStats = useLogStreamStats(currentStream || '');
 	const getRetentionConfig = useRetentionQuery(currentStream || '');
-	const getStreamInfo = useGetStreamInfo(currentStream || '');
+	const getStreamInfo = useGetStreamInfo(currentStream || '', currentStream !== null);
 	const hotTierFetch = useHotTier(currentStream || '');
 
 	// todo - handle loading and error states separately
 	const isAlertsLoading = getStreamAlertsConfig.isError || getStreamAlertsConfig.isLoading;
-	const isRetentionLoading =
-		getRetentionConfig.getLogRetentionIsLoading || instanceConfig === null;
+	const isRetentionLoading = getRetentionConfig.getLogRetentionIsLoading || instanceConfig === null;
 	const isHotTierLoading = hotTierFetch.getHotTierInfoLoading;
 
 	return (
 		<Stack style={{ padding: '1rem', paddingTop: '0', height: '90%' }}>
 			<DeleteStreamModal />
 			<Stack style={{ flexDirection: 'row', height: '40%' }} gap={24}>
-				<Stats isLoading={getStreamStats.getLogStreamStatsDataIsLoading} isError={getStreamStats.getLogStreamStatsDataIsError} />
+				<Stats
+					isLoading={getStreamStats.getLogStreamStatsDataIsLoading}
+					isError={getStreamStats.getLogStreamStatsDataIsError}
+				/>
 				<Info isLoading={getStreamInfo.getStreamInfoLoading} isError={getStreamInfo.getStreamInfoError} />
 			</Stack>
 			<Stack style={{ flexDirection: 'row', height: '57%' }} gap={24}>

src/pages/Stream/Views/Manage/UpdateCustomPartitionField.tsx

@@ -37,7 +37,7 @@ export default function UpdateCustomPartitionField(props: { timePartition: strin
 	const [showEditField, setShowEditField] = useState<boolean>(false);
 	const [error, setError] = useState<string | null>(null);
 	const { updateLogStreamMutation } = useLogStream();
-	const { getStreamInfoRefetch } = useGetStreamInfo(props.currentStream);
+	const { getStreamInfoRefetch } = useGetStreamInfo(props.currentStream, props.currentStream !== null);
 
 	useEffect(() => {
 		const customPartition: string = _.get(info, 'custom_partition', 'EMPTY_VALUE');

src/pages/Stream/Views/Manage/UpdateTimePartitionLimit.tsx

@@ -35,7 +35,7 @@ function UpdateTimePartitionLimit(props: { timePartition: string; currentStream:
 	const [error, setError] = useState<string | null>(null);
 	const [showEditField, setShowEditField] = useState<boolean>(false);
 	const { updateLogStreamMutation } = useLogStream();
-	const { getStreamInfoRefetch } = useGetStreamInfo(props.currentStream);
+	const { getStreamInfoRefetch } = useGetStreamInfo(props.currentStream, props.currentStream !== null);
 
 	useEffect(() => {
 		setValue(timePartitonLimit);

src/pages/Stream/components/EventTimeLineGraph.tsx

@@ -254,6 +254,7 @@ function ChartTooltip({ payload }: ChartTooltipProps) {
 const EventTimeLineGraph = () => {
 	const { fetchQueryMutation } = useQueryResult();
 	const [currentStream] = useAppStore((store) => store.currentStream);
+	const [queryEngine] = useAppStore((store) => store.instanceConfig?.queryEngine);
 	const [appliedQuery] = useFilterStore((store) => store.appliedQuery);
 	const [{ activeMode, custSearchQuery }] = useLogsStore((store) => store.custQuerySearchState);
 	const [{ interval, startTime, endTime }] = useLogsStore((store) => store.timeRange);
@@ -274,12 +275,14 @@ const EventTimeLineGraph = () => {
 			access: [],
 		};
 		const whereClause =
-			activeMode === 'sql' ? extractWhereClause(custSearchQuery) : parseQuery(appliedQuery, localStream).where;
+			activeMode === 'sql'
+				? extractWhereClause(custSearchQuery)
+				: parseQuery(queryEngine, appliedQuery, localStream).where;
 		const query = generateCountQuery(localStream, modifiedStartTime, modifiedEndTime, compactType, whereClause);
 		fetchQueryMutation.mutate({
+			queryEngine: 'Parseable', // query for graph should always hit the endpoint for parseable query
 			logsQuery,
 			query,
-			useTrino: false,
 		});
 	}, [localStream, startTime.toISOString(), endTime.toISOString(), custSearchQuery]);
 

src/pages/Stream/components/Querier/QueryCodeEditor.tsx

@@ -1,6 +1,7 @@
 import React, { FC, MutableRefObject, useCallback, useEffect, useState, useRef } from 'react';
 import Editor from '@monaco-editor/react';
 import { Box, Button, Flex, ScrollArea, Stack, Text, TextInput } from '@mantine/core';
+import { QueryEngineType } from '@/@types/parseable/api/about';
 import { ErrorMarker, errChecker } from '../ErrorMarker';
 import useMountedState from '@/hooks/useMountedState';
 import { notify } from '@/utils/notification';
@@ -32,13 +33,15 @@ const genColumnConfig = (fields: Field[]) => {
 };
 
 export const defaultCustSQLQuery = (
+	queryEngine: QueryEngineType,
 	streamName: string | null,
 	startTime: Date,
 	endTime: Date,
 	timePartitionColumn: string,
 ) => {
 	if (streamName && streamName.length > 0) {
 		const { query } = formQueryOpts({
+			queryEngine,
 			streamName: streamName || '',
 			limit: LOAD_LIMIT,
 			startTime,
@@ -57,7 +60,9 @@ const QueryCodeEditor: FC<{
 	onSqlSearchApply: (query: string) => void;
 	onClear: () => void;
 }> = (props) => {
-	const [llmActive] = useAppStore((store) => store.instanceConfig?.llmActive);
+	const [instanceConfig] = useAppStore((store) => store.instanceConfig);
+	const llmActive = _.get(instanceConfig, 'llmActive', false);
+	const queryEngine = _.get(instanceConfig, 'queryEngine', 'Parseable');
 	const [streamInfo] = useStreamStore((store) => store.info);
 	const timePartitionColumn = _.get(streamInfo, 'time_partition', 'p_timestamp');
 	const [{ isQuerySearchActive, activeMode, savedFilterId, custSearchQuery }] = useLogsStore(
@@ -80,6 +85,7 @@ const QueryCodeEditor: FC<{
 	useEffect(() => {
 		if (props.queryCodeEditorRef.current === '' || currentStream !== localStreamName) {
 			props.queryCodeEditorRef.current = defaultCustSQLQuery(
+				queryEngine,
 				currentStream,
 				timeRange.startTime,
 				timeRange.endTime,