From 0b7d155a1eb1071b6f2120d6a2ee48cdc77c333e Mon Sep 17 00:00:00 2001 From: Kev Date: Tue, 31 Mar 2026 14:35:15 -0400 Subject: [PATCH 1/2] feat(ourlogs): Switch needle in haystack to time based We were doing a static number of fetches before. Depending on cacheing etc. this can be really fast or slow, but it resulted in you clicking 'continue scanning' a LOT. This should bring down the number of times you have to click to ever 20 seconds, aside from the initial faster to return 5 second result. --- static/app/views/explore/logs/constants.tsx | 2 ++ .../views/explore/logs/useLogsQuery.spec.tsx | 22 +++++-------------- .../app/views/explore/logs/useLogsQuery.tsx | 20 ++++++++++------- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/static/app/views/explore/logs/constants.tsx b/static/app/views/explore/logs/constants.tsx index d50ee801f62515..13edab79b5d6d6 100644 --- a/static/app/views/explore/logs/constants.tsx +++ b/static/app/views/explore/logs/constants.tsx @@ -84,6 +84,8 @@ export const LOGS_FILTER_KEY_SECTIONS: FilterKeySection[] = [LOGS_FILTERS]; */ export const LOGS_DRAWER_QUERY_PARAM = 'logsDrawer'; +export const FLEX_TIME_INITIAL_SEARCH_DURATION_MS = 10_000; +export const FLEX_TIME_RESUME_SEARCH_DURATION_MS = 20_000; export const VIRTUAL_STREAMED_INTERVAL_MS = 250; export const MINIMUM_INFINITE_SCROLL_FETCH_COOLDOWN_MS = 1000; diff --git a/static/app/views/explore/logs/useLogsQuery.spec.tsx b/static/app/views/explore/logs/useLogsQuery.spec.tsx index fb98a61d2a8ca2..26a166e62172e9 100644 --- a/static/app/views/explore/logs/useLogsQuery.spec.tsx +++ b/static/app/views/explore/logs/useLogsQuery.spec.tsx @@ -369,41 +369,31 @@ describe('useInfiniteLogsQuery', () => { }; } - it('auto fetches only empty pages pages and end when signaled', async () => { + it('auto fetches empty pages until hasNext is false', async () => { const mockFlextTimeRequests = [ makeMockEventsResponse({cursor: '', nextCursor: 'page2'}), makeMockEventsResponse({cursor: 'page2', nextCursor: 'page3'}), makeMockEventsResponse({cursor: 'page3', nextCursor: 'page4'}), - makeMockEventsResponse({cursor: 'page4', nextCursor: 'page5'}), + makeMockEventsResponse({cursor: 'page4', nextCursor: 'page5', hasNext: false}), makeMockEventsResponse({cursor: 'page5', nextCursor: 'page6', hasNext: false}), - makeMockEventsResponse({cursor: 'page6', nextCursor: 'page7', hasNext: false}), ].map(response => MockApiClient.addMockResponse(response)); const {result} = renderHookWithProviders( - () => useInfiniteLogsQuery({highFidelity: true, maxAutoFetches: 3}), + () => useInfiniteLogsQuery({highFidelity: true}), { additionalWrapper: createWrapper(), } ); - // the first 3 requests should have been called await waitFor(() => expect(mockFlextTimeRequests[0]).toHaveBeenCalledTimes(1)); await waitFor(() => expect(mockFlextTimeRequests[1]).toHaveBeenCalledTimes(1)); await waitFor(() => expect(mockFlextTimeRequests[2]).toHaveBeenCalledTimes(1)); - await waitFor(() => expect(mockFlextTimeRequests[3]).not.toHaveBeenCalled()); - - // should be allowed to resume autofetching - expect(result.current.canResumeAutoFetch).toBe(true); - act(() => result.current.resumeAutoFetch()); - - // the next 3 requests should have been called await waitFor(() => expect(mockFlextTimeRequests[3]).toHaveBeenCalledTimes(1)); - await waitFor(() => expect(mockFlextTimeRequests[4]).toHaveBeenCalledTimes(1)); - // should not be allowed to resume autofetching + // should not be allowed to resume autofetching because hasNext is false expect(result.current.canResumeAutoFetch).toBe(false); - await waitFor(() => expect(mockFlextTimeRequests[5]).not.toHaveBeenCalled()); + await waitFor(() => expect(mockFlextTimeRequests[4]).not.toHaveBeenCalled()); }); it('auto fetches until limit', async () => { @@ -433,7 +423,7 @@ describe('useInfiniteLogsQuery', () => { ].map(response => MockApiClient.addMockResponse(response)); const {result} = renderHookWithProviders( - () => useInfiniteLogsQuery({highFidelity: true, maxAutoFetches: 3}), + () => useInfiniteLogsQuery({highFidelity: true}), { additionalWrapper: createWrapper(), } diff --git a/static/app/views/explore/logs/useLogsQuery.tsx b/static/app/views/explore/logs/useLogsQuery.tsx index eae61961782f01..54d00de3e515c5 100644 --- a/static/app/views/explore/logs/useLogsQuery.tsx +++ b/static/app/views/explore/logs/useLogsQuery.tsx @@ -28,6 +28,8 @@ import {SAMPLING_MODE} from 'sentry/views/explore/hooks/useProgressiveQuery'; import {useTraceItemDetails} from 'sentry/views/explore/hooks/useTraceItemDetails'; import { AlwaysPresentLogFields, + FLEX_TIME_INITIAL_SEARCH_DURATION_MS, + FLEX_TIME_RESUME_SEARCH_DURATION_MS, MAX_LOG_INGEST_DELAY, MAX_LOGS_INFINITE_QUERY_PAGES, QUERY_PAGE_LIMIT, @@ -405,12 +407,10 @@ type QueryKey = [ export function useInfiniteLogsQuery({ disabled, highFidelity, - maxAutoFetches = 5, referrer, }: { disabled?: boolean; highFidelity?: boolean; - maxAutoFetches?: number; referrer?: string; } = {}) { const _referrer = referrer ?? 'api.explore.logs-table'; @@ -689,15 +689,17 @@ export function useInfiniteLogsQuery({ const lastPageLength = data?.pages?.[data.pages.length - 1]?.[0]?.data?.length ?? 0; const limit = autoRefresh ? QUERY_PAGE_LIMIT_WITH_AUTO_REFRESH : QUERY_PAGE_LIMIT; - // the original state starts at -1 because we have to count - // the 1 query made by default outside of the auto fetches - const [autoFetchesRemaining, setAutoFetchesRemaining] = useState(maxAutoFetches - 1); + const [autoFetchStartTime, setAutoFetchStartTime] = useState(() => Date.now()); + const [autoFetchDuration, setAutoFetchDuration] = useState( + FLEX_TIME_INITIAL_SEARCH_DURATION_MS + ); const canAutoFetchNextPage = !!highFidelity && hasNextPage && nextPageHasData && (lastPageLength === 0 || _data.length < limit); - const shouldAutoFetchNextPage = canAutoFetchNextPage && autoFetchesRemaining > 0; + const shouldAutoFetchNextPage = + canAutoFetchNextPage && Date.now() - autoFetchStartTime < autoFetchDuration; useEffect(() => { if (!shouldAutoFetchNextPage) { @@ -708,7 +710,6 @@ export function useInfiniteLogsQuery({ return; } - setAutoFetchesRemaining(remaining => remaining - 1); _fetchNextPage(); }, [shouldAutoFetchNextPage, isFetchingNextPage, _fetchNextPage, nextPageCursor]); @@ -741,7 +742,10 @@ export function useInfiniteLogsQuery({ isFetchingPreviousPage, lastPageLength, canResumeAutoFetch: canAutoFetchNextPage, - resumeAutoFetch: () => setAutoFetchesRemaining(maxAutoFetches), + resumeAutoFetch: () => { + setAutoFetchStartTime(Date.now()); + setAutoFetchDuration(FLEX_TIME_RESUME_SEARCH_DURATION_MS); + }, dataScanned, bytesScanned: totalBytesScanned, }; From 1662cb0d2bc46577e6d7235f63725820e6787613 Mon Sep 17 00:00:00 2001 From: Kev Date: Tue, 31 Mar 2026 16:43:52 -0400 Subject: [PATCH 2/2] feat(logs): Emit metric for flex time pages fetched before data found Made-with: Cursor --- .../app/views/explore/logs/useLogsQuery.tsx | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/static/app/views/explore/logs/useLogsQuery.tsx b/static/app/views/explore/logs/useLogsQuery.tsx index 54d00de3e515c5..5c1200fcc10e9f 100644 --- a/static/app/views/explore/logs/useLogsQuery.tsx +++ b/static/app/views/explore/logs/useLogsQuery.tsx @@ -1,4 +1,5 @@ -import {useCallback, useEffect, useMemo, useState} from 'react'; +import {useCallback, useEffect, useMemo, useRef, useState} from 'react'; +import * as Sentry from '@sentry/react'; import {logger} from '@sentry/react'; import {type ApiResult} from 'sentry/api'; @@ -701,6 +702,29 @@ export function useInfiniteLogsQuery({ const shouldAutoFetchNextPage = canAutoFetchNextPage && Date.now() - autoFetchStartTime < autoFetchDuration; + const autoFetchPageCount = useRef(0); + const prevShouldAutoFetch = useRef(false); + + useEffect(() => { + if (shouldAutoFetchNextPage && !prevShouldAutoFetch.current) { + autoFetchPageCount.current = 0; + } + + if (!shouldAutoFetchNextPage && prevShouldAutoFetch.current && highFidelity) { + Sentry.metrics.distribution( + 'explore.logs.flex_time_pages_before_data', + autoFetchPageCount.current, + { + attributes: { + found_data: _data.length > 0 ? 'true' : 'false', + }, + } + ); + } + + prevShouldAutoFetch.current = shouldAutoFetchNextPage; + }, [shouldAutoFetchNextPage, highFidelity, _data.length]); + useEffect(() => { if (!shouldAutoFetchNextPage) { return; @@ -710,6 +734,7 @@ export function useInfiniteLogsQuery({ return; } + autoFetchPageCount.current += 1; _fetchNextPage(); }, [shouldAutoFetchNextPage, isFetchingNextPage, _fetchNextPage, nextPageCursor]);