Controls how many failed jobs are allowed to remain in Redis queues
| | `REPO_SYNC_RETRY_BASE_SLEEP_SECONDS` | `60` |The base sleep duration (in seconds) for exponential backoff when retrying repository sync operations that fail
| | `GITLAB_CLIENT_QUERY_TIMEOUT_SECONDS` | `600` |The timeout duration (in seconds) for GitLab client queries
| -| `SHARD_MAX_MATCH_COUNT` | `10000` |The maximum shard count per query
| | `SMTP_CONNECTION_URL` | `-` |The url to the SMTP service used for sending transactional emails. See [this doc](/docs/configuration/transactional-emails) for more info.
| | `SOURCEBOT_ENCRYPTION_KEY` | Automatically generated at startup if no value is provided. Generated using `openssl rand -base64 24` |Used to encrypt connection secrets and generate API keys.
| | `SOURCEBOT_PUBLIC_KEY_PATH` | `/app/public.pem` |Sourcebot's public key that's used to verify encrypted license key signatures.
| @@ -36,8 +35,6 @@ The following environment variables allow you to configure your Sourcebot deploy | `SOURCEBOT_STRUCTURED_LOGGING_ENABLED` | `false` |Enables/disable structured JSON logging. See [this doc](/docs/configuration/structured-logging) for more info.
| | `SOURCEBOT_STRUCTURED_LOGGING_FILE` | - |Optional file to log to if structured logging is enabled
| | `SOURCEBOT_TELEMETRY_DISABLED` | `false` |Enables/disables telemetry collection in Sourcebot. See [this doc](/docs/overview.mdx#telemetry) for more info.
| -| `TOTAL_MAX_MATCH_COUNT` | `100000` |The maximum number of matches per query
| -| `ZOEKT_MAX_WALL_TIME_MS` | `10000` |The maximum real world duration (in milliseconds) per zoekt query
| ### Enterprise Environment Variables | Variable | Default | Description | diff --git a/packages/mcp/src/schemas.ts b/packages/mcp/src/schemas.ts index 40736b59..0bb8ff9a 100644 --- a/packages/mcp/src/schemas.ts +++ b/packages/mcp/src/schemas.ts @@ -38,32 +38,82 @@ export const repositoryInfoSchema = z.object({ name: z.string(), displayName: z.string().optional(), webUrl: z.string().optional(), -}) +}); + +// Many of these fields are defined in zoekt/api.go. +export const searchStatsSchema = z.object({ + // The actual number of matches returned by the search. + // This will always be less than or equal to `totalMatchCount`. + actualMatchCount: z.number(), + + // The total number of matches found during the search. + totalMatchCount: z.number(), + + // The duration (in nanoseconds) of the search. + duration: z.number(), + + // Number of files containing a match. + fileCount: z.number(), + + // Candidate files whose contents weren't examined because we + // gathered enough matches. + filesSkipped: z.number(), + + // Amount of I/O for reading contents. + contentBytesLoaded: z.number(), + + // Amount of I/O for reading from index. + indexBytesLoaded: z.number(), + + // Number of search shards that had a crash. + crashes: z.number(), + + // Number of files in shards that we considered. + shardFilesConsidered: z.number(), + + // Files that we evaluated. Equivalent to files for which all + // atom matches (including negations) evaluated to true. + filesConsidered: z.number(), + + // Files for which we loaded file content to verify substring matches + filesLoaded: z.number(), + + // Shards that we scanned to find matches. + shardsScanned: z.number(), + + // Shards that we did not process because a query was canceled. + shardsSkipped: z.number(), + + // Shards that we did not process because the query was rejected by the + // ngram filter indicating it had no matches. + shardsSkippedFilter: z.number(), + + // Number of candidate matches as a result of searching ngrams. + ngramMatches: z.number(), + + // NgramLookups is the number of times we accessed an ngram in the index. + ngramLookups: z.number(), + + // Wall clock time for queued search. + wait: z.number(), + + // Aggregate wall clock time spent constructing and pruning the match tree. + // This accounts for time such as lookups in the trigram index. + matchTreeConstruction: z.number(), + + // Aggregate wall clock time spent searching the match tree. This accounts + // for the bulk of search work done looking for matches. + matchTreeSearch: z.number(), + + // Number of times regexp was called on files that we evaluated. + regexpsConsidered: z.number(), + + // FlushReason explains why results were flushed. + flushReason: z.number(), +}); export const searchResponseSchema = z.object({ - zoektStats: z.object({ - // The duration (in nanoseconds) of the search. - duration: z.number(), - fileCount: z.number(), - matchCount: z.number(), - filesSkipped: z.number(), - contentBytesLoaded: z.number(), - indexBytesLoaded: z.number(), - crashes: z.number(), - shardFilesConsidered: z.number(), - filesConsidered: z.number(), - filesLoaded: z.number(), - shardsScanned: z.number(), - shardsSkipped: z.number(), - shardsSkippedFilter: z.number(), - ngramMatches: z.number(), - ngramLookups: z.number(), - wait: z.number(), - matchTreeConstruction: z.number(), - matchTreeSearch: z.number(), - regexpsConsidered: z.number(), - flushReason: z.number(), - }), + stats: searchStatsSchema, files: z.array(z.object({ fileName: z.object({ // The name of the file @@ -90,6 +140,7 @@ export const searchResponseSchema = z.object({ })), repositoryInfo: z.array(repositoryInfoSchema), isBranchFilteringEnabled: z.boolean(), + isSearchExhaustive: z.boolean(), }); enum RepoIndexingStatus { diff --git a/packages/web/src/app/[domain]/search/page.tsx b/packages/web/src/app/[domain]/search/page.tsx index a92956d9..2d6e497c 100644 --- a/packages/web/src/app/[domain]/search/page.tsx +++ b/packages/web/src/app/[domain]/search/page.tsx @@ -21,19 +21,21 @@ import { FilterPanel } from "./components/filterPanel"; import { SearchResultsPanel } from "./components/searchResultsPanel"; import { useDomain } from "@/hooks/useDomain"; import { useToast } from "@/components/hooks/use-toast"; -import { RepositoryInfo, SearchResultFile } from "@/features/search/types"; +import { RepositoryInfo, SearchResultFile, SearchStats } from "@/features/search/types"; import { AnimatedResizableHandle } from "@/components/ui/animatedResizableHandle"; import { useFilteredMatches } from "./components/filterPanel/useFilterMatches"; import { Button } from "@/components/ui/button"; import { ImperativePanelHandle } from "react-resizable-panels"; -import { FilterIcon } from "lucide-react"; +import { AlertTriangleIcon, BugIcon, FilterIcon } from "lucide-react"; import { useHotkeys } from "react-hotkeys-hook"; import { useLocalStorage } from "@uidotdev/usehooks"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { KeyboardShortcutHint } from "@/app/components/keyboardShortcutHint"; import { SearchBar } from "../components/searchBar"; +import { CodeSnippet } from "@/app/components/codeSnippet"; +import { CopyIconButton } from "../components/copyIconButton"; -const DEFAULT_MAX_MATCH_COUNT = 10000; +const DEFAULT_MAX_MATCH_COUNT = 500; export default function SearchPage() { // We need a suspense boundary here since we are accessing query params @@ -58,7 +60,12 @@ const SearchPageInternal = () => { const _maxMatchCount = parseInt(useNonEmptyQueryParam(SearchQueryParams.matches) ?? `${DEFAULT_MAX_MATCH_COUNT}`); const maxMatchCount = isNaN(_maxMatchCount) ? DEFAULT_MAX_MATCH_COUNT : _maxMatchCount; - const { data: searchResponse, isLoading: isSearchLoading, error } = useQuery({ + const { + data: searchResponse, + isPending: isSearchPending, + isFetching: isFetching, + error + } = useQuery({ queryKey: ["search", searchQuery, maxMatchCount], queryFn: () => measure(() => unwrapServiceError(search({ query: searchQuery, @@ -68,12 +75,12 @@ const SearchPageInternal = () => { }, domain)), "client.search"), select: ({ data, durationMs }) => ({ ...data, - durationMs, + totalClientSearchDurationMs: durationMs, }), enabled: searchQuery.length > 0, refetchOnWindowFocus: false, retry: false, - staleTime: Infinity, + staleTime: 0, }); useEffect(() => { @@ -109,58 +116,31 @@ const SearchPageInternal = () => { const fileLanguages = searchResponse.files?.map(file => file.language) || []; captureEvent("search_finished", { - durationMs: searchResponse.durationMs, - fileCount: searchResponse.zoektStats.fileCount, - matchCount: searchResponse.zoektStats.matchCount, - filesSkipped: searchResponse.zoektStats.filesSkipped, - contentBytesLoaded: searchResponse.zoektStats.contentBytesLoaded, - indexBytesLoaded: searchResponse.zoektStats.indexBytesLoaded, - crashes: searchResponse.zoektStats.crashes, - shardFilesConsidered: searchResponse.zoektStats.shardFilesConsidered, - filesConsidered: searchResponse.zoektStats.filesConsidered, - filesLoaded: searchResponse.zoektStats.filesLoaded, - shardsScanned: searchResponse.zoektStats.shardsScanned, - shardsSkipped: searchResponse.zoektStats.shardsSkipped, - shardsSkippedFilter: searchResponse.zoektStats.shardsSkippedFilter, - ngramMatches: searchResponse.zoektStats.ngramMatches, - ngramLookups: searchResponse.zoektStats.ngramLookups, - wait: searchResponse.zoektStats.wait, - matchTreeConstruction: searchResponse.zoektStats.matchTreeConstruction, - matchTreeSearch: searchResponse.zoektStats.matchTreeSearch, - regexpsConsidered: searchResponse.zoektStats.regexpsConsidered, - flushReason: searchResponse.zoektStats.flushReason, + durationMs: searchResponse.totalClientSearchDurationMs, + fileCount: searchResponse.stats.fileCount, + matchCount: searchResponse.stats.totalMatchCount, + actualMatchCount: searchResponse.stats.actualMatchCount, + filesSkipped: searchResponse.stats.filesSkipped, + contentBytesLoaded: searchResponse.stats.contentBytesLoaded, + indexBytesLoaded: searchResponse.stats.indexBytesLoaded, + crashes: searchResponse.stats.crashes, + shardFilesConsidered: searchResponse.stats.shardFilesConsidered, + filesConsidered: searchResponse.stats.filesConsidered, + filesLoaded: searchResponse.stats.filesLoaded, + shardsScanned: searchResponse.stats.shardsScanned, + shardsSkipped: searchResponse.stats.shardsSkipped, + shardsSkippedFilter: searchResponse.stats.shardsSkippedFilter, + ngramMatches: searchResponse.stats.ngramMatches, + ngramLookups: searchResponse.stats.ngramLookups, + wait: searchResponse.stats.wait, + matchTreeConstruction: searchResponse.stats.matchTreeConstruction, + matchTreeSearch: searchResponse.stats.matchTreeSearch, + regexpsConsidered: searchResponse.stats.regexpsConsidered, + flushReason: searchResponse.stats.flushReason, fileLanguages, }); }, [captureEvent, searchQuery, searchResponse]); - const { fileMatches, searchDurationMs, totalMatchCount, isBranchFilteringEnabled, repositoryInfo, matchCount } = useMemo(() => { - if (!searchResponse) { - return { - fileMatches: [], - searchDurationMs: 0, - totalMatchCount: 0, - isBranchFilteringEnabled: false, - repositoryInfo: {}, - matchCount: 0, - }; - } - - return { - fileMatches: searchResponse.files ?? [], - searchDurationMs: Math.round(searchResponse.durationMs), - totalMatchCount: searchResponse.zoektStats.matchCount, - isBranchFilteringEnabled: searchResponse.isBranchFilteringEnabled, - repositoryInfo: searchResponse.repositoryInfo.reduce((acc, repo) => { - acc[repo.id] = repo; - return acc; - }, {} as RecordSearching...
Failed to search
+{error.message}
+Search stats for nerds
+{`[${searchDurationMs} ms] Found ${numMatches} matches in ${fileMatches.length} ${fileMatches.length > 1 ? 'files' : 'file'}`}
diff --git a/packages/web/src/app/components/codeSnippet.tsx b/packages/web/src/app/components/codeSnippet.tsx index 93ca4de3..e77a24b7 100644 --- a/packages/web/src/app/components/codeSnippet.tsx +++ b/packages/web/src/app/components/codeSnippet.tsx @@ -1,12 +1,12 @@ import { cn } from "@/lib/utils" -export const CodeSnippet = ({ children, className, title }: { children: React.ReactNode, className?: string, title?: string }) => { +export const CodeSnippet = ({ children, className, title, renderNewlines = false }: { children: React.ReactNode, className?: string, title?: string, renderNewlines?: boolean }) => { return (
- {children}
+ {renderNewlines ? {children} : children}
)
}
\ No newline at end of file
diff --git a/packages/web/src/env.mjs b/packages/web/src/env.mjs
index 922b2b84..7a9c1589 100644
--- a/packages/web/src/env.mjs
+++ b/packages/web/src/env.mjs
@@ -15,9 +15,6 @@ export const env = createEnv({
server: {
// Zoekt
ZOEKT_WEBSERVER_URL: z.string().url().default("http://localhost:6070"),
- SHARD_MAX_MATCH_COUNT: numberSchema.default(10000),
- TOTAL_MAX_MATCH_COUNT: numberSchema.default(100000),
- ZOEKT_MAX_WALL_TIME_MS: numberSchema.default(10000),
// Auth
FORCE_ENABLE_ANONYMOUS_ACCESS: booleanSchema.default('false'),
diff --git a/packages/web/src/features/codeNav/actions.ts b/packages/web/src/features/codeNav/actions.ts
index b55cfa30..839ef381 100644
--- a/packages/web/src/features/codeNav/actions.ts
+++ b/packages/web/src/features/codeNav/actions.ts
@@ -80,7 +80,7 @@ export const findSearchBasedSymbolDefinitions = async (
const parseRelatedSymbolsSearchResponse = (searchResult: SearchResponse) => {
const parser = searchResponseSchema.transform(async ({ files }) => ({
stats: {
- matchCount: searchResult.stats.matchCount,
+ matchCount: searchResult.stats.actualMatchCount,
},
files: files.flatMap((file) => {
const chunks = file.chunks;
diff --git a/packages/web/src/features/search/schemas.ts b/packages/web/src/features/search/schemas.ts
index 18dfd8d4..1867d849 100644
--- a/packages/web/src/features/search/schemas.ts
+++ b/packages/web/src/features/search/schemas.ts
@@ -37,35 +37,82 @@ export const repositoryInfoSchema = z.object({
name: z.string(),
displayName: z.string().optional(),
webUrl: z.string().optional(),
-})
+});
+
+// Many of these fields are defined in zoekt/api.go.
+export const searchStatsSchema = z.object({
+ // The actual number of matches returned by the search.
+ // This will always be less than or equal to `totalMatchCount`.
+ actualMatchCount: z.number(),
+
+ // The total number of matches found during the search.
+ totalMatchCount: z.number(),
+
+ // The duration (in nanoseconds) of the search.
+ duration: z.number(),
+
+ // Number of files containing a match.
+ fileCount: z.number(),
+
+ // Candidate files whose contents weren't examined because we
+ // gathered enough matches.
+ filesSkipped: z.number(),
+
+ // Amount of I/O for reading contents.
+ contentBytesLoaded: z.number(),
+
+ // Amount of I/O for reading from index.
+ indexBytesLoaded: z.number(),
+
+ // Number of search shards that had a crash.
+ crashes: z.number(),
+
+ // Number of files in shards that we considered.
+ shardFilesConsidered: z.number(),
+
+ // Files that we evaluated. Equivalent to files for which all
+ // atom matches (including negations) evaluated to true.
+ filesConsidered: z.number(),
+
+ // Files for which we loaded file content to verify substring matches
+ filesLoaded: z.number(),
+
+ // Shards that we scanned to find matches.
+ shardsScanned: z.number(),
+
+ // Shards that we did not process because a query was canceled.
+ shardsSkipped: z.number(),
+
+ // Shards that we did not process because the query was rejected by the
+ // ngram filter indicating it had no matches.
+ shardsSkippedFilter: z.number(),
+
+ // Number of candidate matches as a result of searching ngrams.
+ ngramMatches: z.number(),
+
+ // NgramLookups is the number of times we accessed an ngram in the index.
+ ngramLookups: z.number(),
+
+ // Wall clock time for queued search.
+ wait: z.number(),
+
+ // Aggregate wall clock time spent constructing and pruning the match tree.
+ // This accounts for time such as lookups in the trigram index.
+ matchTreeConstruction: z.number(),
+
+ // Aggregate wall clock time spent searching the match tree. This accounts
+ // for the bulk of search work done looking for matches.
+ matchTreeSearch: z.number(),
+
+ // Number of times regexp was called on files that we evaluated.
+ regexpsConsidered: z.number(),
+
+ // FlushReason explains why results were flushed.
+ flushReason: z.number(),
+});
export const searchResponseSchema = z.object({
- zoektStats: z.object({
- // The duration (in nanoseconds) of the search.
- duration: z.number(),
- fileCount: z.number(),
- matchCount: z.number(),
- filesSkipped: z.number(),
- contentBytesLoaded: z.number(),
- indexBytesLoaded: z.number(),
- crashes: z.number(),
- shardFilesConsidered: z.number(),
- filesConsidered: z.number(),
- filesLoaded: z.number(),
- shardsScanned: z.number(),
- shardsSkipped: z.number(),
- shardsSkippedFilter: z.number(),
- ngramMatches: z.number(),
- ngramLookups: z.number(),
- wait: z.number(),
- matchTreeConstruction: z.number(),
- matchTreeSearch: z.number(),
- regexpsConsidered: z.number(),
- flushReason: z.number(),
- }),
- stats: z.object({
- matchCount: z.number(),
- }),
+ stats: searchStatsSchema,
files: z.array(z.object({
fileName: z.object({
// The name of the file
@@ -92,6 +139,7 @@ export const searchResponseSchema = z.object({
})),
repositoryInfo: z.array(repositoryInfoSchema),
isBranchFilteringEnabled: z.boolean(),
+ isSearchExhaustive: z.boolean(),
});
export const fileSourceRequestSchema = z.object({
diff --git a/packages/web/src/features/search/searchApi.ts b/packages/web/src/features/search/searchApi.ts
index 60d04dec..3c7ea373 100644
--- a/packages/web/src/features/search/searchApi.ts
+++ b/packages/web/src/features/search/searchApi.ts
@@ -151,12 +151,48 @@ export const search = async ({ query, matches, contextLines, whole }: SearchRequ
// @see: https://github.com/sourcebot-dev/zoekt/blob/main/api.go#L892
opts: {
ChunkMatches: true,
+ // @note: Zoekt has several different ways to limit a given search. The two that
+ // we care about are `MaxMatchDisplayCount` and `TotalMaxMatchCount`:
+ // - `MaxMatchDisplayCount` truncates the number of matches AFTER performing
+ // a search (specifically, after collating and sorting the results). The number of
+ // results returned by the API will be less than or equal to this value.
+ //
+ // - `TotalMaxMatchCount` truncates the number of matches DURING a search. The results
+ // returned by the API the API can be less than, equal to, or greater than this value.
+ // Why greater? Because this value is compared _after_ a given shard has finished
+ // being processed, the number of matches returned by the last shard may have exceeded
+ // this value.
+ //
+ // Let's define two variables:
+ // - `actualMatchCount` : The number of matches that are returned by the API. This is
+ // always less than or equal to `MaxMatchDisplayCount`.
+ // - `totalMatchCount` : The number of matches that zoekt found before it either
+ // 1) found all matches or 2) hit the `TotalMaxMatchCount` limit. This number is
+ // not bounded and can be less than, equal to, or greater than both `TotalMaxMatchCount`
+ // and `MaxMatchDisplayCount`.
+ //
+ //
+ // Our challenge is to determine whether or not the search returned all possible matches/
+ // (it was exaustive) or if it was truncated. By setting the `TotalMaxMatchCount` to
+ // `MaxMatchDisplayCount + 1`, we can determine which of these occurred by comparing
+ // `totalMatchCount` to `MaxMatchDisplayCount`.
+ //
+ // if (totalMatchCount ≤ actualMatchCount):
+ // Search is EXHAUSTIVE (found all possible matches)
+ // Proof: totalMatchCount ≤ MaxMatchDisplayCount < TotalMaxMatchCount
+ // Therefore Zoekt stopped naturally, not due to limit
+ //
+ // if (totalMatchCount > actualMatchCount):
+ // Search is TRUNCATED (more matches exist)
+ // Proof: totalMatchCount > MaxMatchDisplayCount + 1 = TotalMaxMatchCount
+ // Therefore Zoekt hit the limit and stopped searching
+ //
MaxMatchDisplayCount: matches,
+ TotalMaxMatchCount: matches + 1,
NumContextLines: contextLines,
Whole: !!whole,
- TotalMaxMatchCount: env.TOTAL_MAX_MATCH_COUNT,
- ShardMaxMatchCount: env.SHARD_MAX_MATCH_COUNT,
- MaxWallTime: env.ZOEKT_MAX_WALL_TIME_MS * 1000 * 1000, // zoekt expects a duration in nanoseconds
+ ShardMaxMatchCount: -1,
+ MaxWallTime: 0, // zoekt expects a duration in nanoseconds
}
});
@@ -296,11 +332,35 @@ export const search = async ({ query, matches, contextLines, whole }: SearchRequ
}
}).filter((file) => file !== undefined) ?? [];
+ const actualMatchCount = files.reduce(
+ (acc, file) =>
+ // Match count is the sum of the number of chunk matches and file name matches.
+ acc + file.chunks.reduce(
+ (acc, chunk) => acc + chunk.matchRanges.length,
+ 0,
+ ) + file.fileName.matchRanges.length,
+ 0,
+ );
+
+ const totalMatchCount = Result.MatchCount;
+ const isSearchExhaustive = totalMatchCount <= actualMatchCount;
+
return {
- zoektStats: {
+ files,
+ repositoryInfo: Array.from(repos.values()).map((repo) => ({
+ id: repo.id,
+ codeHostType: repo.external_codeHostType,
+ name: repo.name,
+ displayName: repo.displayName ?? undefined,
+ webUrl: repo.webUrl ?? undefined,
+ })),
+ isBranchFilteringEnabled,
+ isSearchExhaustive,
+ stats: {
+ actualMatchCount,
+ totalMatchCount,
duration: Result.Duration,
fileCount: Result.FileCount,
- matchCount: Result.MatchCount,
filesSkipped: Result.FilesSkipped,
contentBytesLoaded: Result.ContentBytesLoaded,
indexBytesLoaded: Result.IndexBytesLoaded,
@@ -318,25 +378,6 @@ export const search = async ({ query, matches, contextLines, whole }: SearchRequ
matchTreeSearch: Result.MatchTreeSearch,
regexpsConsidered: Result.RegexpsConsidered,
flushReason: Result.FlushReason,
- },
- files,
- repositoryInfo: Array.from(repos.values()).map((repo) => ({
- id: repo.id,
- codeHostType: repo.external_codeHostType,
- name: repo.name,
- displayName: repo.displayName ?? undefined,
- webUrl: repo.webUrl ?? undefined,
- })),
- isBranchFilteringEnabled: isBranchFilteringEnabled,
- stats: {
- matchCount: files.reduce(
- (acc, file) =>
- acc + file.chunks.reduce(
- (acc, chunk) => acc + chunk.matchRanges.length,
- 0,
- ),
- 0,
- )
}
} satisfies SearchResponse;
});
diff --git a/packages/web/src/features/search/types.ts b/packages/web/src/features/search/types.ts
index f9af8dbe..2a238857 100644
--- a/packages/web/src/features/search/types.ts
+++ b/packages/web/src/features/search/types.ts
@@ -8,6 +8,7 @@ import {
fileSourceRequestSchema,
symbolSchema,
repositoryInfoSchema,
+ searchStatsSchema,
} from "./schemas";
import { z } from "zod";
@@ -22,4 +23,5 @@ export type FileSourceRequest = z.infer