Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

symf: support cancelling index #2202

Merged
merged 2 commits into from
Dec 8, 2023
Merged

symf: support cancelling index #2202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7d87d73
symf: support cancelling index
beyang Dec 8, 2023
b050ea3
Update vscode/src/local-context/symf.ts
beyang Dec 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 69 additions & 54 deletions vscode/src/local-context/symf.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,33 @@ import { getSymfPath } from './download-symf'

const execFile = promisify(_execFile)

export class SymfRunner implements IndexedKeywordContextFetcher {
export interface IndexStartEvent {
scopeDir: string
cancel: () => void
done: Promise<void>
}

export interface IndexEndEvent {
scopeDir: string
}

export class SymfRunner implements IndexedKeywordContextFetcher, vscode.Disposable {
// The root of all symf index directories
private indexRoot: string

private indexLocks: Map<string, RWLock> = new Map()

private indexStartEmitter = new vscode.EventEmitter<IndexStartEvent>()
public onIndexStart(cb: (e: IndexStartEvent) => void): vscode.Disposable {
return this.indexStartEmitter.event(cb)
}

private indexEndEmitter = new vscode.EventEmitter<IndexEndEvent>()
public onIndexEnd(cb: (e: IndexEndEvent) => void): vscode.Disposable {
return this.indexEndEmitter.event(cb)
}

private disposables: vscode.Disposable[] = [this.indexStartEmitter, this.indexEndEmitter]

constructor(
private context: vscode.ExtensionContext,
private sourcegraphServerEndpoint: string | null,
Expand All @@ -31,28 +52,15 @@ export class SymfRunner implements IndexedKeywordContextFetcher {
this.indexRoot = path.join(os.homedir(), '.cody-symf')
}

public dispose(): void {
this.disposables.forEach(d => d.dispose())
}

public setSourcegraphAuth(endpoint: string | null, authToken: string | null): void {
this.sourcegraphServerEndpoint = endpoint
this.authToken = authToken
}

private indexListeners: Set<(scopeDir: string) => void> = new Set()

public registerIndexListener(onIndexChange: (scopeDir: string) => void): vscode.Disposable {
this.indexListeners.add(onIndexChange)
return {
dispose: () => {
this.indexListeners.delete(onIndexChange)
},
}
}

private fireIndexListeners(scopeDir: string): void {
for (const listener of this.indexListeners) {
listener(scopeDir)
}
}

private async getSymfInfo(): Promise<{ symfPath: string; serverEndpoint: string; accessToken: string }> {
const accessToken = this.authToken
if (!accessToken) {
Expand All @@ -69,11 +77,7 @@ export class SymfRunner implements IndexedKeywordContextFetcher {
return { accessToken, serverEndpoint, symfPath }
}

public async getResults(
userQuery: string,
scopeDirs: string[],
showIndexProgress?: (scopeDir: string, indexDone: Promise<void>) => void
): Promise<Promise<Result[]>[]> {
public async getResults(userQuery: string, scopeDirs: string[]): Promise<Promise<Result[]>[]> {
const { symfPath, serverEndpoint, accessToken } = await this.getSymfInfo()
const expandedQuery = execFile(symfPath, ['expand-query', userQuery], {
env: {
Expand All @@ -84,25 +88,21 @@ export class SymfRunner implements IndexedKeywordContextFetcher {
timeout: 1000 * 10, // timeout in 10 seconds
}).then(({ stdout }) => stdout.trim())

return scopeDirs.map(scopeDir => this.getResultsForScopeDir(expandedQuery, scopeDir, showIndexProgress))
return scopeDirs.map(scopeDir => this.getResultsForScopeDir(expandedQuery, scopeDir))
}

/**
* Returns the list of results from symf for a single directory scope.
* @param keywordQuery is a promise, because query expansion might be an expensive
* operation that is best done concurrently with querying and (re)building the index.
*/
private async getResultsForScopeDir(
keywordQuery: Promise<string>,
scopeDir: string,
showIndexProgress?: (scopeDir: string, indexDone: Promise<void>) => void
): Promise<Result[]> {
private async getResultsForScopeDir(keywordQuery: Promise<string>, scopeDir: string): Promise<Result[]> {
const maxRetries = 10

// Run in a loop in case the index is deleted before we can query it
for (let i = 0; i < maxRetries; i++) {
await this.getIndexLock(scopeDir).withWrite(async () => {
await this.unsafeEnsureIndex(scopeDir, showIndexProgress, { hard: i === 0 })
await this.unsafeEnsureIndex(scopeDir, { hard: i === 0 })
})

let indexNotFound = false
Expand Down Expand Up @@ -135,13 +135,9 @@ export class SymfRunner implements IndexedKeywordContextFetcher {
})
}

public async ensureIndex(
scopeDir: string,
showIndexProgress?: (scopeDir: string, indexDone: Promise<void>) => void,
options: { hard: boolean } = { hard: false }
): Promise<void> {
public async ensureIndex(scopeDir: string, options: { hard: boolean } = { hard: false }): Promise<void> {
await this.getIndexLock(scopeDir).withWrite(async () => {
await this.unsafeEnsureIndex(scopeDir, showIndexProgress, options)
await this.unsafeEnsureIndex(scopeDir, options)
})
}

Expand Down Expand Up @@ -205,11 +201,7 @@ export class SymfRunner implements IndexedKeywordContextFetcher {
return fileExists(path.join(indexDir, 'index.json'))
}

private async unsafeEnsureIndex(
scopeDir: string,
showIndexProgress?: (scopeDir: string, indexDone: Promise<void>) => void,
options: { hard: boolean } = { hard: false }
): Promise<void> {
private async unsafeEnsureIndex(scopeDir: string, options: { hard: boolean } = { hard: false }): Promise<void> {
const indexExists = await this.unsafeIndexExists(scopeDir)
if (indexExists) {
return
Expand All @@ -223,7 +215,7 @@ export class SymfRunner implements IndexedKeywordContextFetcher {

const { indexDir, tmpDir } = this.getIndexDir(scopeDir)
try {
await this.unsafeUpsertIndex(indexDir, tmpDir, scopeDir, showIndexProgress)
await this.unsafeUpsertIndex(indexDir, tmpDir, scopeDir)
} catch (error) {
logDebug('symf', 'symf index creation failed', error)
await this.markIndexFailed(scopeDir)
Expand All @@ -240,21 +232,20 @@ export class SymfRunner implements IndexedKeywordContextFetcher {
}
}

private unsafeUpsertIndex(
private unsafeUpsertIndex(indexDir: string, tmpIndexDir: string, scopeDir: string): Promise<void> {
const cancellation = new vscode.CancellationTokenSource()
const upsert = this._unsafeUpsertIndex(indexDir, tmpIndexDir, scopeDir, cancellation.token)
this.indexStartEmitter.fire({ scopeDir, done: upsert, cancel: () => cancellation.cancel() })
void upsert.then(() => this.indexEndEmitter.fire({ scopeDir })).finally(() => cancellation.dispose())
return upsert
}

private async _unsafeUpsertIndex(
indexDir: string,
tmpIndexDir: string,
scopeDir: string,
showIndexProgress?: (scopeDir: string, indexDone: Promise<void>) => void
cancellationToken: vscode.CancellationToken
): Promise<void> {
const upsert = this._unsafeUpsertIndex(indexDir, tmpIndexDir, scopeDir)
void upsert.then(() => this.fireIndexListeners(scopeDir))
if (showIndexProgress) {
showIndexProgress(scopeDir, upsert)
}
return upsert
}

private async _unsafeUpsertIndex(indexDir: string, tmpIndexDir: string, scopeDir: string): Promise<void> {
const symfPath = await getSymfPath(this.context)
if (!symfPath) {
return
Expand All @@ -269,6 +260,13 @@ export class SymfRunner implements IndexedKeywordContextFetcher {
if (os.cpus().length > 4) {
maxCPUs = 2
}

const disposeOnFinish: vscode.Disposable[] = []
if (cancellationToken.isCancellationRequested) {
throw new vscode.CancellationError()
}

let wasCancelled = false
try {
const proc = spawn(symfPath, ['--index-root', tmpIndexDir, 'add', scopeDir], {
env: {
Expand All @@ -278,6 +276,19 @@ export class SymfRunner implements IndexedKeywordContextFetcher {
stdio: ['ignore', 'ignore', 'ignore'],
timeout: 1000 * 60 * 10, // timeout in 10 minutes
})

if (cancellationToken.isCancellationRequested) {
wasCancelled = true
proc.kill('SIGKILL')
} else {
disposeOnFinish.push(
cancellationToken.onCancellationRequested(() => {
wasCancelled = true
proc.kill('SIGKILL')
})
)
}

// wait for proc to finish
await new Promise<void>((resolve, reject) => {
proc.on('error', reject)
Expand All @@ -292,8 +303,12 @@ export class SymfRunner implements IndexedKeywordContextFetcher {
await mkdirp(path.dirname(indexDir))
await rename(tmpIndexDir, indexDir)
} catch (error) {
if (wasCancelled) {
throw new vscode.CancellationError()
}
throw toSymfError(error)
} finally {
disposeOnFinish.forEach(d => d.dispose())
await rm(tmpIndexDir, { recursive: true, force: true })
}
}
Expand Down
5 changes: 4 additions & 1 deletion vscode/src/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,9 @@ const register = async (
await authProvider.init()

const symfRunner = platform.createSymfRunner?.(context, initialConfig.serverEndpoint, initialConfig.accessToken)
if (symfRunner) {
disposables.push(symfRunner)
}

graphqlClient.onConfigurationChange(initialConfig)
void featureFlagProvider.syncAuthStatus()
Expand Down Expand Up @@ -200,8 +203,8 @@ const register = async (

if (symfRunner) {
const searchViewProvider = new SearchViewProvider(context.extensionUri, symfRunner)
searchViewProvider.initialize()
disposables.push(searchViewProvider)
searchViewProvider.initialize()
disposables.push(
vscode.window.registerWebviewViewProvider('cody.search', searchViewProvider, {
webviewOptions: { retainContextWhenHidden: true },
Expand Down
51 changes: 35 additions & 16 deletions vscode/src/search/SearchViewProvider.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { Result, SearchPanelFile, SearchPanelSnippet } from '@sourcegraph/cody-s

import { WebviewMessage } from '../chat/protocol'
import { getActiveEditor } from '../editor/active-editor'
import { SymfRunner } from '../local-context/symf'
import { IndexStartEvent, SymfRunner } from '../local-context/symf'

const searchDecorationType = vscode.window.createTextEditorDecorationType({
backgroundColor: new vscode.ThemeColor('searchEditor.findMatchBackground'),
Expand Down Expand Up @@ -36,11 +36,18 @@ class CancellationManager implements vscode.Disposable {
}
}

class IndexManager {
class IndexManager implements vscode.Disposable {
private currentlyRefreshing = new Set<string>()
private scopeDirIndexInProgress: Map<string, Promise<void>> = new Map()
private disposables: vscode.Disposable[] = []

constructor(private symf: SymfRunner) {
this.disposables.push(this.symf.onIndexStart(event => this.showIndexProgress(event)))
}

constructor(private symf: SymfRunner) {}
public dispose(): void {
this.disposables.forEach(d => d.dispose())
}

/**
* Show a warning message if indexing is already in progress for scopeDirs.
Expand All @@ -62,26 +69,31 @@ class IndexManager {
void vscode.window.showWarningMessage(`Still indexing: ${indexingScopeDirs.join(', ')}`)
}

public showIndexProgress = (scopeDir: string, indexDone: Promise<void>): void => {
public showIndexProgress({ scopeDir, cancel, done }: IndexStartEvent): void {
const { base, dir, wsName } = getRenderableComponents(scopeDir)
const prettyScopeDir = wsName ? path.join(wsName, dir, base) : path.join(dir, base)
if (this.scopeDirIndexInProgress.has(scopeDir)) {
void vscode.window.showWarningMessage(`Duplicate index request for ${prettyScopeDir}`)
return
}
this.scopeDirIndexInProgress.set(scopeDir, indexDone)
void indexDone.finally(() => {
this.scopeDirIndexInProgress.set(scopeDir, done)
void done.finally(() => {
this.scopeDirIndexInProgress.delete(scopeDir)
})

void vscode.window.withProgress(
{
location: vscode.ProgressLocation.Notification,
title: `Building Cody search index for ${prettyScopeDir}`,
cancellable: false,
cancellable: true,
},
async () => {
await indexDone
async (_progress, token) => {
if (token.isCancellationRequested) {
cancel()
} else {
token.onCancellationRequested(() => cancel())
}
await done
}
)
}
Expand All @@ -94,9 +106,11 @@ class IndexManager {
this.currentlyRefreshing.add(scopeDir)

await this.symf.deleteIndex(scopeDir)
await this.symf.ensureIndex(scopeDir, this.showIndexProgress, { hard: true })
await this.symf.ensureIndex(scopeDir, { hard: true })
} catch (error) {
void vscode.window.showErrorMessage(`Error refreshing search index for ${scopeDir}: ${error}`)
if (!(error instanceof vscode.CancellationError)) {
void vscode.window.showErrorMessage(`Error refreshing search index for ${scopeDir}: ${error}`)
}
} finally {
this.currentlyRefreshing.delete(scopeDir)
}
Expand All @@ -114,6 +128,7 @@ export class SearchViewProvider implements vscode.WebviewViewProvider, vscode.Di
private symfRunner: SymfRunner
) {
this.indexManager = new IndexManager(this.symfRunner)
this.disposables.push(this.indexManager)
this.disposables.push(this.cancellationManager)
}

Expand Down Expand Up @@ -147,19 +162,19 @@ export class SearchViewProvider implements vscode.WebviewViewProvider, vscode.Di
)
// Kick off search index creation for all workspace folders
vscode.workspace.workspaceFolders?.forEach(folder => {
void this.symfRunner.ensureIndex(folder.uri.fsPath, this.indexManager.showIndexProgress, { hard: false })
void this.symfRunner.ensureIndex(folder.uri.fsPath, { hard: false })
})
this.disposables.push(
vscode.workspace.onDidChangeWorkspaceFolders(event => {
event.added.forEach(folder => {
void this.symfRunner.ensureIndex(folder.uri.fsPath, this.indexManager.showIndexProgress, {
void this.symfRunner.ensureIndex(folder.uri.fsPath, {
hard: false,
})
})
})
)
this.disposables.push(
this.symfRunner.registerIndexListener(scopeDir => {
this.symfRunner.onIndexEnd(({ scopeDir }) => {
void this.webview?.postMessage({ type: 'index-updated', scopeDir })
})
)
Expand Down Expand Up @@ -254,7 +269,7 @@ export class SearchViewProvider implements vscode.WebviewViewProvider, vscode.Di
await vscode.window.withProgress({ location: { viewId: 'cody.search' } }, async () => {
const cumulativeResults: SearchPanelFile[] = []
this.indexManager.showMessageIfIndexingInProgress(scopeDirs)
const resultSets = await symf.getResults(query, scopeDirs, this.indexManager.showIndexProgress)
const resultSets = await symf.getResults(query, scopeDirs)
for (const resultSet of resultSets) {
try {
cumulativeResults.push(...(await resultsToDisplayResults(await resultSet)))
Expand All @@ -264,7 +279,11 @@ export class SearchViewProvider implements vscode.WebviewViewProvider, vscode.Di
query,
})
} catch (error) {
void vscode.window.showErrorMessage(`Error fetching results for query, "${query}": ${error}`)
if (error instanceof vscode.CancellationError) {
void vscode.window.showErrorMessage('No search results because indexing was canceled')
} else {
void vscode.window.showErrorMessage(`Error fetching results for query "${query}": ${error}`)
}
}
}
})
Expand Down
Loading