feat: filter context with .cody/.ignore

agent/src/editor.ts

@@ -1,5 +1,6 @@
 import type * as vscode from 'vscode'
 
+import { isCodyIgnoredFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
 import {
     ActiveTextEditor,
     ActiveTextEditorDiagnostic,
@@ -36,6 +37,9 @@ export class AgentEditor implements Editor {
         if (this.agent.workspace.activeDocumentFilePath === null) {
             return undefined
         }
+        if (isCodyIgnoredFile(this.agent.workspace.activeDocumentFilePath)) {
+            return undefined
+        }
         return this.agent.workspace.getDocument(this.agent.workspace.activeDocumentFilePath)
     }
 

lib/shared/src/chat/context-filter.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, it } from 'vitest'
+
+import { isCodyIgnoredFile, setCodyIgnoreList } from './context-filter'
+
+describe('isCodyIgnoredFile', () => {
+    // Set the ignore list to the following content:
+    const codyignoreFileContent = `
+        node_modules/
+        **/cody
+        **/foo/**
+        /bar
+        fooz
+        barz/*
+        .git
+        `
+    setCodyIgnoreList(codyignoreFileContent)
+
+    it('returns false for no file name', () => {
+        expect(isCodyIgnoredFile()).toBe(false)
+    })
+
+    it('returns true for .env file even if it is not in the ignore list', () => {
+        expect(isCodyIgnoredFile('.env')).toBe(true)
+    })
+
+    it.each([
+        'node_modules/foo',
+        'cody',
+        'cody/test.ts',
+        'foo/foobarz.js',
+        'foo/bar',
+        'fooz',
+        '.git',
+        'barz/index.css',
+        'barz/foo/index.css',
+        'foo/bar/index.css',
+        'foo/.git',
+        '.git/foo',
+    ])('returns true for file in ignore list %s', (file: string) => {
+        expect(isCodyIgnoredFile(file)).toBe(true)
+    })
+
+    it.each(['src/app.ts', 'barz', 'env/foobarz.js', 'foobar.go', '.barz', '.gitignore', 'cody.ts'])(
+        'returns false for file not in ignore list %s',
+        (file: string) => {
+            expect(isCodyIgnoredFile(file)).toBe(false)
+        }
+    )
+
+    it('returns updated value after modifying the ignore list', () => {
+        const beforeModifiedCodyIgnoreFileContent = `
+        node_modules
+        cody/
+        `
+        setCodyIgnoreList(beforeModifiedCodyIgnoreFileContent)
+
+        expect(isCodyIgnoredFile('cody/index.html')).toBe(true)
+
+        const afterModifiedCodyIgnoreFileContent = `
+        node_modules
+        `
+        setCodyIgnoreList(afterModifiedCodyIgnoreFileContent)
+        expect(isCodyIgnoredFile('cody/index.html')).toBe(false)
+    })
+})

lib/shared/src/chat/context-filter.ts

@@ -0,0 +1,44 @@
+import path from 'path'
+
+import ignore from 'ignore'
+
+export const CODY_IGNORE_FILENAME = '.cody/.ignore'
+
+let codyIgnored = ignore().add(['.env'])
+
+/**
+ * Checks if a file should be ignored by Cody based on the .codyignore rules.
+ */
+export function isCodyIgnoredFile(fileName?: string): boolean {
+    if (!fileName) {
+        return false
+    }
+    // file must be path.relative - remove '/' from the start of the path
+    return codyIgnored.ignores(path.join(fileName.replace(/^\//, '')))
+}
+
+/**
+ * Sets the ignore rules for Cody by parsing a .codyignore file.
+ *
+ *  * NOTE: Each client must call this function at startup + every time the .cody/.ignore file & workspace changes.
+ *
+ * Takes the contents of the .codyignore file and the workspace root directory as input.
+ * Splits the file contents into lines, trims whitespace, and adds each non-empty line to a Set of ignore patterns.
+ * Also adds .env files to the ignore patterns by default.
+ * Converts the Set into an Array and passes it to the ignore() library to generate the ignore rules.
+ * Stores the ignore instance and workspace root globally for use later in checking if a file path is ignored.
+ */
+export function setCodyIgnoreList(codyIgnoreFileContent: string): void {
+    // Get a list of files to exclude from the codyignore file
+    const patternList = new Set<string>()
+    patternList.add('.env')
+    // Split the content of the file by new lines
+    const codyIgnoreFileLines = codyIgnoreFileContent.toString().split('\n')
+    // Loop through each line of the gitignore file
+    for (const line of codyIgnoreFileLines) {
+        if (line.trim()) {
+            patternList.add(line.trim())
+        }
+    }
+    codyIgnored = ignore().add(Array.from(patternList))
+}

lib/shared/src/chat/transcript/interaction.ts

@@ -1,4 +1,5 @@
 import { ContextFile, ContextMessage, PreciseContext } from '../../codebase-context/messages'
+import { isCodyIgnoredFile } from '../context-filter'
 
 import { ChatMessage, InteractionMessage } from './messages'
 
@@ -24,6 +25,33 @@ export class Interaction {
         public readonly timestamp: string = new Date().toISOString()
     ) {}
 
+    /**
+     * Removes context messages for files that should be ignored.
+     *
+     * Loops through the context messages and builds a new array, omitting any
+     * messages for files that match the CODY_IGNORE files filter.
+     * Also omits the assistant message after any ignored human message.
+     *
+     * This ensures context from ignored files does not get used.
+     */
+    private async removeCodyIgnoredFiles(): Promise<ContextMessage[]> {
+        const contextMessages = await this.fullContext
+        const newMessages = []
+        for (let i = 0; i < contextMessages.length; i++) {
+            const message = contextMessages[i]
+            if (message.speaker === 'human' && message.file?.fileName) {
+                // find filesToExclude to see if the filename matches
+                if (isCodyIgnoredFile(message.file?.fileName)) {
+                    i++
+                    continue
+                }
+            }
+            newMessages.push(message)
+        }
+        this.fullContext = Promise.resolve(newMessages)
+        return newMessages
+    }
+
     public getAssistantMessage(): InteractionMessage {
         return { ...this.assistantMessage }
     }
@@ -37,12 +65,12 @@ export class Interaction {
     }
 
     public async getFullContext(): Promise<ContextMessage[]> {
-        const msgs = await this.fullContext
+        const msgs = await this.removeCodyIgnoredFiles()
         return msgs.map(msg => ({ ...msg }))
     }
 
     public async hasContext(): Promise<boolean> {
-        const contextMessages = await this.fullContext
+        const contextMessages = await this.removeCodyIgnoredFiles()
         return contextMessages.length > 0
     }
 

package.json

@@ -60,6 +60,7 @@
     "vitest": "^0.33.0"
   },
   "dependencies": {
+    "ignore": "^5.2.4",
     "react": "18.2.0",
     "react-dom": "18.2.0"
   },

vscode/src/completions/context/context-graph.ts

@@ -1,5 +1,7 @@
 import * as vscode from 'vscode'
 
+import { isCodyIgnoredFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
+
 import { logDebug } from '../../log'
 import { ContextSnippet } from '../types'
 
@@ -45,6 +47,10 @@ export async function getContextFromGraph(options: Options): Promise<GetContextR
     let totalChars = 0
     let includedGraphMatches = 0
     for (const match of graphMatches) {
+        // skip file that is on the ignore list
+        if (isCodyIgnoredFile(match.fileName)) {
+            continue
+        }
         if (totalChars + match.content.length > options.maxChars) {
             continue
         }

vscode/src/completions/context/context-local.ts

@@ -2,6 +2,8 @@ import path from 'path'
 
 import * as vscode from 'vscode'
 
+import { isCodyIgnoredFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
+
 import { ContextSnippet } from '../types'
 
 import { bestJaccardMatch, JaccardMatch } from './bestJaccardMatch'
@@ -21,12 +23,21 @@ interface Options {
 
 export async function getContextFromCurrentEditor(options: Options): Promise<ContextSnippet[]> {
     const { document, history, prefix, jaccardDistanceWindowSize } = options
+    // Return early if current document is on the ignore list
+    if (isCodyIgnoredFile(document.uri.fsPath)) {
+        return []
+    }
 
     const targetText = lastNLines(prefix, jaccardDistanceWindowSize)
     const files = await getRelevantFiles(document, history)
 
     const matches: JaccardMatchWithFilename[] = []
     for (const { uri, contents } of files) {
+        // skip file that is on the ignore list
+        if (isCodyIgnoredFile(uri.fsPath)) {
+            continue
+        }
+
         const match = bestJaccardMatch(targetText, contents, jaccardDistanceWindowSize)
         if (!match) {
             continue
@@ -81,6 +92,11 @@ async function getRelevantFiles(
             return
         }
 
+        // Do not add files that are on the codyignore list
+        if (isCodyIgnoredFile(document.uri.fsPath)) {
+            return
+        }
+
         if (baseLanguageId(document.languageId) !== baseLanguageId(curLang)) {
             return
         }

vscode/src/completions/context/context.ts

@@ -1,5 +1,6 @@
 import * as vscode from 'vscode'
 
+import { isCodyIgnoredFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
 import { CodebaseContext } from '@sourcegraph/cody-shared/src/codebase-context'
 
 import { ContextSnippet } from '../types'
@@ -55,6 +56,10 @@ export async function getContext(options: GetContextOptions): Promise<GetContext
     let totalChars = 0
     function addMatch(match: ContextSnippet): boolean {
         // TODO(@philipp-spiess): We should de-dupe on the snippet range and not
+        // return early if the file is on ignore list
+        if (isCodyIgnoredFile(match.fileName)) {
+            return false
+        }
         // the file name to allow for more than one snippet of the same file
         if (usedFilenames.has(match.fileName)) {
             return false

vscode/src/completions/inline-completion-item-provider.ts

@@ -1,6 +1,7 @@
 import { formatDistance } from 'date-fns'
 import * as vscode from 'vscode'
 
+import { isCodyIgnoredFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
 import { CodebaseContext } from '@sourcegraph/cody-shared/src/codebase-context'
 import { FeatureFlag, featureFlagProvider } from '@sourcegraph/cody-shared/src/experimentation/FeatureFlagProvider'
 import { RateLimitError } from '@sourcegraph/cody-shared/src/sourcegraph-api/errors'
@@ -126,6 +127,11 @@ export class InlineCompletionItemProvider implements vscode.InlineCompletionItem
         // Making it optional here to execute multiple suggestion in parallel from the CLI script.
         token?: vscode.CancellationToken
     ): Promise<AutocompleteResult | null> {
+        // Do not create item for files that are on the cody ignore list
+        if (isCodyIgnoredFile(document.fileName)) {
+            return null
+        }
+
         // Update the last request
         const lastCompletionRequest = this.lastCompletionRequest
         const completionRequest: CompletionRequest = { document, position, context }

vscode/src/editor/vscode-editor.ts

@@ -1,5 +1,6 @@
 import * as vscode from 'vscode'
 
+import { isCodyIgnoredFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
 import type {
     ActiveTextEditor,
     ActiveTextEditorDiagnostic,
@@ -56,6 +57,9 @@ export class VSCodeEditor implements Editor<InlineController, FixupController, C
         if (!activeEditor) {
             return null
         }
+        if (isCodyIgnoredFile(activeEditor.document.fileName)) {
+            return null
+        }
         const documentUri = activeEditor.document.uri
         const documentText = activeEditor.document.getText()
         const documentSelection = activeEditor.selection

vscode/src/main.ts

@@ -22,6 +22,7 @@ import { configureExternalServices } from './external-services'
 import { FixupController } from './non-stop/FixupController'
 import { showSetupNotification } from './notifications/setup-notification'
 import { AuthProvider } from './services/AuthProvider'
+import { getCodyignoreFileWatcher } from './services/context-filter'
 import { showFeedbackSupportQuickPick } from './services/FeedbackOptions'
 import { GuardrailsProvider } from './services/GuardrailsProvider'
 import { Comment, InlineController } from './services/InlineController'
@@ -79,6 +80,12 @@ const register = async (
         context.extensionMode === vscode.ExtensionMode.Test
     await createOrUpdateEventLogger(initialConfig, isExtensionModeDevOrTest)
 
+    // Set codyignore list on startup
+    const codyIgnoreWatcher = await getCodyignoreFileWatcher()
+    if (codyIgnoreWatcher) {
+        disposables.push(codyIgnoreWatcher)
+    }
+
     // Controller for inline Chat
     const commentController = new InlineController(context.extensionPath)
     // Controller for Non-Stop Cody