feat: filter context with .cody/.ignore

agent/src/editor.ts

@@ -1,5 +1,6 @@
 import type * as vscode from 'vscode'
 
+import { isCodyIgnoreFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
 import {
     ActiveTextEditor,
     ActiveTextEditorDiagnostic,
@@ -36,6 +37,9 @@ export class AgentEditor implements Editor {
         if (this.agent.workspace.activeDocumentFilePath === null) {
             return undefined
         }
+        if (isCodyIgnoreFile(this.agent.workspace.activeDocumentFilePath)) {
+            return undefined
+        }
         return this.agent.workspace.getDocument(this.agent.workspace.activeDocumentFilePath)
     }
 

lib/shared/src/chat/context-filter.test.ts

@@ -0,0 +1,66 @@
+import { describe, expect, it } from 'vitest'
+
+import { isCodyIgnoreFile, setCodyIgnoreList } from './context-filter'
+
+describe('isCodyIgnoreFile', () => {
+    // Set the ignore list to the following content:
+    const codyIgnoreFileContent = `
+        node_modules
+        **/cody
+        **/foo/**
+        /bar
+        fooz
+        barz/**
+        .git
+        `
+    setCodyIgnoreList(codyIgnoreFileContent)
+
+    it('returns false for no file name', () => {
+        expect(isCodyIgnoreFile()).toBe(false)
+    })
+
+    it('returns true for .env file even if it is not in the ignore list', () => {
+        expect(isCodyIgnoreFile('.env')).toBe(true)
+    })
+
+    it.each([
+        'node_modules/foo',
+        'cody',
+        'cody.ts',
+        'cody/test.ts',
+        'foo/bar',
+        'foo/bar/index.css',
+        'foo/foobarz.js',
+        'fooz',
+        'barz/foo',
+        '.git',
+        '.gitignore',
+        '.git/foo',
+        'foo/.git',
+    ])('returns true for file in ignore list %s', (file: string) => {
+        expect(isCodyIgnoreFile(file)).toBe(true)
+    })
+
+    it.each(['src/app.ts', 'env/foobarz.js', 'foobar.go', '.barz', 'bar'])(
+        'returns false for file not in ignore list %s',
+        (file: string) => {
+            expect(isCodyIgnoreFile(file)).toBe(false)
+        }
+    )
+
+    it('returns updated value after modifying the ignore list', () => {
+        const beforeModifiedCodyIgnoreFileContent = `
+        node_modules
+        cody
+        `
+        setCodyIgnoreList(beforeModifiedCodyIgnoreFileContent)
+
+        expect(isCodyIgnoreFile('cody/index.html')).toBe(true)
+
+        const afterModifiedCodyIgnoreFileContent = `
+        node_modules
+        `
+        setCodyIgnoreList(afterModifiedCodyIgnoreFileContent)
+        expect(isCodyIgnoreFile('cody/index.html')).toBe(false)
+    })
+})

lib/shared/src/chat/context-filter.ts

@@ -0,0 +1,64 @@
+export const CodyIgnoreFileName = '.codyignore'
+
+let codyIgnoreList: string[] = []
+
+/**
+ * Checks if the given file name should be ignored by Cody.
+ *
+ * @param fileName - The file name to check.
+ * @returns A boolean indicating if the file should be ignored.
+ */
+export function isCodyIgnoreFile(fileName?: string): boolean {
+    if (!fileName) {
+        return false
+    }
+    // check if the file name is included by one of the gitignore patterns defined in codyIgnoreList
+    // the pattern is a regex, so we need to escape the special characters
+    for (const pattern of codyIgnoreList) {
+        if (new RegExp(pattern).test(fileName)) {
+            return true
+        }
+    }
+    return false
+}
+
+/**
+ * Parses a Cody ignore file content and sets the global codyIgnoreList array with the rules.
+ *
+ * NOTE: Each client should call this function at the start of the client, and every time the.codyignore file changes.
+ *
+ * The codyIgnoreFileContent string is split into lines. Each non-comment, non-blank line has
+ * the leading ! removed and is added to a Set to remove duplicates. Finally the Set is converted to an array.
+ *
+ * This allows efficiently parsing a ignore file while removing duplicate rules.
+ *
+ * @param codyIgnoreFileContent - The raw string content of the .codyignore file
+ */
+export function setCodyIgnoreList(codyIgnoreFileContent: string): void {
+    // Get a list of files to exclude from the codyignore file
+    const filesToExclude = new Set<string>(['.env'])
+    // Split the content of the file by new lines
+    const codyIgnoreFileLines = codyIgnoreFileContent.toString().split('\n')
+    // Loop through each line of the gitignore file
+    for (const line of codyIgnoreFileLines) {
+        // If the line starts with a #, then it is a comment and we can ignore it
+        if (line.startsWith('#')) {
+            continue
+        }
+        // If the line is blank, then we can ignore it
+        if (!line.trim()) {
+            continue
+        }
+        // Add the rule to the list of rules to exclude
+        filesToExclude.add(patternToRegExpString(line.trim()))
+    }
+    codyIgnoreList = Array.from(filesToExclude)
+}
+
+function patternToRegExpString(pattern: string): string {
+    // Escape special characters and convert '*' and '?' to their regex equivalents
+    return pattern
+        .replace(/^\*\*\//, '')
+        .replaceAll('*', '.*')
+        .replaceAll('?', '.')
+}

lib/shared/src/chat/transcript/interaction.ts

@@ -1,4 +1,5 @@
 import { ContextFile, ContextMessage, PreciseContext } from '../../codebase-context/messages'
+import { isCodyIgnoreFile } from '../context-filter'
 
 import { ChatMessage, InteractionMessage } from './messages'
 
@@ -22,7 +23,35 @@ export class Interaction {
         private usedContextFiles: ContextFile[],
         private usedPreciseContext: PreciseContext[] = [],
         public readonly timestamp: string = new Date().toISOString()
-    ) {}
+    ) {
+        this.removeCodyIgnoredFiles().catch(() => {})
+    }
+
+    /**
+     * Removes context messages for files that should be ignored.
+     *
+     * Loops through the context messages and builds a new array, omitting any
+     * messages for files that match the CODY_IGNORE files filter.
+     * Also omits the assistant message after any ignored human message.
+     *
+     * This ensures context from ignored files does not get used.
+     */
+    private async removeCodyIgnoredFiles(): Promise<void> {
+        const contextMessages = await this.fullContext
+        const newMessages = []
+        for (let i = 0; i < contextMessages.length; i++) {
+            const message = contextMessages[i]
+            if (message.speaker === 'human' && message.file?.fileName) {
+                // find filesToExclude to see if the filename matches
+                if (isCodyIgnoreFile(message.file?.fileName)) {
+                    i++
+                    continue
+                }
+            }
+            newMessages.push(message)
+        }
+        this.fullContext = Promise.resolve(newMessages)
+    }
 
     public getAssistantMessage(): InteractionMessage {
         return { ...this.assistantMessage }

vscode/src/completions/context/context-graph.ts

@@ -1,5 +1,7 @@
 import * as vscode from 'vscode'
 
+import { isCodyIgnoreFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
+
 import { logDebug } from '../../log'
 import { ContextSnippet } from '../types'
 
@@ -45,6 +47,10 @@ export async function getContextFromGraph(options: Options): Promise<GetContextR
     let totalChars = 0
     let includedGraphMatches = 0
     for (const match of graphMatches) {
+        // skip file that is on the ignore list
+        if (isCodyIgnoreFile(match.fileName)) {
+            continue
+        }
         if (totalChars + match.content.length > options.maxChars) {
             continue
         }

vscode/src/completions/context/context-local.ts

@@ -2,6 +2,8 @@ import path from 'path'
 
 import * as vscode from 'vscode'
 
+import { isCodyIgnoreFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
+
 import { ContextSnippet } from '../types'
 
 import { bestJaccardMatch, JaccardMatch } from './bestJaccardMatch'
@@ -21,12 +23,21 @@ interface Options {
 
 export async function getContextFromCurrentEditor(options: Options): Promise<ContextSnippet[]> {
     const { document, history, prefix, jaccardDistanceWindowSize } = options
+    // Return early if current document is on the ignore list
+    if (isCodyIgnoreFile(document.uri.fsPath)) {
+        return []
+    }
 
     const targetText = lastNLines(prefix, jaccardDistanceWindowSize)
     const files = await getRelevantFiles(document, history)
 
     const matches: JaccardMatchWithFilename[] = []
     for (const { uri, contents } of files) {
+        // skip file that is on the ignore list
+        if (isCodyIgnoreFile(uri.fsPath)) {
+            continue
+        }
+
         const match = bestJaccardMatch(targetText, contents, jaccardDistanceWindowSize)
         if (!match) {
             continue
@@ -81,6 +92,11 @@ async function getRelevantFiles(
             return
         }
 
+        // Do not add files that are on the codyignore list
+        if (isCodyIgnoreFile(document.uri.fsPath)) {
+            return
+        }
+
         if (baseLanguageId(document.languageId) !== baseLanguageId(curLang)) {
             return
         }

vscode/src/completions/context/context.ts

@@ -1,5 +1,6 @@
 import * as vscode from 'vscode'
 
+import { isCodyIgnoreFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
 import { CodebaseContext } from '@sourcegraph/cody-shared/src/codebase-context'
 
 import { ContextSnippet } from '../types'
@@ -55,6 +56,10 @@ export async function getContext(options: GetContextOptions): Promise<GetContext
     let totalChars = 0
     function addMatch(match: ContextSnippet): boolean {
         // TODO(@philipp-spiess): We should de-dupe on the snippet range and not
+        // return early if the file is on ignore list
+        if (isCodyIgnoreFile(match.fileName)) {
+            return false
+        }
         // the file name to allow for more than one snippet of the same file
         if (usedFilenames.has(match.fileName)) {
             return false

vscode/src/completions/inline-completion-item-provider.ts

@@ -1,6 +1,7 @@
 import { formatDistance } from 'date-fns'
 import * as vscode from 'vscode'
 
+import { isCodyIgnoreFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
 import { CodebaseContext } from '@sourcegraph/cody-shared/src/codebase-context'
 import { FeatureFlag, featureFlagProvider } from '@sourcegraph/cody-shared/src/experimentation/FeatureFlagProvider'
 import { RateLimitError } from '@sourcegraph/cody-shared/src/sourcegraph-api/errors'
@@ -126,6 +127,11 @@ export class InlineCompletionItemProvider implements vscode.InlineCompletionItem
         // Making it optional here to execute multiple suggestion in parallel from the CLI script.
         token?: vscode.CancellationToken
     ): Promise<AutocompleteResult | null> {
+        // Do not create item for files that are on the cody ignore list
+        if (isCodyIgnoreFile(document.fileName)) {
+            return null
+        }
+
         // Update the last request
         const lastCompletionRequest = this.lastCompletionRequest
         const completionRequest: CompletionRequest = { document, position, context }

vscode/src/editor/vscode-editor.ts

@@ -1,5 +1,6 @@
 import * as vscode from 'vscode'
 
+import { isCodyIgnoreFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
 import type {
     ActiveTextEditor,
     ActiveTextEditorDiagnostic,
@@ -14,6 +15,7 @@ import { SURROUNDING_LINES } from '@sourcegraph/cody-shared/src/prompt/constants
 
 import { CommandsController } from '../custom-prompts/CommandsController'
 import { FixupController } from '../non-stop/FixupController'
+import { updateCodyIgnoreList } from '../services/context-filter'
 import { InlineController } from '../services/InlineController'
 
 import { EditorCodeLenses } from './EditorCodeLenses'
@@ -28,6 +30,8 @@ export class VSCodeEditor implements Editor<InlineController, FixupController, C
         >
     ) {
         new EditorCodeLenses()
+        // Set codyignore list on startup
+        updateCodyIgnoreList().catch(() => {})
     }
 
     public get fileName(): string {
@@ -56,6 +60,9 @@ export class VSCodeEditor implements Editor<InlineController, FixupController, C
         if (!activeEditor) {
             return null
         }
+        if (isCodyIgnoreFile(activeEditor.document.fileName)) {
+            return null
+        }
         const documentUri = activeEditor.document.uri
         const documentText = activeEditor.document.getText()
         const documentSelection = activeEditor.selection

vscode/src/services/context-filter.ts

@@ -0,0 +1,40 @@
+import * as vscode from 'vscode'
+
+import { CodyIgnoreFileName, setCodyIgnoreList } from '@sourcegraph/cody-shared/src/chat/context-filter'
+
+export async function updateCodyIgnoreList(): Promise<void> {
+    // Get the current cody ignore list
+    const update = async (): Promise<void> => {
+        const currentIgnoreList = await getCodyIgnoreFile()
+        if (!currentIgnoreList) {
+            return
+        }
+        // Set the updated cody ignore list
+        setCodyIgnoreList(currentIgnoreList)
+    }
+
+    // Update ignore list on editor change
+    vscode.workspace.onDidChangeTextDocument(async e => {
+        if (e.document.uri.scheme !== 'file') {
+            return
+        }
+        if (e.document.uri.fsPath.endsWith(CodyIgnoreFileName)) {
+            await update()
+        }
+    })
+
+    await update()
+}
+
+async function getCodyIgnoreFile(): Promise<string | undefined> {
+    // Get git ignore file context using the vs code api
+    // First, get the gitignore file from the workspace root directory
+    const codyIgnoreFile = await vscode.workspace.findFiles(CodyIgnoreFileName)
+    // If the gitignore file exists, get the content of the file
+    if (!codyIgnoreFile.length) {
+        return undefined
+    }
+    const bytes = await vscode.workspace.fs.readFile(codyIgnoreFile[0])
+    const decoded = new TextDecoder('utf-8').decode(bytes)
+    return decoded
+}