feat: filter context from .cody/.ignore (#1382)

CLOSE #1049

For details, please see [RFC 852 File Exclusion Patterns for
Cody](https://docs.google.com/document/d/15XJnQ-qa-Y3SM2eltuKfuESmgYYfx7JhEdi2447YF_0/edit?usp=sharing)

feat: filter context from ignored files

> IMPORTANT: Currently behind the `cody.internal.unstable` config and
must be enabled to try the feature. When this is out of internal
experimental stage, this should be enabled by default.

- Add `isCodyIgnoreFile` utility to check if file is defined in
.cody/.ignore (at workspace level)
- Update Interaction to filter context messages from ignored files when
first created
- Update context gathering in completions to skip ignored files
- Add check in editor to skip ignored files
- ignore all `.env` files / path by default

All clients should provide file content from the .cody/.ignore file at
start up, and update the ignoreList when the file is changed.

## Note

This PR includes update to the VS Code clients so that the ignore list
is set at Editor start up, and whenever the .cody/.ignore file is
updated, the ignore list will be updated accordingly.

This does not make any changes to the current embedding steps or BFG,
which will be a feature to be implemented in the future as listed by Dom
in the discussion below.

- [x] PRFAQ to agree on the `.cody/.ignore`name

## Test plan

<!-- Required. See
https://docs.sourcegraph.com/dev/background-information/testing_principles.
-->

Prep: 

<img width="1000" alt="image"
src="https://github.com/sourcegraph/cody/assets/68532117/703840e0-042e-431e-b90e-149a254d1d83">


- Build Cody from this branch, start in VS Code debug mode
- Turn on the Unstable Experimental Features from the Cody Settings
- Open the `cody` repository 
- Create a .codyignore file
- Add the following to the .codyignore file
```
vscode/**
utils.ts
```

### Chat

Ask cody about fixup in Chat View: "what is fixup?" 

You should see Cody did not read any files from the `vscode` directory
to answer your question:

<img width="2001" alt="image"
src="https://github.com/sourcegraph/cody/assets/68532117/b53493e7-f382-4325-8c47-5b2229340aac">

### Autocomplete 

Go to any of the file inside the `vscode` directory, you should not get
any autocomplete suggestions.

<img width="1010" alt="image"
src="https://github.com/sourcegraph/cody/assets/68532117/aabc747f-3734-43bb-b3b2-f17e8bac3cdc">

### Command

Go to lib/shared/src/utils.ts file, and then select some code, right
click and select `Explain Code`:

![image](https://github.com/sourcegraph/cody/assets/68532117/37f2626e-4aeb-4fe7-af56-1bc6ba16ef8a)

You should see cody explain a random code (hallucinating because we
didn't provide any code as context) that is not your selected code.

<img width="2011" alt="image"
src="https://github.com/sourcegraph/cody/assets/68532117/735d3a17-587b-46e2-8c26-1f1e9762eb7a">

You can check the output channel to confirm no context was sent to the
LLM:

<img width="2006" alt="image"
src="https://github.com/sourcegraph/cody/assets/68532117/a8bede49-f161-455c-98e6-d852b95f3870">


## Demo


https://github.com/sourcegraph/cody/assets/68532117/9ea47b4c-311b-40b4-954b-db6d91769a21


### Update

Here is a loom video of codyignore working in Simple chat panel:

https://www.loom.com/share/83fe3e0b0728449385d130d45c317cfa

---------

Co-authored-by: Philipp Spiess <hello@philippspiess.com>
Co-authored-by: Danny Tuppeny <danny@tuppeny.com>

agent/src/editor.ts

@@ -1,6 +1,7 @@
 import type * as vscode from 'vscode'
 import { URI } from 'vscode-uri'
 
+import { isCodyIgnoredFile } from '@sourcegraph/cody-shared/src/chat/context-filter'
 import {
     ActiveTextEditor,
     ActiveTextEditorDiagnostic,
@@ -38,6 +39,9 @@ export class AgentEditor implements Editor {
         if (this.agent.workspace.activeDocumentFilePath === null) {
             return undefined
         }
+        if (isCodyIgnoredFile(URI.file(this.agent.workspace.activeDocumentFilePath.fsPath))) {
+            return undefined
+        }
         return this.agent.workspace.getDocument(this.agent.workspace.activeDocumentFilePath)?.underlying
     }
 

lib/shared/src/chat/context-filter.ts

@@ -0,0 +1,24 @@
+import { URI } from 'vscode-uri'
+
+import { IgnoreHelper } from './ignore-helper'
+
+export const ignores = new IgnoreHelper()
+
+/**
+ * Checks if a file should be ignored by Cody based on the ignore rules.
+ *
+ * Takes URI to ensure absolute file paths are ignored correctly across workspaces
+ */
+export function isCodyIgnoredFile(uri: URI): boolean {
+    return ignores.isIgnored(uri)
+}
+
+/**
+ * Checks if the relative path of a file should be ignored by Cody based on the ignore rules.
+ *
+ * Use for matching search results returned from the embeddings client that do not contain absolute path/URI
+ * In isIgnoredByCurrentWorkspace, we will construct a URI with the relative path and workspace root before checking
+ */
+export function isCodyIgnoredFilePath(codebase: string, relativePath: string): boolean {
+    return ignores.isIgnoredByCodebase(codebase.trim(), relativePath)
+}

lib/shared/src/chat/ignore-helper.test.ts

@@ -0,0 +1,190 @@
+import path from 'path'
+
+import { beforeEach, describe, expect, it } from 'vitest'
+// TODO(dantup): Determine whether we should be using vscode-uri URI here, or vscode.URI/shim
+//  (or if we should be testing both).
+import { URI, Utils } from 'vscode-uri'
+
+import { testFilePath } from '../test/path-helpers'
+
+import { CODY_IGNORE_FILENAME, IgnoreHelper } from './ignore-helper'
+
+describe('IgnoreHelper', () => {
+    let ignore: IgnoreHelper
+    const workspace1Root = URI.file(testFilePath('foo/workspace1'))
+    const workspace2Root = URI.file(testFilePath('foo/workspace2'))
+
+    function setIgnores(workspaceRoot: string, ignoreFolder: string, rules: string[]) {
+        ignore.setIgnoreFiles(workspaceRoot, [
+            {
+                filePath: path.join(ignoreFolder, CODY_IGNORE_FILENAME),
+                content: rules.join('\n'),
+            },
+        ])
+    }
+
+    function setWorkspace1Ignores(rules: string[]) {
+        setIgnores(workspace1Root.fsPath, workspace1Root.fsPath, rules)
+    }
+
+    function setWorkspace2Ignores(rules: string[]) {
+        setIgnores(workspace2Root.fsPath, workspace2Root.fsPath, rules)
+    }
+
+    function setWorkspace1NestedIgnores(folder: string, rules: string[]) {
+        setIgnores(workspace1Root.fsPath, path.join(workspace1Root.fsPath, folder), rules)
+    }
+
+    beforeEach(() => {
+        ignore = new IgnoreHelper()
+        ignore.setActiveState(true)
+    })
+
+    it('returns false for an undefined workspace', () => {
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'foo.txt'))).toBe(false)
+    })
+
+    it('returns false for a workspace with no ignores', () => {
+        setWorkspace1Ignores([])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'foo.txt'))).toBe(false)
+    })
+
+    it('returns true for ".env" in an undefined workspace', () => {
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, '.env'))).toBe(true)
+    })
+
+    it('returns true for ".env" in a workspace with no ignores', () => {
+        setWorkspace1Ignores([])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'a', '.env'))).toBe(true)
+    })
+
+    it('returns true for a nested ".env" in a workspace with no ignores', () => {
+        setWorkspace1Ignores([])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'a', '.env'))).toBe(true)
+    })
+
+    it('returns true for a nested ".env" in a workspace with unrelated ignores', () => {
+        setWorkspace1Ignores(['ignored.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'a', '.env'))).toBe(true)
+    })
+
+    it('returns true for a top-level file ignored at the top level', () => {
+        setWorkspace1Ignores(['ignored.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'ignored.txt'))).toBe(true)
+    })
+
+    it('returns false for a top-level file not ignored at the top level', () => {
+        setWorkspace1Ignores(['ignored.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'not_ignored.txt'))).toBe(false)
+    })
+
+    it('returns false for a top-level file unignored at the top level', () => {
+        setWorkspace1Ignores(['*ignored.txt', '!not_ignored.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'not_ignored.txt'))).toBe(false)
+    })
+
+    it('returns true for a nested file ignored at the top level', () => {
+        setWorkspace1Ignores(['always_ignored.txt', 'a/explitly_ignored.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'a/always_ignored.txt'))).toBe(true)
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'a/explitly_ignored.txt'))).toBe(true)
+    })
+
+    it('returns false for a nested file not ignored at the top level', () => {
+        setWorkspace1Ignores(['a/ignored.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'b/ignored.txt'))).toBe(false)
+    })
+
+    it('returns false for a nested file unignored at the top level', () => {
+        setWorkspace1Ignores(['*ignored.txt', '!not_ignored.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'b/not_ignored.txt'))).toBe(false)
+    })
+
+    it('returns true for a nested file ignored at the nested level', () => {
+        setWorkspace1NestedIgnores('a', ['ignored.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'a/ignored.txt'))).toBe(true)
+    })
+
+    it('returns false for a nested file not ignored at the nested level', () => {
+        setWorkspace1NestedIgnores('a', ['ignored.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'a/not_ignored.txt'))).toBe(false)
+    })
+
+    it('returns false for a nested file unignored at the nested level', () => {
+        setWorkspace1NestedIgnores('a', ['*ignored.txt', '!not_ignored.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'a/not_ignored.txt'))).toBe(false)
+    })
+
+    it('tracks ignores independently for each workspace root', () => {
+        setWorkspace1Ignores(['ignored_1.txt'])
+        setWorkspace2Ignores(['ignored_2.txt'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'ignored_1.txt'))).toBe(true)
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, 'ignored_2.txt'))).toBe(false)
+        expect(ignore.isIgnored(Utils.joinPath(workspace2Root, 'ignored_1.txt'))).toBe(false)
+        expect(ignore.isIgnored(Utils.joinPath(workspace2Root, 'ignored_2.txt'))).toBe(true)
+    })
+
+    it('throws on an empty URI', () => {
+        expect(() => ignore.isIgnored(URI.file(''))).toThrow()
+    })
+
+    it.skip('throws on a relative Uri', () => {
+        const relativeFileUri = URI.file('a')
+        expect(() => ignore.isIgnored(relativeFileUri)).toThrow()
+    })
+
+    it('handles comments and blank lines in the ignore file', () => {
+        setWorkspace1Ignores(['#.foo', '', 'bar'])
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, '.env'))).toBe(true)
+        expect(ignore.isIgnored(Utils.joinPath(workspace1Root, '.foo'))).toBe(false)
+    })
+
+    describe('returns the correct value for a sample of rules', () => {
+        beforeEach(() => {
+            setWorkspace1Ignores([
+                'node_modules/',
+                '**/cody',
+                '**/foo/**',
+                '/bar',
+                'fooz',
+                'barz/*',
+                '.git',
+                'one/**/two',
+            ])
+        })
+
+        it.each([
+            'node_modules/foo',
+            'cody',
+            'cody/test.ts',
+            'foo/foobarz.js',
+            'foo/bar',
+            'fooz',
+            '.git',
+            'barz/index.css',
+            'barz/foo/index.css',
+            'foo/bar/index.css',
+            'foo/.git',
+            '.git/foo',
+            'one/two',
+            'one/two/three',
+            'one/a/two',
+            'one/a/two/three',
+        ])('returns true for file in ignore list %s', (filePath: string) => {
+            expect(ignore.isIgnored(Utils.joinPath(workspace1Root, filePath))).toBe(true)
+        })
+
+        it.each([
+            'src/app.ts',
+            'barz',
+            'env/foobarz.js',
+            'foobar.go',
+            '.barz',
+            '.gitignore',
+            'cody.ts',
+            'one/three',
+            'two/one',
+        ])('returns false for file not in ignore list %s', (filePath: string) => {
+            expect(ignore.isIgnored(Utils.joinPath(workspace1Root, filePath))).toBe(false)
+        })
+    })
+})