configurable .gitignore in search files (#548)

* making gitignore optional * updating deps * docs * docs * more docs
microsoft · Jun 18, 2024 · f18adc8 · f18adc8
1 parent 0745136
commit f18adc8
Show file tree

Hide file tree

Showing 31 changed files with 859 additions and 827 deletions.
diff --git a/demo/genaisrc/genaiscript.d.ts b/demo/genaisrc/genaiscript.d.ts
diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts
diff --git a/docs/package.json b/docs/package.json
@@ -20,7 +20,7 @@
   "dependencies": {
     "@astrojs/check": "^0.7.0",
     "@astrojs/starlight": "^0.24.2",
-    "astro": "^4.10.2",
+    "astro": "^4.10.3",
     "typescript": "5.4.5"
   },
   "devDependencies": {

diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md
@@ -16,6 +16,7 @@ Runs a GenAIScript against files.
 
 Options:
   -ef, --excluded-files <string...>          excluded files
+  -egi, --exclude-git-ignore                 exclude files that are ignore through the .gitignore file in the workspace root
   -o, --out <string>                         output folder. Extra markdown fields for output and trace will also be generated
   -rmo, --remove-out                         remove output folder if it exists
   -ot, --out-trace <string>                  output file for trace
@@ -56,6 +57,8 @@ Run a tool on a batch of specs
 
 Options:
   -ef, --excluded-files <string...>  excluded files
+  -egi, --exclude-git-ignore         exclude files that are ignore through the
+                                     .gitignore file in the workspace root
   -o, --out <folder>                 output folder. Extra markdown fields for
                                      output and trace will also be generated
   -rmo, --remove-out                 remove output folder if it exists
@@ -115,7 +118,7 @@ Options:
   -td, --test-delay <string>          delay between tests in seconds
   --no-cache                          disable LLM result cache
   -v, --verbose                       verbose output
-  -pv, --promptfoo-version [version]  promptfoo version, default is ^0.63.2
+  -pv, --promptfoo-version [version]  promptfoo version, default is ^0.65.0
   -os, --out-summary <file>           append output summary in file
   -h, --help                          display help for command
 ```

diff --git a/docs/src/content/docs/reference/cli/run.md b/docs/src/content/docs/reference/cli/run.md
@@ -36,22 +36,30 @@ See [configuration](/genaiscript/getting-started/configuration).
 
 Excludes the specified files from the file set.
 
-```sh
-npx genaiscript run <script> <spec> --excluded-files <excluded-files...>
+```sh "--excluded-files <excluded-files...>"
+npx genaiscript run <script> <files> --excluded-files <excluded-files...>
+```
+
+### --exclude-git-ignore
+
+Exclude files ignored by the `.gitignore` file at the workspace root.
+
+```sh "--exclude-git-ignore"
+npx genaiscript run <script> <files> --exclude-git-ignore
 ```
 
 ### --out <file|directory>
 
 Saves the results in a JSON file, along with markdown files of the output and the trace.
 
-```sh
-npx genaiscript run <script> <spec> --out output/results.json
+```sh "--out tmp"
+npx genaiscript run <script> <files> --out out/res.json
 ```
 
 If `file` does not end with `.json`, the path is treated as a directory path.
 
-```sh
-npx genaiscript run <script> <spec> --out output
+```sh "--out tmp"
+npx genaiscript run <script> <files> --out tmp
 ```
 
 ### --json
@@ -71,29 +79,29 @@ Populate values in the `env.vars` map that can be used when running the prompt.
 Save the markdown trace to the specified file.
 
 ```sh
-npx genaiscript run <script> <spec> --out-trace &lt;file&gt;
+npx genaiscript run <script> <files> --out-trace &lt;file&gt;
 ```
 
 In a GitHub Actions workflow, you can use this feature to save the trace as a step summary (`GITHUB_STEP_SUMMARY`):
 
 ```yaml
 - name: Run GenAIScript tool on spec
   run: |
-      genaiscript run <script> <spec> --out-trace $GITHUB_STEP_SUMMARY
+      genaiscript run <script> <files> --out-trace $GITHUB_STEP_SUMMARY
 ```
 
 ### --out-annotations &lt;file&gt;
 
 Emit annotations in the specified file as a JSON array, JSON Lines, [SARIF](https://sarifweb.azurewebsites.net/) or a CSV file if the file ends with `.csv`.
 
 ```sh
-npx genaiscript run <script> <spec> --out-annotations diags.csv
+npx genaiscript run <script> <files> --out-annotations diags.csv
 ```
 
 Use JSON lines (`.jsonl`) to aggregate annotations from multiple runs in a single file.
 
 ```sh
-npx genaiscript run <script> <spec> --out-annotations diags.jsonl
+npx genaiscript run <script> <files> --out-annotations diags.jsonl
 ```
 
 ### --out-data &lt;file&gt;
@@ -102,15 +110,15 @@ Emits parsed data as JSON, YAML or JSONL. If a JSON schema is specified
 and availabe, the JSON validation result is also stored.
 
 ```sh
-npx genaiscript run <script> <spec> --out-data data.jsonl
+npx genaiscript run <script> <files> --out-data data.jsonl
 ```
 
 ### --out-changelogs &lt;file&gt;
 
 Emit changelogs in the specified file as text.
 
 ```sh
-npx genaiscript run <script> <spec> --out-changelogs changelogs.txt
+npx genaiscript run <script> <files> --out-changelogs changelogs.txt
 ```
 
 ### --prompt

diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -26,7 +26,7 @@
     "@azure/identity": "^4.2.1",
     "dockerode": "^4.0.2",
     "pdfjs-dist": "4.3.136",
-    "promptfoo": "^0.63.2",
+    "promptfoo": "^0.65.0",
     "tree-sitter-wasms": "^0.1.11",
     "typescript": "5.4.5",
     "web-tree-sitter": "^0.22.2",
@@ -43,7 +43,7 @@
     "@types/dockerode": "^3.3.29",
     "@types/fs-extra": "^11.0.4",
     "@types/memorystream": "^0.3.4",
-    "@types/node": "^20.14.2",
+    "@types/node": "^20.14.5",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.2",
     "@types/prompts": "^2.4.9",
@@ -59,8 +59,7 @@
     "genaiscript-core": "*",
     "get-stdin": "^9.0.0",
     "glob": "^10.4.1",
-    "ignore": "^5.3.1",
-    "mammoth": "^1.7.2",
+    "mammoth": "^1.8.0",
     "memorystream": "^0.3.1",
     "node-sarif-builder": "^3.1.0",
     "openai": "^4.51.0",
@@ -69,7 +68,7 @@
     "prompts": "^2.4.2",
     "replace-ext": "^2.0.0",
     "semver": "^7.6.2",
-    "tsx": "^4.15.4",
+    "tsx": "^4.15.6",
     "zx": "^8.1.2"
   },
   "scripts": {

diff --git a/packages/cli/src/batch.ts b/packages/cli/src/batch.ts
@@ -35,6 +35,7 @@ export async function batchScript(
     specs: string[],
     options: {
         excludedFiles: string[]
+        excludeGitIgnore: boolean
         out: string
         outSummary: string
         removeOut: boolean
@@ -69,6 +70,7 @@ export async function batchScript(
         outSummary,
         applyEdits,
         excludedFiles,
+        excludeGitIgnore,
     } = options
     const outAnnotations = join(out, "annotations.jsonl")
     const outData = join(out, "data.jsonl")
@@ -89,7 +91,9 @@ export async function batchScript(
     if (GENAI_JS_REGEX.test(tool)) toolFiles.push(tool)
     const specFiles = new Set<string>()
     for (const arg of specs) {
-        const ffs = await host.findFiles(arg)
+        const ffs = await host.findFiles(arg, {
+            applyGitIgnore: excludeGitIgnore,
+        })
         if (!ffs.length)
             fail(`no files matching ${arg}`, FILES_NOT_FOUND_ERROR_CODE)
 

diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts
@@ -85,6 +85,10 @@ export async function cli() {
         .description("Runs a GenAIScript against files.")
         .arguments("<script> [files...]")
         .option("-ef, --excluded-files <string...>", "excluded files")
+        .option(
+            "-egi, --exclude-git-ignore",
+            "exclude files that are ignore through the .gitignore file in the workspace root"
+        )
         .option(
             "-o, --out <string>",
             "output folder. Extra markdown fields for output and trace will also be generated"
@@ -155,6 +159,10 @@ export async function cli() {
         .description("Run a tool on a batch of specs")
         .arguments("<script> [files...]")
         .option("-ef, --excluded-files <string...>", "excluded files")
+        .option(
+            "-egi, --exclude-git-ignore",
+            "exclude files that are ignore through the .gitignore file in the workspace root"
+        )
         .option(
             "-o, --out <folder>",
             "output folder. Extra markdown fields for output and trace will also be generated"

diff --git a/packages/cli/src/nodehost.ts b/packages/cli/src/nodehost.ts
@@ -21,17 +21,18 @@ import {
     UTF8Encoder,
     createBundledParsers,
     createFileSystem,
+    filterGitIgnore,
     parseDefaultsFromEnv,
     parseTokenFromEnv,
     resolveLanguageModel,
     setHost,
+    unique,
 } from "genaiscript-core"
 import { TextDecoder, TextEncoder } from "util"
 import { readFile, unlink, writeFile } from "node:fs/promises"
 import { ensureDir, existsSync, remove } from "fs-extra"
 import { resolve, dirname } from "node:path"
 import { glob } from "glob"
-import ignorer from "ignore"
 import { debug, error, info, warn } from "./log"
 import { execa } from "execa"
 import { join } from "node:path"
@@ -204,21 +205,19 @@ export class NodeHost implements Host {
     }
     async findFiles(
         path: string | string[],
-        ignore?: string | string[]
+        options: {
+            ignore?: string | string[]
+            applyGitIgnore?: boolean
+        }
     ): Promise<string[]> {
+        const { ignore, applyGitIgnore } = options || {}
         let files = await glob(path, {
             nodir: true,
             windowsPathsNoEscape: true,
             ignore,
         })
-        if (existsSync(".gitignore")) {
-            const gitignore = await readFile(".gitignore", {
-                encoding: "utf-8",
-            })
-            const ig = ignorer().add(gitignore)
-            files = ig.filter(files)
-        }
-        return files
+        if (applyGitIgnore) files = await filterGitIgnore(files)
+        return unique(files)
     }
     async writeFile(name: string, content: Uint8Array): Promise<void> {
         await ensureDir(dirname(name))

diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts
@@ -49,6 +49,7 @@ export async function runScript(
     specs: string[],
     options: {
         excludedFiles: string[]
+        excludeGitIgnore: boolean
         out: string
         retry: string
         retryDelay: string
@@ -80,6 +81,7 @@ export async function runScript(
     }
 ) {
     const excludedFiles = options.excludedFiles
+    const excludeGitIgnore = !!options.excludeGitIgnore
     const out = options.out
     const stream = !options.json && !options.yaml && !out
     const skipLLM = !!options.prompt
@@ -134,7 +136,9 @@ export async function runScript(
         for (const arg of specs) {
             if (HTTPS_REGEX.test(arg)) files.add(arg)
             else {
-                const ffs = await host.findFiles(arg)
+                const ffs = await host.findFiles(arg, {
+                    applyGitIgnore: excludeGitIgnore,
+                })
                 if (!ffs.length)
                     fail(`no files matching ${arg}`, FILES_NOT_FOUND_ERROR_CODE)
 
@@ -234,7 +238,7 @@ ${Array.from(files)
             stats: {
                 toolCalls: 0,
                 repairs: 0,
-                turns: 0
+                turns: 0,
             },
         })
     } catch (err) {