Add ability to process csvs as test input and display results

src/commands/file.ts

@@ -1,6 +1,6 @@
 import { Args, Command, Flags } from '@oclif/core'
 import { parse as yamlParse } from 'yaml'
-import { readFileSync, readdir } from 'node:fs'
+import { readFileSync, createReadStream, readdir } from 'node:fs'
 import { LLMBase } from '../providers/base/llm-base'
 import { LLM } from '../providers/openai/llm'
 import { parse as csvParse } from 'csv-parse'
@@ -32,41 +32,70 @@ export default class File extends Command {
     throw new Error('Unrecognized command')
   }
 
+  processFile = async (filePath: string, prompt: string, client: LLMBase): Promise<number[]> => {
+    const readStream = createReadStream(filePath)
+    // eslint-disable-next-line camelcase
+    .pipe(csvParse({ delimiter: ',', from_line: 2 }))
+
+    let count = 0
+    let passed = 0
+    for await (const row of readStream) {
+      try {
+        const processedRow = await client.textCompletion('gpt-3.5-turbo', 0, prompt, row[0].toString())
+
+        count += 1
+        // Try regex for the expected values instead?
+        console.log({ processedRow: processedRow.trim().toLowerCase(), expected: row[1].trim().toLowerCase() })
+        if (processedRow.trim().toLowerCase() === row[1].trim().toLowerCase()) passed += 1
+      } catch (error) {
+        console.error('An error occurred while processing the row:', error)
+      }
+    }
+
+    return [count, passed]
+  }
+
+  testFile = async (filePath: string, client: LLMBase, prompt: string): Promise<boolean> => {
+    const [count, passed] = await this.processFile(filePath, prompt, client)
+
+    console.log(`Results for ${filePath}: ${passed}/${count}`)
+    return count === passed
+  }
+
   public async run(): Promise<void> {
-    const { args, flags } = await this.parse(File)
+    const { args } = await this.parse(File)
 
     const file = readFileSync(args.filePath, 'utf8')
     const fileContent = yamlParse(file)
 
     const actions = fileContent.actions
     for (const action of Object.keys(fileContent.actions)) {
       const name = action
-      console.log(actions[name])
-      const { command, providers, prompt, test_directory } = actions[name]
+      const { command, providers, prompt, test_directory: testDirectory } = actions[name]
 
-      console.log({ providers })
       for (const provider of providers) {
         const client = this.commandToClient(command, provider)
 
         // Iterate through files in test_directory relative to Waffiefile
-        const testDirectoryPath = path.join(process.cwd(), args.filePath, '..', test_directory)
+        const testDirectoryPath = path.join(process.cwd(), args.filePath, '..', testDirectory)
 
-        console.log({ testDirectoryPath })
-        readdir(testDirectoryPath, (err, files) => {
+        console.log(`Testing ${provider}:`)
+
+        readdir(testDirectoryPath, async (err, files) => {
           if (err) {
             console.error('Error reading directory:', err)
             return
           }
 
-          // Iterate through the files
+          const resultsPromises = []
           for (const file of files) {
-            // Do something with each file in the test_directory
-            console.log(file)
+            resultsPromises.push(this.testFile(path.join(testDirectoryPath, file), client, prompt))
           }
+
+          const results = await Promise.all(resultsPromises)
+          return results.every(result => result)
         })
       }
-
-      console.log(`Testing ${name}:`)
     }
   }
 }

src/commands/text-completion.ts

@@ -41,7 +41,7 @@ export default class TextCompletion extends Command {
 
     const client = this.providerToClient(flags.provider)
 
-    const result = await client.textCompletion('gpt-3.5-turbo', 0, 'can you write me a 10 line poem to hype up a internal medicine intern')
+    const result = await client.textCompletion('gpt-3.5-turbo', 0, '', 'can you write me a 10 line poem to hype up a internal medicine intern')
 
     console.log(result)
   }

src/providers/base/llm-base.ts

@@ -1,5 +1,5 @@
 export class LLMBase {
-  textCompletion(model: string, temperature: number, message: string): Promise<string> {
-    return Promise.resolve(`model: ${model}, temperature: ${temperature}, message: ${message}`)
+  textCompletion(model: string, temperature: number, prompt: string, message: string): Promise<string> {
+    return Promise.resolve(`model: ${model}, temperature: ${temperature}, prompt: ${prompt}, message: ${message}`)
   }
 }

src/providers/openai/llm.ts

@@ -13,12 +13,18 @@ export class LLM extends LLMBase {
     this.client = new OpenAIApi(configuration)
   }
 
-  async textCompletion(model: string, temperature: number, message: string): Promise<string> {
+  async textCompletion(model: string, temperature: number, prompt: string, message: string): Promise<string> {
     return new Promise((resolve, _reject) => {
       this.client.createChatCompletion({
         model,
         temperature,
-        messages: [{ role: 'user', content: message }],
+        messages: [
+          {
+            role: 'system',
+            content: prompt,
+          },
+          { role: 'user', content: message },
+        ],
       }).then(chatCompletion => {
         console.log(chatCompletion.data.choices[0].message)
         resolve(chatCompletion.data.choices[0].message?.content || 'error')