wip

src/lib/experiments/experiment.ts

@@ -12,7 +12,7 @@ class Experiment {
   schema: any; // TODO
   runTrials: (this: Experiment, trials: number, ds: DatasetProfile, model: Model) => Promise<string[]>;
   validateTrial: (ds: DatasetProfile, data: string) => Promise<ValidationResult>;
-  validate: (ds: DatasetProfile, vals: ValidationResult[]) => Promise<AggregatedValidationResult>;
+  validate: (ds: DatasetProfile, data: string[]) => Promise<{ trialResults: ValidationResult[], combinedResult: AggregatedValidationResult }>;
   perform: (this: Experiment, trials: number, ds: DatasetProfile, model: Model) => Promise<ExperimentData>;
 
   constructor(
@@ -40,22 +40,22 @@ class Experiment {
       return results;
     }
     this.validateTrial = validateTrial;
-    this.validate = async function(this: Experiment, ds: DatasetProfile, validations: ValidationResult[]) {
-      return combineValidations(validations);
+    this.validate = async function(this: Experiment, ds: DatasetProfile, data: string[]) {
+      const trialResults = await Promise.all(data.map((d) => this.validateTrial(ds, d)));
+      return { trialResults, combinedResult: await combineValidations(trialResults) };
     };
     this.perform = async function(this: Experiment, trials: number, ds: DatasetProfile, model: Model): Promise<ExperimentData> {
       const results = await this.runTrials(trials, ds, model);
-      const validations = await Promise.all(results.map((res) => this.validateTrial(ds, res)));
-      const avr = await this.validate(ds, validations);
+      const { trialResults, combinedResult } = await this.validate(ds, results);
 
       return {
         timestamp: Date.now(),
         prompt: this.genPrompt(ds),
         schema: this.schema,
         dsId: ds.id,
         modelId: model.modelId,
-        trials: validations,
-        result: avr,
+        trialResults,
+        combinedResult
       }
     }
   }
@@ -67,8 +67,8 @@ export interface ExperimentData {
   schema: any;
   dsId: string;
   modelId: string;
-  trials: ValidationResult[],
-  result: AggregatedValidationResult,
+  trialResults: ValidationResult[],
+  combinedResult: AggregatedValidationResult,
 }
 
 export interface AggregatedValidationResult {

src/lib/validation/index.ts

@@ -57,7 +57,7 @@ export class DataIncorrect extends ValidationResult {
 
 export class NoData extends ValidationResult {
   constructor() {
-    super('no-data', false);
+    super('no-data', false, '');
   }
 }
 

src/scripts/dsSampleFromDsName.ts

@@ -0,0 +1,35 @@
+import { DatasetProfile } from "grillo-datasets/src/lib/types";
+import { dsSampleFromDsName } from "../lib/experiments";
+import { gpt35turbo, gpt4, gpt4turbo } from "../lib/models";
+import { rg65 } from "grillo-datasets";
+
+const sampleFromName = async (ds: DatasetProfile) => {
+
+  console.log('\n\n\nStarting')
+
+  console.log('GPT-3.5 Turbo 1106');
+  const gpt35turbo_res = await dsSampleFromDsName.runTrials(1, ds, gpt35turbo);
+  let res = await dsSampleFromDsName.validate(ds, gpt35turbo_res);
+  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
+  console.log('');
+
+  console.log('GPT-4 0613');
+  const gpt4_res = await dsSampleFromDsName.runTrials(1, ds, gpt4);
+  //console.log('XXXXXXXXXXXXXXx', JSON.stringify(gpt4_res, null, 2));
+  res = await dsSampleFromDsName.validate(ds, gpt4_res);
+  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
+  console.log('');
+
+  console.log('GPT-4 1106 Preview');
+  const gpt4turbo_res = await dsSampleFromDsName.runTrials(1, ds, gpt4turbo);
+  //console.log('XXXXXXXXXXXXXXx', JSON.stringify(gpt4turbo_res, null, 2));
+  res = await dsSampleFromDsName.validate(ds, gpt4turbo_res);
+  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
+  console.log('');
+}
+
+
+sampleFromName(rg65).then(() => {
+  console.log('Done');
+  process.exit(0);
+}