Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrefs
andrefs committed Nov 22, 2023
1 parent 62468eb commit 6850b76
Show file tree
Hide file tree
Showing 8 changed files with 80 additions and 71 deletions.
24 changes: 12 additions & 12 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,21 @@ const sampleFromName = async (ds: DatasetProfile) => {
console.log('\n\n\nStarting')

console.log('GPT-3.5 Turbo 1106');
const gpt35turbo_res = await dsSampleFromDsName.runTrials(ds, gpt35turbo);
const gpt35turbo_res = await dsSampleFromDsName.runTrials(1, ds, gpt35turbo);
//console.log('XXXXXXXXXXXXXXx', JSON.stringify(gpt35turbo_res, null, 2));
let res = await dsSampleFromDsName.validate(ds, gpt35turbo_res);
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
console.log('');

console.log('GPT-4 0613');
const gpt4_res = await dsSampleFromDsName.runTrials(ds, gpt4);
const gpt4_res = await dsSampleFromDsName.runTrials(1, ds, gpt4);
//console.log('XXXXXXXXXXXXXXx', JSON.stringify(gpt4_res, null, 2));
res = await dsSampleFromDsName.validate(ds, gpt4_res);
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
console.log('');

console.log('GPT-4 1106 Preview');
const gpt4turbo_res = await dsSampleFromDsName.runTrials(ds, gpt4turbo);
const gpt4turbo_res = await dsSampleFromDsName.runTrials(1, ds, gpt4turbo);
//console.log('XXXXXXXXXXXXXXx', JSON.stringify(gpt4turbo_res, null, 2));
res = await dsSampleFromDsName.validate(ds, gpt4turbo_res);
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
Expand All @@ -43,19 +43,19 @@ const paperFromName = async (ds: DatasetProfile) => {

console.log('\n\n\nStarting')
console.log('GPT-3.5 Turbo 1106');
const gpt35turbo_res = await dsPaperFromDsName.runTrials(ds, gpt35turbo);
const gpt35turbo_res = await dsPaperFromDsName.runTrials(1, ds, gpt35turbo);
let res = await dsPaperFromDsName.validate(ds, gpt35turbo_res);
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
console.log('');

console.log('GPT-4 0613');
const gpt4_res = await dsPaperFromDsName.runTrials(ds, gpt4);
const gpt4_res = await dsPaperFromDsName.runTrials(1, ds, gpt4);
res = await dsPaperFromDsName.validate(ds, gpt4_res);
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
console.log('');

console.log('GPT-4 1106 Preview');
const gpt4turbo_res = await dsPaperFromDsName.runTrials(ds, gpt4turbo);
const gpt4turbo_res = await dsPaperFromDsName.runTrials(1, ds, gpt4turbo);
res = await dsPaperFromDsName.validate(ds, gpt4turbo_res);
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
console.log('');
Expand All @@ -65,21 +65,21 @@ const sampleFromSample = async (ds: DatasetProfile) => {

console.log('\n\n\nStarting')
console.log('GPT-3.5 Turbo 1106');
const gpt35turbo_res = await dsSampleFromDsSample.runTrials(ds, gpt35turbo);
const gpt35turbo_res = await dsSampleFromDsSample.runTrials(1, ds, gpt35turbo);
let res = await dsSampleFromDsSample.validate(ds, gpt35turbo_res);
console.log('XXXXXXXXXXX', JSON.stringify(gpt35turbo_res, null, 2))
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
console.log('');

console.log('GPT-4 0613');
const gpt4_res = await dsSampleFromDsSample.runTrials(ds, gpt4);
const gpt4_res = await dsSampleFromDsSample.runTrials(1, ds, gpt4);
res = await dsSampleFromDsSample.validate(ds, gpt4_res);
console.log('XXXXXXXXXXX', JSON.stringify(gpt4_res, null, 2))
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
console.log('');

console.log('GPT-4 1106 Preview');
const gpt4turbo_res = await dsSampleFromDsSample.runTrials(ds, gpt4turbo);
const gpt4turbo_res = await dsSampleFromDsSample.runTrials(1, ds, gpt4turbo);
res = await dsSampleFromDsSample.validate(ds, gpt4turbo_res);
console.log('XXXXXXXXXXX', JSON.stringify(gpt4turbo_res, null, 2))
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
Expand All @@ -90,21 +90,21 @@ const nameFromSample = async (ds: DatasetProfile) => {

console.log('\n\n\nStarting')
console.log('GPT-3.5 Turbo 1106');
const gpt35turbo_res = await dsNameFromDsSample.runTrials(ds, gpt35turbo);
const gpt35turbo_res = await dsNameFromDsSample.runTrials(1, ds, gpt35turbo);
let res = await dsNameFromDsSample.validate(ds, gpt35turbo_res);
console.log('XXXXXXXXXXX', JSON.stringify(gpt35turbo_res, null, 2))
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
console.log('');

console.log('GPT-4 0613');
const gpt4_res = await dsNameFromDsSample.runTrials(ds, gpt4);
const gpt4_res = await dsNameFromDsSample.runTrials(1, ds, gpt4);
res = await dsNameFromDsSample.validate(ds, gpt4_res);
console.log('XXXXXXXXXXX', JSON.stringify(gpt4_res, null, 2))
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
console.log('');

console.log('GPT-4 1106 Preview');
const gpt4turbo_res = await dsNameFromDsSample.runTrials(ds, gpt4turbo);
const gpt4turbo_res = await dsNameFromDsSample.runTrials(1, ds, gpt4turbo);
res = await dsNameFromDsSample.validate(ds, gpt4turbo_res);
console.log('XXXXXXXXXXX', JSON.stringify(gpt4turbo_res, null, 2))
console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
Expand Down
41 changes: 26 additions & 15 deletions src/lib/experiments/experiment.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import OpenAI from "openai";
import { Model } from "../models";
import { DatasetProfile } from "../types";
import { ValidationResult, ValidationType } from "../validation";
import { ValidationResult, ValidationType, combineValidations } from "../validation";
import logger from "../logger";


Expand All @@ -11,16 +11,17 @@ class Experiment {
genPrompt: (ds: DatasetProfile) => string;
schema: any; // TODO
runTrials: (this: Experiment, trials: number, ds: DatasetProfile, model: Model) => Promise<string[]>;
validate: (ds: DatasetProfile, data: string[]) => Promise<ValidationResult>;
perform: (this: Experiment, trials: number, ds: DatasetProfile, model: Model) => Promise<ValidationResult>;
validateTrial: (ds: DatasetProfile, data: string) => Promise<ValidationResult>;
validate: (ds: DatasetProfile, vals: ValidationResult[]) => Promise<AggregatedValidationResult>;
perform: (this: Experiment, trials: number, ds: DatasetProfile, model: Model) => Promise<ExperimentData>;

constructor(
name: string,
description: string,
genPrompt: (ds: DatasetProfile) => string,
schema: any,
runTrial: (prompt: string, schema: any, ds: DatasetProfile, model: Model) => Promise<TrialResult>,
validate: (ds: DatasetProfile, data: string[]) => Promise<ValidationResult>,
validateTrial: (ds: DatasetProfile, data: string) => Promise<ValidationResult>,
) {
this.name = name;
this.description = description;
Expand All @@ -38,29 +39,39 @@ class Experiment {
}
return results;
}
this.validate = validate;
this.perform = async function(this: Experiment, trials: number, ds: DatasetProfile, model: Model) {
this.validateTrial = validateTrial;
this.validate = async function(this: Experiment, ds: DatasetProfile, validations: ValidationResult[]) {
return combineValidations(validations);
};
this.perform = async function(this: Experiment, trials: number, ds: DatasetProfile, model: Model): Promise<ExperimentData> {
const results = await this.runTrials(trials, ds, model);
return this.validate(ds, results);
const validations = await Promise.all(results.map((res) => this.validateTrial(ds, res)));
const avr = await this.validate(ds, validations);

return {
timestamp: Date.now(),
prompt: this.genPrompt(ds),
schema: this.schema,
dsId: ds.id,
modelId: model.modelId,
trials: validations,
result: avr,
}
}
}
}

export interface ExperimentData {
timestamp: number;
prompt: string;
prompt: string
schema: any;
dsId: string;
modelId: string;
trials: [
rawData: string,
data: any,
result: TrialResult
],
result: ValidationResult,
trials: ValidationResult[],
result: AggregatedValidationResult,
}

export interface ValidationResult {
export interface AggregatedValidationResult {
avg: number;
resultTypes: {
[key in ValidationType]: number;
Expand Down
26 changes: 11 additions & 15 deletions src/lib/experiments/prior-knowledge/dsNameFromDsSample.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import Experiment, { ExperimentResult, TrialResult } from "../experiment";
import Experiment from "../experiment";
import { Model } from "../../models";
import { DatasetProfile } from "../../types";
import { DataCorrect, DataIncomplete, DataIncorrect, DataPartiallyIncorrect, JsonSyntaxError, NoData, ValidationResult, ValidationType, combineValidations } from "../../validation";
Expand Down Expand Up @@ -40,20 +40,16 @@ async function run(prompt: string, schema: any, _: DatasetProfile, model: Model)
}


async function validate(ds: DatasetProfile, data: string[]) {
const validations = data.map(d => {
if (!d.trim()) { return new NoData(); }
try {
const got = JSON.parse(d);
async function validateTrial(ds: DatasetProfile, data: string) {
if (!data.trim()) { return new NoData(); }
try {
const got = JSON.parse(data);

console.log('XXXXXXXXXXXXXXX', JSON.stringify(got, null, 2));
return new DataCorrect(got);
} catch (e) {
return new JsonSyntaxError(d);
}
});

return combineValidations(validations);
console.log('XXXXXXXXXXXXXXX', JSON.stringify(got, null, 2));
return new DataCorrect(got);
} catch (e) {
return new JsonSyntaxError(data);
}
}


Expand All @@ -65,7 +61,7 @@ export default new Experiment(
genPrompt,
resultSchema,
run,
validate
validateTrial
);


Expand Down
4 changes: 2 additions & 2 deletions src/lib/experiments/prior-knowledge/dsPaperFromDsName.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ async function run(prompt: string, schema: any, _: DatasetProfile, model: Model)
}


async function validate(ds: DatasetProfile, data: string) {
async function validateTrial(ds: DatasetProfile, data: string) {
if (!data.trim()) { return new NoData(); }
try {
const got = JSON.parse(data);
Expand Down Expand Up @@ -67,7 +67,7 @@ export default new Experiment(
genPrompt,
resultSchema,
run,
validate
validateTrial
);


Expand Down
40 changes: 20 additions & 20 deletions src/lib/experiments/prior-knowledge/dsSampleFromDsName.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,47 +103,47 @@ describe('dsSampleFromDsName', () => {

});

describe('validate', () => {
describe('validateTrial', () => {
test('should return NoData if data is empty', async () => {
const ds: DatasetProfile = createMockDataset();

const result = await dsSampleFromDsName.validate(ds, ['']);
const result = await dsSampleFromDsName.validateTrial(ds, '');
expect(result.type).toEqual('no-data');
});

test('should return DataIncorrect if data is incorrect', async () => {
const ds: DatasetProfile = createMockDataset();

const result = await dsSampleFromDsName.validate(ds,
[
JSON.stringify({
pairs: [
['test', 'test2'],
]
})
]);
const result = await dsSampleFromDsName.validateTrial(ds,

JSON.stringify({
pairs: [
['test', 'test2'],
]
})
);
expect(result.type).toEqual('data-incorrect');
});

test('should return DataPartiallyIncorrect if data is partially incorrect', async () => {
const ds: DatasetProfile = createMockDataset();

const result = await dsSampleFromDsName.validate(ds,
[
JSON.stringify({
pairs: [
['test', 'test2'],
['test', 'test'],
]
})
]);
const result = await dsSampleFromDsName.validateTrial(ds,

JSON.stringify({
pairs: [
['test', 'test2'],
['test', 'test'],
]
})
);
expect(result.type).toEqual('data-partially-incorrect');
});

test('should return JsonSyntaxError if data is not valid JSON', async () => {
const ds: DatasetProfile = createMockDataset();

const result = await dsSampleFromDsName.validate(ds, ['not valid json']);
const result = await dsSampleFromDsName.validateTrial(ds, 'not valid json');
expect(result.type).toEqual('json-syntax-error');
});
});
Expand Down
4 changes: 2 additions & 2 deletions src/lib/experiments/prior-knowledge/dsSampleFromDsName.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ async function run(prompt: string, schema: any, _: DatasetProfile, model: Model)
}


async function validate(ds: DatasetProfile, data: string) {
async function validateTrial(ds: DatasetProfile, data: string) {
if (!data.trim()) { return new NoData(); }
try {
const got = JSON.parse(data);
Expand Down Expand Up @@ -93,7 +93,7 @@ export default new Experiment(
genPrompt,
resultSchema,
run,
validate
validateTrial
);


Expand Down
4 changes: 2 additions & 2 deletions src/lib/experiments/prior-knowledge/dsSampleFromDsSample.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ async function run(prompt: string, schema: any, _: DatasetProfile, model: Model)
}


async function validate(ds: DatasetProfile, data: string) {
async function validateTrial(ds: DatasetProfile, data: string) {
if (!data.trim()) { return new NoData(); }
try {
const got = JSON.parse(data);
Expand Down Expand Up @@ -93,7 +93,7 @@ export default new Experiment(
genPrompt,
resultSchema,
run,
validate
validateTrial
);


Expand Down
8 changes: 5 additions & 3 deletions src/lib/validation/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
import { ExperimentResult } from "../experiments";
import { AggregatedValidationResult } from "../experiments";

export type ValidationType = 'json-syntax-error' | 'json-schema-error' | 'data-incomplete' | 'data-partially-incorrect' | 'data-incorrect' | 'no-data' | 'data-correct';

export class ValidationResult {
type: ValidationType;
ok: boolean;
rawData: string;
data?: any;

constructor(type: ValidationType, ok: boolean, data?: any) {
constructor(type: ValidationType, ok: boolean, rawdata: string, data?: any) {
this.type = type;
this.ok = ok;
this.rawData = rawdata;
if (data) {
this.data = data;
}
Expand Down Expand Up @@ -67,7 +69,7 @@ export class DataCorrect extends ValidationResult {
}


export async function combineValidations(vs: ValidationResult[]): Promise<ExperimentResult> {
export async function combineValidations(vs: ValidationResult[]): Promise<AggregatedValidationResult> {
let sum = 0;
const resultTypes = {} as { [key in ValidationType]: number };

Expand Down

0 comments on commit 6850b76

Please sign in to comment.