scripts for running prior-knowledge functions

andrefs · Nov 22, 2023 · 6247d71 · 6247d71
1 parent 340a41a
commit 6247d71
Show file tree

Hide file tree

Showing 7 changed files with 81 additions and 109 deletions.
diff --git a/src/index.ts b/src/index.ts
@@ -1,116 +1,13 @@
 import 'dotenv/config';
 
 import {
-  dsSampleFromDsName,
-  dsPaperFromDsName,
-  dsSampleFromDsSample,
-  dsNameFromDsSample,
   compareMc30
 } from "./lib/experiments";
 import rg65 from 'grillo-datasets/rg65';
 import { gpt4, gpt4turbo, gpt35turbo } from './lib/models';
 import { DatasetProfile } from './lib/types';
 import logger from './lib/logger';
 
-
-const sampleFromName = async (ds: DatasetProfile) => {
-
-  console.log('\n\n\nStarting')
-
-  console.log('GPT-3.5 Turbo 1106');
-  const gpt35turbo_res = await dsSampleFromDsName.runTrials(1, ds, gpt35turbo);
-  //console.log('XXXXXXXXXXXXXXx', JSON.stringify(gpt35turbo_res, null, 2));
-  let res = await dsSampleFromDsName.validate(ds, gpt35turbo_res);
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-
-  console.log('GPT-4 0613');
-  const gpt4_res = await dsSampleFromDsName.runTrials(1, ds, gpt4);
-  //console.log('XXXXXXXXXXXXXXx', JSON.stringify(gpt4_res, null, 2));
-  res = await dsSampleFromDsName.validate(ds, gpt4_res);
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-
-  console.log('GPT-4 1106 Preview');
-  const gpt4turbo_res = await dsSampleFromDsName.runTrials(1, ds, gpt4turbo);
-  //console.log('XXXXXXXXXXXXXXx', JSON.stringify(gpt4turbo_res, null, 2));
-  res = await dsSampleFromDsName.validate(ds, gpt4turbo_res);
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-}
-
-const paperFromName = async (ds: DatasetProfile) => {
-
-  console.log('\n\n\nStarting')
-  console.log('GPT-3.5 Turbo 1106');
-  const gpt35turbo_res = await dsPaperFromDsName.runTrials(1, ds, gpt35turbo);
-  let res = await dsPaperFromDsName.validate(ds, gpt35turbo_res);
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-
-  console.log('GPT-4 0613');
-  const gpt4_res = await dsPaperFromDsName.runTrials(1, ds, gpt4);
-  res = await dsPaperFromDsName.validate(ds, gpt4_res);
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-
-  console.log('GPT-4 1106 Preview');
-  const gpt4turbo_res = await dsPaperFromDsName.runTrials(1, ds, gpt4turbo);
-  res = await dsPaperFromDsName.validate(ds, gpt4turbo_res);
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-}
-
-const sampleFromSample = async (ds: DatasetProfile) => {
-
-  console.log('\n\n\nStarting')
-  console.log('GPT-3.5 Turbo 1106');
-  const gpt35turbo_res = await dsSampleFromDsSample.runTrials(1, ds, gpt35turbo);
-  let res = await dsSampleFromDsSample.validate(ds, gpt35turbo_res);
-  console.log('XXXXXXXXXXX', JSON.stringify(gpt35turbo_res, null, 2))
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-
-  console.log('GPT-4 0613');
-  const gpt4_res = await dsSampleFromDsSample.runTrials(1, ds, gpt4);
-  res = await dsSampleFromDsSample.validate(ds, gpt4_res);
-  console.log('XXXXXXXXXXX', JSON.stringify(gpt4_res, null, 2))
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-
-  console.log('GPT-4 1106 Preview');
-  const gpt4turbo_res = await dsSampleFromDsSample.runTrials(1, ds, gpt4turbo);
-  res = await dsSampleFromDsSample.validate(ds, gpt4turbo_res);
-  console.log('XXXXXXXXXXX', JSON.stringify(gpt4turbo_res, null, 2))
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-}
-
-const nameFromSample = async (ds: DatasetProfile) => {
-
-  console.log('\n\n\nStarting')
-  console.log('GPT-3.5 Turbo 1106');
-  const gpt35turbo_res = await dsNameFromDsSample.runTrials(1, ds, gpt35turbo);
-  let res = await dsNameFromDsSample.validate(ds, gpt35turbo_res);
-  console.log('XXXXXXXXXXX', JSON.stringify(gpt35turbo_res, null, 2))
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-
-  console.log('GPT-4 0613');
-  const gpt4_res = await dsNameFromDsSample.runTrials(1, ds, gpt4);
-  res = await dsNameFromDsSample.validate(ds, gpt4_res);
-  console.log('XXXXXXXXXXX', JSON.stringify(gpt4_res, null, 2))
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-
-  console.log('GPT-4 1106 Preview');
-  const gpt4turbo_res = await dsNameFromDsSample.runTrials(1, ds, gpt4turbo);
-  res = await dsNameFromDsSample.validate(ds, gpt4turbo_res);
-  console.log('XXXXXXXXXXX', JSON.stringify(gpt4turbo_res, null, 2))
-  console.log(res.ok ? res.type : JSON.stringify(res, null, 2));
-  console.log('');
-}
-
 //const compareMC30 = async () => {
 //  console.log('\n\n\nStarting')
 //  console.log('GPT-3.5 Turbo 1106');

diff --git a/src/lib/experiments/experiment.ts b/src/lib/experiments/experiment.ts
@@ -29,7 +29,7 @@ class Experiment {
     this.schema = schema;
     this.runTrials = async function(this: Experiment, trials: number, ds: DatasetProfile, model: Model) {
       const prompt = this.genPrompt(ds);
-      logger.info(`Running experiment ${this.name} ${trials} times on model (${model.modelId}).`);
+      logger.info(`Running experiment ${this.name} ${trials} times on model ${model.modelId}.`);
       logger.debug(`Prompt: ${prompt}`);
 
       const results = [];

diff --git a/src/scripts/compareMC30.ts b/src/scripts/compareMC30.ts
diff --git a/src/scripts/dsNameFromDsSample.ts b/src/scripts/dsNameFromDsSample.ts
@@ -0,0 +1,26 @@
+import { DatasetProfile } from "grillo-datasets/src/lib/types";
+import logger from "../lib/logger";
+import { dsNameFromDsSample } from "../lib/experiments";
+import { gpt35turbo, gpt4, gpt4turbo } from "../lib/models";
+import { rg65 } from "grillo-datasets";
+
+
+const nameFromSample = async (ds: DatasetProfile) => {
+  logger.info('Starting')
+  console.log('GPT-3.5 Turbo 1106');
+  const gpt35turbo_res = await dsNameFromDsSample.perform(1, ds, gpt35turbo);
+  const gpt4_res = await dsNameFromDsSample.perform(1, ds, gpt4);
+  const gpt4turbo_res = await dsNameFromDsSample.perform(1, ds, gpt4turbo);
+
+  logger.info({ ...gpt35turbo_res.combinedResult.resultTypes }, `gpt35turbo_res ${gpt35turbo_res.combinedResult.avg}`);
+  logger.info({ ...gpt4_res.combinedResult.resultTypes }, `gpt4_res ${gpt4_res.combinedResult.avg}`);
+  logger.info({ ...gpt4turbo_res.combinedResult.resultTypes }, `gpt4turbo_res ${gpt4turbo_res.combinedResult.avg}`);
+}
+
+nameFromSample(rg65).then(() => {
+  logger.info('Done');
+  process.exit(0);
+});
+
+
+
diff --git a/src/scripts/dsPaperFromDsName.ts b/src/scripts/dsPaperFromDsName.ts
@@ -0,0 +1,26 @@
+import { DatasetProfile } from "grillo-datasets/src/lib/types";
+import { dsPaperFromDsName } from "../lib/experiments";
+import { gpt35turbo, gpt4, gpt4turbo } from "../lib/models";
+import logger from "../lib/logger";
+import { rg65 } from "grillo-datasets";
+
+const paperFromName = async (ds: DatasetProfile) => {
+
+  logger.info('Starting')
+  const gpt35turbo_res = await dsPaperFromDsName.perform(1, ds, gpt35turbo);
+  const gpt4_res = await dsPaperFromDsName.perform(1, ds, gpt4);
+  const gpt4turbo_res = await dsPaperFromDsName.perform(1, ds, gpt4turbo);
+
+  logger.info({ ...gpt35turbo_res.combinedResult.resultTypes }, `gpt35turbo_res ${gpt35turbo_res.combinedResult.avg}`);
+  logger.info({ ...gpt4_res.combinedResult.resultTypes }, `gpt4_res ${gpt4_res.combinedResult.avg}`);
+  logger.info({ ...gpt4turbo_res.combinedResult.resultTypes }, `gpt4turbo_res ${gpt4turbo_res.combinedResult.avg}`);
+}
+
+paperFromName(rg65).then(() => {
+  logger.info('Done');
+  process.exit(0);
+});
+
+
+
+
diff --git a/src/scripts/dsSampleFromDsName.ts b/src/scripts/dsSampleFromDsName.ts
@@ -2,22 +2,23 @@ import { DatasetProfile } from "grillo-datasets/src/lib/types";
 import { dsSampleFromDsName } from "../lib/experiments";
 import { gpt35turbo, gpt4, gpt4turbo } from "../lib/models";
 import { rg65 } from "grillo-datasets";
+import logger from "../lib/logger";
 
 const sampleFromName = async (ds: DatasetProfile) => {
 
-  console.log('\n\n\nStarting')
+  logger.info('Starting')
 
   const gpt35turbo_res = await dsSampleFromDsName.perform(10, ds, gpt35turbo);
   const gpt4_res = await dsSampleFromDsName.perform(10, ds, gpt4);
   const gpt4turbo_res = await dsSampleFromDsName.perform(10, ds, gpt4turbo);
 
-  console.log('gpt35turbo_res', gpt35turbo_res.combinedResult.avg);
-  console.log('gpt4_res', gpt4_res.combinedResult.avg);
-  console.log('gpt4turbo_res', gpt4turbo_res.combinedResult.avg);
+  logger.info({ ...gpt35turbo_res.combinedResult.resultTypes }, `gpt35turbo_res ${gpt35turbo_res.combinedResult.avg}`);
+  logger.info({ ...gpt4_res.combinedResult.resultTypes }, `gpt4_res ${gpt4_res.combinedResult.avg}`);
+  logger.info({ ...gpt4turbo_res.combinedResult.resultTypes }, `gpt4turbo_res ${gpt4turbo_res.combinedResult.avg}`);
 }
 
 
 sampleFromName(rg65).then(() => {
-  console.log('Done');
+  logger.info('Done');
   process.exit(0);
 });
diff --git a/src/scripts/dsSampleFromDsSample.ts b/src/scripts/dsSampleFromDsSample.ts
@@ -0,0 +1,22 @@
+import { DatasetProfile } from "grillo-datasets/src/lib/types";
+import logger from "../lib/logger";
+import { gpt35turbo, gpt4, gpt4turbo } from "../lib/models";
+import { dsSampleFromDsSample } from "../lib/experiments";
+import { rg65 } from "grillo-datasets";
+
+const sampleFromSample = async (ds: DatasetProfile) => {
+  logger.info('Starting')
+  const gpt35turbo_res = await dsSampleFromDsSample.perform(1, ds, gpt35turbo);
+  const gpt4_res = await dsSampleFromDsSample.perform(1, ds, gpt4);
+  const gpt4turbo_res = await dsSampleFromDsSample.perform(1, ds, gpt4turbo);
+
+  logger.info({ ...gpt35turbo_res.combinedResult.resultTypes }, `gpt35turbo_res ${gpt35turbo_res.combinedResult.avg}`);
+  logger.info({ ...gpt4_res.combinedResult.resultTypes }, `gpt4_res ${gpt4_res.combinedResult.avg}`);
+  logger.info({ ...gpt4turbo_res.combinedResult.resultTypes }, `gpt4turbo_res ${gpt4turbo_res.combinedResult.avg}`);
+}
+
+sampleFromSample(rg65).then(() => {
+  logger.info('Done');
+  process.exit(0);
+});
+