forked from mongodb/chatbot
-
Notifications
You must be signed in to change notification settings - Fork 0
/
EvalConfig.ts
43 lines (40 loc) · 1.27 KB
/
EvalConfig.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
import { CommandMetadataStore } from "./CommandMetadataStore";
import { EvaluateQualityFunc } from "./evaluate/EvaluateQualityFunc";
import { EvaluationStore } from "./evaluate/EvaluationStore";
import { GenerateDataFunc } from "./generate/GenerateDataFunc";
import { GeneratedDataStore } from "./generate/GeneratedDataStore";
import { SomeTestCase } from "./generate/TestCase";
import { ReportEvalFunc } from "./report/ReportEvalFunc";
import { ReportStore } from "./report/ReportStore";
export interface EvalConfig {
metadataStore: CommandMetadataStore;
generatedDataStore: GeneratedDataStore;
evaluationStore: EvaluationStore;
reportStore: ReportStore;
commands: {
generate?: {
[k: string]: {
type: string;
testCases: SomeTestCase[];
generator: GenerateDataFunc;
};
};
evaluate?: {
[k: string]: {
evaluator: EvaluateQualityFunc;
};
};
report?: {
[k: string]: {
reporter: ReportEvalFunc;
};
};
};
/**
Function that runs after all actions in the eval command.
Can use for things like cleaning up resources besides the stores.
The stores are cleaned up automatically.
*/
afterAll?: () => Promise<void>;
}
export type ConfigConstructor = () => Promise<EvalConfig>;