-
Notifications
You must be signed in to change notification settings - Fork 2.1k
/
types.ts
49 lines (41 loc) · 1.29 KB
/
types.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
import {
AgentTrajectoryEvaluator,
LLMPairwiseStringEvaluator,
LLMStringEvaluator,
PairwiseStringEvaluator,
StringEvaluator,
} from "./base.js";
export interface EvaluatorType {
/**
* The criteria evaluator, which evaluates a model based on a
* custom set of criteria without any reference labels.
* */
criteria: LLMStringEvaluator;
/**
* The labeled criteria evaluator, which evaluates a model based on a
* custom set of criteria, with a reference label.
* */
labeled_criteria: LLMStringEvaluator;
/**
* The pairwise string evaluator, which predicts the preferred prediction from
* between two models.
*/
pairwise_string: LLMPairwiseStringEvaluator;
/**
* The labeled pairwise string evaluator, which predicts the preferred prediction
* from between two models based on a ground truth reference label.
* */
labeled_pairwise_string: LLMPairwiseStringEvaluator;
/**
* The agent trajectory evaluator, which grades the agent's intermediate steps.
*/
trajectory: AgentTrajectoryEvaluator;
/**
* Compare a prediction to a reference label using embedding distance.
* */
embedding_distance: StringEvaluator;
/**
* Compare two predictions using embedding distance.
* */
pairwise_embedding_distance: PairwiseStringEvaluator;
}