-
Notifications
You must be signed in to change notification settings - Fork 305
/
Relevancy.ts
139 lines (117 loc) · 3.52 KB
/
Relevancy.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
import { Document, MetadataMode } from "../Node.js";
import type { ServiceContext } from "../ServiceContext.js";
import { SummaryIndex } from "../indices/summary/index.js";
import { PromptMixin } from "../prompts/Mixin.js";
import type { RelevancyEvalPrompt, RelevancyRefinePrompt } from "./prompts.js";
import {
defaultRelevancyEvalPrompt,
defaultRelevancyRefinePrompt,
} from "./prompts.js";
import type {
BaseEvaluator,
EvaluationResult,
EvaluatorParams,
EvaluatorResponseParams,
} from "./types.js";
type RelevancyParams = {
serviceContext?: ServiceContext;
raiseError?: boolean;
evalTemplate?: RelevancyEvalPrompt;
refineTemplate?: RelevancyRefinePrompt;
};
export class RelevancyEvaluator extends PromptMixin implements BaseEvaluator {
private serviceContext?: ServiceContext;
private raiseError: boolean;
private evalTemplate: RelevancyEvalPrompt;
private refineTemplate: RelevancyRefinePrompt;
constructor(params?: RelevancyParams) {
super();
this.serviceContext = params?.serviceContext;
this.raiseError = params?.raiseError ?? false;
this.evalTemplate = params?.evalTemplate ?? defaultRelevancyEvalPrompt;
this.refineTemplate =
params?.refineTemplate ?? defaultRelevancyRefinePrompt;
}
_getPrompts() {
return {
evalTemplate: this.evalTemplate,
refineTemplate: this.refineTemplate,
};
}
_updatePrompts(prompts: {
evalTemplate: RelevancyEvalPrompt;
refineTemplate: RelevancyRefinePrompt;
}): void {
if ("evalTemplate" in prompts) {
this.evalTemplate = prompts["evalTemplate"];
}
if ("refineTemplate" in prompts) {
this.refineTemplate = prompts["refineTemplate"];
}
}
async evaluate({
query,
response,
contexts = [],
sleepTimeInSeconds = 0,
}: EvaluatorParams): Promise<EvaluationResult> {
if (query === null || response === null) {
throw new Error("query, contexts, and response must be provided");
}
await new Promise((resolve) =>
setTimeout(resolve, sleepTimeInSeconds * 1000),
);
const docs = contexts?.map((context) => new Document({ text: context }));
const index = await SummaryIndex.fromDocuments(docs, {
serviceContext: this.serviceContext,
});
const queryResponse = `Question: ${query}\nResponse: ${response}`;
const queryEngine = index.asQueryEngine();
queryEngine.updatePrompts({
"responseSynthesizer:textQATemplate": this.evalTemplate,
"responseSynthesizer:refineTemplate": this.refineTemplate,
});
const responseObj = await queryEngine.query({
query: queryResponse,
});
const rawResponseTxt = responseObj.toString();
let passing: boolean;
if (rawResponseTxt.toLowerCase().includes("yes")) {
passing = true;
} else {
passing = false;
if (this.raiseError) {
throw new Error("The response is invalid");
}
}
return {
query,
contexts,
response,
passing,
score: passing ? 1.0 : 0.0,
feedback: rawResponseTxt,
};
}
/**
* @param query Query to evaluate
* @param response Response to evaluate
*/
async evaluateResponse({
query,
response,
}: EvaluatorResponseParams): Promise<EvaluationResult> {
const responseStr = response?.response;
const contexts = [];
if (response) {
for (const node of response.sourceNodes || []) {
contexts.push(node.node.getContent(MetadataMode.ALL));
}
}
return this.evaluate({
query,
response: responseStr,
contexts,
});
}
}