forked from mongodb/chatbot
-
Notifications
You must be signed in to change notification settings - Fork 0
/
evaluateConversationQuality.ts
79 lines (72 loc) · 2.73 KB
/
evaluateConversationQuality.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
import { ConversationGeneratedData } from "../generate/GeneratedDataStore";
import { EvaluateQualityFunc } from "./EvaluateQualityFunc";
import { strict as assert } from "assert";
import {
ResponseQualityExample,
checkResponseQuality,
} from "./checkResponseQuality";
import { ObjectId, OpenAIClient } from "mongodb-rag-core";
import { EvalResult } from "./EvaluationStore";
import { stringifyConversation } from "./stringifyConversation";
export interface EvaluateConversationQualityParams {
openAiClient: OpenAIClient;
/**
The name of the OpenAI ChatGPT API deployment to use.
@example "gpt-3.5-turbo"
*/
deploymentName: string;
/**
Provide a few examples of conversation transcripts, expected outputs,
and what the LLM output should be.
This is _extremely_ useful for helping the LLM understand
its expected behavior for your use case.
While not strictly necessary, it is highly recommended
to include a few representative examples.
Here, the LLM utilizes a prompting technique called ["few-shot prompting"](https://www.promptingguide.ai/techniques/fewshot.en).
*/
fewShotExamples?: ResponseQualityExample[];
}
/**
Construct a a {@link EvaluateQualityFunc} that evaluates the quality of a conversation
using an OpenAI ChatGPT LLM.
The returned {@link EvalResult} has the following properties:
- In {@link EvalResult.result}, `1` if the conversation meets quality standards and `0` if it does not.
- In {@link EvalResult.metadata}, `reason` for the result, as generated by the LLM.
*/
export function makeEvaluateConversationQuality({
openAiClient,
deploymentName,
}: EvaluateConversationQualityParams): EvaluateQualityFunc {
return async ({ runId, generatedData }) => {
assert(
generatedData.type === "conversation",
"Invalid data type. Expected 'conversation' data."
);
const conversationData = generatedData as ConversationGeneratedData;
const {
data: { messages },
} = conversationData;
const conversationTranscript = stringifyConversation(messages);
const { qualitativeFinalAssistantMessageExpectation } =
conversationData.evalData;
const { meetsChatQualityStandards, reason } = await checkResponseQuality({
deploymentName,
openAiClient,
expectedOutputDescription: qualitativeFinalAssistantMessageExpectation,
received: conversationTranscript,
});
const result = {
_id: new ObjectId(),
generatedDataId: generatedData._id,
commandRunMetadataId: runId,
evalName: "conversation_quality",
result: meetsChatQualityStandards ? 1 : 0,
createdAt: new Date(),
metadata: {
reason,
conversationTranscript,
},
} satisfies EvalResult;
return result;
};
}