-
Notifications
You must be signed in to change notification settings - Fork 2.5k
/
includes.py
57 lines (48 loc) · 1.78 KB
/
includes.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
from typing import Any
import evals
import evals.metrics
from evals.api import CompletionFn
from evals.elsuite import utils
class Includes(evals.Eval):
def __init__(
self,
completion_fns: list[CompletionFn],
samples_jsonl: str,
ignore_case: bool = False,
*args,
**kwargs,
):
super().__init__(completion_fns, *args, **kwargs)
assert len(completion_fns) == 1, "Includes only supports one completion fn"
self.samples_jsonl = samples_jsonl
self.ignore_case = ignore_case
def eval_sample(self, sample: Any, *_):
assert isinstance(sample, dict), "sample must be a dict"
assert "input" in sample, "sample must have an 'input' key"
assert "ideal" in sample, "sample must have an 'ideal' key"
prompt = sample["input"]
result = self.completion_fn(
prompt=prompt,
)
sampled = result.get_completions()[0]
ideal = sample["ideal"]
if not isinstance(ideal, list):
ideal = [ideal]
assert isinstance(ideal, list) and all(
isinstance(i, str) for i in ideal
), "ideal must be a list of strings"
includes_answer = any(
[utils.get_answer(sampled, ref, self.ignore_case) is not None for ref in ideal]
)
evals.record.record_match(
includes_answer, expected=sample["ideal"], picked=sampled, sampled=sampled
)
return includes_answer
def run(self, recorder):
samples = self.get_samples()
self.eval_all_samples(recorder, samples)
events = recorder.get_events("match")
return {
"accuracy": evals.metrics.get_accuracy(events),
"boostrap_std": evals.metrics.get_bootstrap_accuracy_std(events),
}