-
Notifications
You must be signed in to change notification settings - Fork 0
/
evaluating.py
64 lines (53 loc) · 1.87 KB
/
evaluating.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
from torch.utils.data import DataLoader
from tqdm import tqdm
from model import BertForSequenceClassification
import numpy as np
from sklearn.metrics import classification_report, f1_score
def make_prediction(
loader: DataLoader,
device: str,
model: BertForSequenceClassification,
) -> np.array:
"""
Calculates predictions for loader.
Args:
loader: DataLoader - from TextClassifierDataset
device: str - `cuda` or `cpu`
model: BertForSequenceClassification - pretrained model
Returns:
array with predictions
size: (bs, 1)
"""
pred_labels = []
for batch in tqdm(loader["test"]):
inputs = batch['features'].to(device)
attention_mask = batch['attention_mask'].to(device)
output = model(inputs, attention_mask).argmax(axis=1).cpu()
pred_labels.append(output)
pred_labels = np.concatenate(pred_labels, axis=0)
return pred_labels
def classification_rep(
loader: DataLoader,
device: str,
model: BertForSequenceClassification,
):
"""
Prints classification report from sklearn for validation data.
Args:
loader: DataLoader - with `valid` key
device: str - `cuda` or `cpu`
model: BertForSequenceClassification - pretrained model
"""
pred_labels = []
true_labels = []
for batch in tqdm(loader["valid"]):
inputs = batch['features'].to(device)
labels = batch['targets'].to(device)
attention_mask = batch['attention_mask'].to(device)
output = model(inputs, attention_mask).argmax(axis=1).cpu()
pred_labels.append(output)
true_labels.append(labels.cpu().numpy())
true_labels = np.concatenate(true_labels, axis=0)
pred_labels = np.concatenate(pred_labels, axis=0)
print(f"\nf1_score: {f1_score(true_labels, pred_labels)}")
print(classification_report(true_labels, pred_labels))