-
Notifications
You must be signed in to change notification settings - Fork 0
/
temperature_scaling.py
66 lines (51 loc) · 2.17 KB
/
temperature_scaling.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import torch
from torch.nn import functional as F
from torch.utils.data import DataLoader
from torchvision import transforms as T
from model import SAT
from util import CocoCaptionDataset
def main():
checkpoint_path = "logs/default/version_131/last.ckpt"
workers = 0
batch = 16
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
model = SAT.load_from_checkpoint(checkpoint_path, map_location=device).to(device)
model.freeze()
valid_transforms = T.Compose([
T.Resize(model.hparams.input_size),
T.CenterCrop(model.hparams.input_size),
T.ToTensor()
])
valid_ds = CocoCaptionDataset(jsonpath=model.hparams.json, split="val", transforms=valid_transforms)
val_loader = DataLoader(dataset=valid_ds, batch_size=batch, num_workers=workers, shuffle=False,
persistent_workers=(True if workers > 0 else False), pin_memory=True)
# Get all the logits and targets
logits, targets = [], []
with torch.no_grad():
for i, batch in enumerate(val_loader):
img, encoded_captions, lengths = batch
img = img.to(device)
lengths = lengths.to(device)
encoded_captions = encoded_captions.to(device)
# Forward pass
logits_packed, targets_packed, _ = model.train_batch([img, encoded_captions, lengths], epsilon=1)
# Keep all predictions and targets in a list
logits.append(logits_packed.data)
targets.append(targets_packed.data)
print(i)
if i>40: break
# Combine all logits and targets into 1 tensor
logits = torch.cat(logits)
targets = torch.cat(targets)
# to() creates a copy, so I detach it to make this a leaf tensor
temperature = (torch.ones(1)*1.5).to(device).detach().requires_grad_(True)
optimizer = torch.optim.SGD([temperature], lr=1e-2, momentum=0.8, nesterov=True)
for i in range(70):
loss = F.cross_entropy(logits/temperature, targets)
print(f"{temperature = }")
loss.backward()
optimizer.step()
optimizer.zero_grad()
print(f"{temperature = }")
if __name__ == "__main__":
main()