-
Notifications
You must be signed in to change notification settings - Fork 1
/
main.py
80 lines (60 loc) · 2.8 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
import numpy as np
import tensorflow as tf
from tensorflow import keras
#from tensorflow.keras import layers
##########################################################################
def softmax(x):
exp_x = np.exp(x)
sm_x = exp_x / np.sum(exp_x, axis=-1, keepdims=True)
return sm_x
def softmax_norm(x):
e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
return e_x / np.sum(e_x, axis=-1, keepdims=True)
def nl(input, target):
return -np.mean(np.log(input[range(target.shape[0]), target]))
##########################################################################
def nll(input, target):
return - np.mean(input[range(target.shape[0]), target])
def log_softmax(logits):
c = logits.max()
logsumexp = np.log(np.sum(np.exp(logits - c), axis=-1, keepdims=True))
return logits - c - logsumexp
################################################################################
x = np.array([[ 0.9826, 1.0630, -0.4096],
[-0.6213, 0.2511, 0.5659],
[ 0.5662, 0.7360, -0.6783],
[-0.4638, -1.4961, -1.0877],
[ 1.8186, -0.2998, 0.1128]])
target = np.array([1, 0, 1, 1, 1])
################################################################################
# SoftMax + NL = CrossEntropy + Loss
pred = softmax(x)
loss = nl(pred, target)
# loss = (1.4904)
################################################################################
# LogSoftMax + NLL = CrossEntropy + Loss
pred2 = log_softmax(x)
loss2 = nll(pred2, target)
# loss = (1.4904)
################################################################################
# tfr.keras.losses.SoftmaxLoss
# Computes Softmax cross-entropy loss between y_TRUE and y_PRED.
pred_k = log_softmax(np.array([0.6, 0.8]))
pred_k = np.multiply([1., 0.], pred_k)
loss = -np.sum(pred_k)
# loss = 0.798138
###############################################################################
#pred_k = log_softmax(np.array([[0.0, 0.6, 0.8], [0.5, 0.8, 0.4]]))
#pred_k = np.multiply(np.array([[0.0, 1., 0.], [0., 1., 0.]]), pred_k)
#loss = -np.sum(pred_k, axis=-1)
def softmax_crossentropy_with_logits(logits, reference_answers):
"""Compute crossentropy from logits[batch, n_classes] and ids of correct answers"""
logits_for_answers = logits[np.arange(len(logits)), reference_answers]
xentropy = - logits_for_answers + np.log(np.sum(np.exp(logits), axis=-1))
return xentropy
def grad_softmax_crossentropy_with_logits(logits, reference_answers):
"""Compute crossentropy gradient from logits[batch, n_classes] and ids of correct answers"""
one_hot = np.zeros_like(logits)
one_hot[np.arange(len(logits)), reference_answers] = 1
softmax = np.exp(logits) / np.exp(logits).sum(axis=-1, keepdims=True)
return (- one_hot + softmax) / logits.shape[0]