commit 1: numerically unstable may exist when computing softmax for M…

…NIST and CIFAR10 model; the solution is referred to section 4.1 of Deep Learning Book by Goodfellow et al.

RawModels/MNISTConv.py

@@ -10,6 +10,7 @@
 import os
 import sys
 
+import torch
 import torch.nn as nn
 import torch.nn.functional as F
 
@@ -64,4 +65,5 @@ def forward(self, x):
         out = self.dropout(out)
         out = F.relu(self.fc2(out))
         out = self.fc3(out)
+        out = out - torch.max(out, dim=1, keepdim=True)[0]
         return out

RawModels/ResNet.py

@@ -8,7 +8,7 @@
 import math
 import os
 import sys
-
+import torch
 import torch.nn as nn
 
 sys.path.append('%s/../' % os.path.dirname(os.path.realpath(__file__)))
@@ -136,6 +136,7 @@ def forward(self, x):
         x = x.view(x.size(0), -1)
         x = self.fc(x)
 
+        x = x - torch.max(x, dim=1, keepdim=True)[0]
         return x