Extractive: Use '-9e3' instead of '-9e9' to replace padding values in…

… classifiers before loss calculation

src/classifier.py

@@ -62,7 +62,7 @@ def forward(self, x, mask):
         x = self.linear2(x)
         # x = self.sigmoid(x)
         sent_scores = x.squeeze(-1) * mask.float()
-        sent_scores[sent_scores == 0] = -9e9
+        sent_scores[sent_scores == 0] = -9e3
         return sent_scores
 
 
@@ -88,7 +88,7 @@ def forward(self, x, mask):
         x = self.linear(x).squeeze(-1)
         # x = self.sigmoid(x)
         sent_scores = x * mask.float()
-        sent_scores[sent_scores == 0] = -9e9
+        sent_scores[sent_scores == 0] = -9e3
         return sent_scores
 
 
@@ -183,5 +183,5 @@ def forward(self, x, mask):
         # x is shape (batch size, source sequence length, 1)
         # mask is shape (batch size, source sequence length)
         sent_scores = x.squeeze(-1) * mask.float()
-        sent_scores[sent_scores == 0] = -9e9
+        sent_scores[sent_scores == 0] = -9e3
         return sent_scores

src/extractive.py

@@ -197,8 +197,8 @@ def __init__(self, hparams, embedding_model_config=None, classifier_obj=None):
         # trick for numerical stability. Padding values are 0 and if 0 is the input
         # to the sigmoid function the output will be 0.5. This will cause issues when
         # inputs with more padding will have higher loss values. To solve this, all
-        # padding values are set to -9e9 as the last step of each encoder. The sigmoid
-        # function transforms -9e9 to nearly 0, thus preserving the proper loss
+        # padding values are set to -9e3 as the last step of each encoder. The sigmoid
+        # function transforms -9e3 to nearly 0, thus preserving the proper loss
         # calculation. See `compute_loss()` for more info.
         self.loss_func = nn.BCEWithLogitsLoss(reduction="none")