fix

namisan · Oct 1, 2020 · f0b40b6 · f0b40b6
1 parent 27c4450
commit f0b40b6
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/alum/adv_masked_lm.py b/alum/adv_masked_lm.py
@@ -99,9 +99,10 @@ def forward(self, model, sample, reduce=True):
                 }
                 return loss, sample_size, logging_output
             # line 6 inner sum
-            newembed = embed + delta_grad * self.args.adv_step_size
+            noise = noise + delta_grad * self.args.adv_step_size
             # line 6 projection
-            newembed = self.adv_project(newembed, norm_type=self.args.project_norm_type, eps=self.args.noise_gamma) 
+            noise = self.adv_project(noise, norm_type=self.args.project_norm_type, eps=self.args.noise_gamma)
+            newembed = embed.data.detach() + noise
             newembed = newembed.detach()
             adv_logits, _ = model(**sample['net_input'], masked_tokens=masked_tokens, task_id=1, embed=newembed, player=0)
             # line 8 symmetric KL