fix bug with masking in causal case and contextual keys. encoder / de…

…coder architecture now works
lucidrains · Apr 19, 2020 · 7f2a164 · 7f2a164
1 parent 596847a
commit 7f2a164
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/reformer_pytorch/reformer_pytorch.py b/reformer_pytorch/reformer_pytorch.py
@@ -303,7 +303,9 @@ def look_one_back(x):
 
         # Causal masking
         if self.causal:
-            mask = bq_t[:, :, :, None] < bkv_t[:, :, None, :].clamp(max=query_len - 1)
+            mask = bq_t[:, :, :, None] < bkv_t[:, :, None, :]
+            if seqlen > query_len:
+                mask = mask & (bkv_t[:, :, None, :] < query_len)
             dots.masked_fill_(mask, masked_value)
             del mask
 

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'reformer_pytorch',
   packages = find_packages(exclude=['examples', 'pretraining']),
-  version = '0.20.2',
+  version = '0.21.0',
   license='MIT',
   description = 'Reformer, the Efficient Transformer, Pytorch',
   author = 'Phil Wang',