chore: update ngrams

src/cophi/utils.py

@@ -6,6 +6,7 @@
 """
 
 import collections
+import itertools
 
 import pandas as pd
 import regex as re
@@ -18,7 +19,9 @@ def construct_ngrams(tokens, n=2, sep=" "):
         n (int): Number of tokens per ngram.
         sep (str): Separator between tokens within an ngram.
     """
-    return (sep.join(ngram) for ngram in zip(*[tokens[i:] for i in range(n)]))
+    return (sep.join(ngram)
+            for ngram in zip(*(itertools.islice(i, token, None)
+            for token, i in enumerate(itertools.tee(tokens, 2)))))
 
 def find_tokens(document, token_pattern=r"\p{L}+\p{P}?\p{L}+", maximum=None):
     """