Merge pull request #98 from SebastinSanty/non-ascii

Account for non-ASCII characters
neulab · Mar 26, 2019 · a7f96e1 · a7f96e1
2 parents c0ade6d + 610b9d2
commit a7f96e1
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/compare_mt/corpus_utils.py b/compare_mt/corpus_utils.py
@@ -1,11 +1,11 @@
 
 def iterate_tokens(filename):
-  with open(filename, "r") as f:
+  with open(filename, "r", encoding="utf-8") as f:
     for line in f:
       yield line.strip().split()
 
 def iterate_nums(filename):
-  with open(filename, "r") as f:
+  with open(filename, "r", encoding="utf-8") as f:
     for line in f:
       yield [float(i) for i in line.strip().split()]