Properly skip invalid docs in normalizer

markgw · Jul 8, 2020 · 97a0aa6 · 97a0aa6
1 parent 8fba3d8
commit 97a0aa6
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/python/pimlico/modules/text/normalize/execute.py b/src/python/pimlico/modules/text/normalize/execute.py
@@ -1,5 +1,6 @@
 import re
 
+from pimlico.core.modules.map import skip_invalid
 from pimlico.core.modules.map.multiproc import multiprocessing_executor_factory
 
 
@@ -17,6 +18,7 @@ def worker_setup(worker):
 num_re = re.compile(r"[0-9]")
 
 
+@skip_invalid
 def process_document(worker, archive_name, doc_name, doc):
     sentences = doc.sentences