data

allenai · May 23, 2024 · 155319c · 155319c
1 parent 67b3bda
commit 155319c
Showing 1 changed file with 1 addition and 2 deletions.
diff --git a/python/dolma/tokenizer/executor.py b/python/dolma/tokenizer/executor.py
@@ -256,8 +256,7 @@ def __call__(self, num_readers: Optional[int] = None, **process_single_kwargs: A
         )
 
         # finally run the processors
-        fn = self._debug_run_all if self.debug else self._multiprocessing_run_all
-        fn(
+        self._run_all(
             all_source_paths=source_indices,
             all_destination_paths=all_destination_paths,
             all_metadata_paths=all_metadata_path,