changed default number of processes to prevent bug on MacOS and pytho…

…n 3.8
dataiku · May 12, 2023 · b91eb8e · b91eb8e
1 parent eee133d
commit b91eb8e
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/python-lib/spacy_tokenizer.py b/python-lib/spacy_tokenizer.py
@@ -122,7 +122,8 @@ class MultilingualTokenizer:
 
     DEFAULT_BATCH_SIZE = 1000
     MAX_NUM_CHARACTERS = 10 ** 7
-    DEFAULT_NUM_PROCESS = 2
+    # Set to 1 to prevent pickling issues when spawning multiple processes on MacOS
+    DEFAULT_NUM_PROCESS = 1
     DEFAULT_FILTER_TOKEN_ATTRIBUTES = {
         "is_space": "Whitespace",
         "is_punct": "Punctuation",