fix opencc serialization error (#83)

modelscope · Nov 17, 2023 · 62c5fb5 · 62c5fb5
1 parent 9497ce5
commit 62c5fb5
Showing 1 changed file with 8 additions and 4 deletions.
diff --git a/data_juicer/ops/mapper/chinese_convert_mapper.py b/data_juicer/ops/mapper/chinese_convert_mapper.py
@@ -1,8 +1,12 @@
-import opencc
-
 from ..base_op import OPERATORS, Mapper
 
 
+def prepare_converter(mode):
+    global OPENCC_CONVERTER
+    import opencc
+    OPENCC_CONVERTER = opencc.OpenCC(mode + '.json')
+
+
 @OPERATORS.register_module('chinese_convert_mapper')
 class ChineseConvertMapper(Mapper):
     """Mapper to convert Chinese between Traditional Chinese, Simplified Chinese
@@ -39,9 +43,9 @@ def __init__(self, mode: str = 's2t', *args, **kwargs):
         ]
         assert mode in mode_list, 'Please make sure mode is one of {}'.format(
             mode_list)
-        self.converter = opencc.OpenCC(mode + '.json')
+        prepare_converter(mode)
 
     def process(self, sample):
 
-        sample[self.text_key] = self.converter.convert(sample[self.text_key])
+        sample[self.text_key] = OPENCC_CONVERTER.convert(sample[self.text_key])
         return sample