Skip to content
Permalink
Browse files

fix(preprocessor): add min_len to split preprocessor

  • Loading branch information...
hanxiao committed Aug 29, 2019
1 parent c83448b commit 31bffeb7fa585414f576e6a3f21c6ccbf15a1d33
Showing with 5 additions and 2 deletions.
  1. +5 −2 gnes/preprocessor/text/split.py
@@ -22,11 +22,14 @@


class SentSplitPreprocessor(BaseTextPreprocessor):
def __init__(self, max_sent_len: int = 256,
def __init__(self,
min_sent_len: int = 8,
max_sent_len: int = 256,
deliminator: str = '.!?。!?',
is_json: bool = False,
*args, **kwargs):
super().__init__(*args, **kwargs)
self.min_sent_len = min_sent_len
self.max_sent_len = max_sent_len
self.deliminator = deliminator
self.is_json = is_json
@@ -46,7 +49,7 @@ def apply(self, doc: 'gnes_pb2.Document') -> None:
for ci, (r, s, e) in enumerate(ret):
f = ''.join(filter(lambda x: x in string.printable, r))
f = re.sub('\n+', ' ', f).strip()
if f:
if len(f) > self.min_sent_len:
c = doc.chunks.add()
c.doc_id = doc.doc_id
c.text = f[:self.max_sent_len]

0 comments on commit 31bffeb

Please sign in to comment.
You can’t perform that action at this time.