feat(tokenizer): add new tokenizer configurations for SimpleTokenizer…

…, MecabTokenizer, NLTKTokenizer, add new tagger configurations for mecab and nltk

src/corprep/conf/tokenizer/__init__.yaml

@@ -0,0 +1,39 @@
+defaults:
+  - /normalizer: __init__
+  - /stopwords: __init__
+
+_target_: corprep.tokenizer.SimpleTokenizer
+lowercase: false
+flatten: true
+strip_pos: false
+postag_delim: /
+postag_length:
+include_whitespace_token: true
+tokenize_each_word: false
+sentence_separator: '\n'
+userdic_path:
+wordpieces_prefix: "##"
+postags:
+noun_postags:
+  - NNG
+  - NNP
+  - XSN
+  - SL
+  - XR
+  - NNB
+  - NR
+punct_postags:
+  - SF
+  - SP
+  - SSO
+  - SSC
+  - SY
+stop_postags:
+  - SP
+  - SF
+  - SE
+  - SSO
+  - SSC
+  - SC
+  - SY
+  - SH

src/corprep/conf/tokenizer/mecab.yaml

@@ -0,0 +1,22 @@
+defaults:
+  - __init__
+  - tagger: mecab
+
+_target_: corprep.tokenizer.MecabTokenizer
+noun_postags:
+  - NNG
+  - NNP
+  - XSN
+  - SL
+  - XR
+  - NNB
+  - NR
+stop_postags:
+  - SP
+  - SF
+  - SE
+  - SSO
+  - SSC
+  - SC
+  - SY
+  - SH

src/corprep/conf/tokenizer/nltk.yaml

@@ -0,0 +1,12 @@
+defaults:
+  - __init__
+  - tagger: nltk
+
+_target_: corprep.tokenizer.NLTKTokenizer
+noun_postags:
+  - NN
+  - NNP
+  - NNS
+  - NNPS
+stop_postags:
+  - "."

src/corprep/conf/tokenizer/simple.yaml

@@ -0,0 +1,4 @@
+defaults:
+  - __init__
+
+_target_: corprep.tokenizer.SimpleTokenizer

src/corprep/conf/tokenizer/tagger/mecab.yaml

@@ -0,0 +1,3 @@
+userdic_path: ${..userdic_path}
+backend: ekonlpy
+verbose: ${oc.select:..verbose, false}

src/corprep/conf/tokenizer/tagger/nltk.yaml

@@ -0,0 +1,6 @@
+lemmatize: false
+stem: true
+lemmatizer:
+  _target_: nltk.stem.WordNetLemmatizer
+stemmer:
+  _target_: nltk.stem.PorterStemmer